diff --git a/.vscode/launch.json b/.vscode/launch.json index 5500823..8eb9836 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,8 +10,11 @@ "request": "launch", "mode": "debug", "program": "${workspaceRoot}", + "buildFlags": "-race", "args": [ - "-debug", + // "-debug", + "-debug-port", + "6060", "-config", "${workspaceRoot}/config.yaml" ] diff --git a/CHANGELOG.md b/CHANGELOG.md index d39848b..6c77db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.1.1 +* Fix a crash when `archived_project_handling` is set to "hidden" in gitlab +* Refactored caching, laying the groundwork for automatic cache invalidation. + # v1.1.0 * Now default to using a loopback instead of symlinks. This should improve compatibility. diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..7ff77d1 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,90 @@ +package cache + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/badjware/gitforgefs/types" +) + +type Cache struct { + backend types.GitForge + logger *slog.Logger + + rootContentLock sync.RWMutex + cachedRootContent map[string]types.RepositoryGroupSource + + // contentLock sync.RWMutex + // cachedRepositoryGroupSource map[string]*CachedContent + cachedRepositoryGroupSource sync.Map +} + +func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForgeCacher { + return &Cache{ + backend: backend, + logger: logger, + } +} + +type CachedContent struct { + GetContent func() (types.RepositoryGroupContent, error) + creationTime time.Time +} + +func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + c.rootContentLock.RLock() + if c.cachedRootContent == nil { + c.rootContentLock.RUnlock() + + // acquire write lock + c.rootContentLock.Lock() + defer c.rootContentLock.Unlock() + + // check to make sure the data is still not there, + // since RWMutex is not upgradeable and another thread may have grabbed the lock in the meantime + if c.cachedRootContent == nil { + c.logger.Info("Fetching root content from backend") + content, err := c.backend.FetchRootGroupContent(ctx) + if err != nil { + return nil, err + } + c.cachedRootContent = content + } + return c.cachedRootContent, nil + } + c.rootContentLock.RUnlock() + return c.cachedRootContent, nil +} + +func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + logger := c.logger.With("groupID", source.GetGroupID()).With("groupPath", source.GetGroupPath()) + + cachedContent := CachedContent{ + GetContent: sync.OnceValues(func() (types.RepositoryGroupContent, error) { + logger.Info("Fetching content from backend") + return c.backend.FetchGroupContent(ctx, source) + }), + creationTime: time.Now(), + } + actual, loaded := c.cachedRepositoryGroupSource.LoadOrStore(source.GetGroupPath(), &cachedContent) + if loaded { + logger.Debug("Cache hit") + // If already loaded, return the existing cached content or wait for it to be available + return actual.(*CachedContent).GetContent() + } else { + logger.Info("Cache miss") + // Do the actual fetch in the background + content, err := cachedContent.GetContent() + if err != nil { + // If there was an error fetching the content, remove the cache entry to allow for retries + c.cachedRepositoryGroupSource.Delete(source.GetGroupPath()) + } + return content, err + } +} + +func (c *Cache) InvalidateCache(source types.RepositoryGroupSource) { + c.cachedRepositoryGroupSource.Delete(source.GetGroupPath()) +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..b35ab7a --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,142 @@ +package cache_test + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/badjware/gitforgefs/cache" + "github.com/badjware/gitforgefs/types" +) + +type mockRepoGroupSource struct { + ID uint64 + Name string + Path string +} + +func (m mockRepoGroupSource) GetGroupID() uint64 { return m.ID } +func (m mockRepoGroupSource) GetGroupName() string { return m.Name } +func (m mockRepoGroupSource) GetGroupPath() string { return m.Path } + +type mockBackend struct { + RootCalls int + GroupCalls int + + rootContentValue map[string]types.RepositoryGroupSource + groupContentValue types.RepositoryGroupContent +} + +func (m *mockBackend) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + m.RootCalls++ + return m.rootContentValue, nil +} + +func (m *mockBackend) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + m.GroupCalls++ + return m.groupContentValue, nil +} + +func newLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})) +} + +func TestFetchRootGroupContent(t *testing.T) { + backend := &mockBackend{ + rootContentValue: map[string]types.RepositoryGroupSource{ + "g": mockRepoGroupSource{ID: 1, Name: "g", Path: "g"}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + ctx := context.Background() + + result1, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("first FetchRootGroupContent failed: %v", err) + } + if content, ok := result1["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result1) + } + + result2, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("second FetchRootGroupContent failed: %v", err) + } + if content, ok := result2["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result2) + } + + if backend.RootCalls != 1 { + t.Fatalf("expected backend.FetchRootGroupContent to be called once, got %d", backend.RootCalls) + } +} + +func TestFetchGroupContent(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + result1, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + if content, ok := result1.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result1) + } + + result2, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + if content, ok := result2.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result2) + } + + if backend.GroupCalls != 1 { + t.Fatalf("expected backend.FetchGroupContent to be called once, got %d", backend.GroupCalls) + } +} + +func TestInvalidateCache(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + + c.InvalidateCache(src) + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + + if backend.GroupCalls != 2 { + t.Fatalf("expected backend.FetchGroupContent to be called twice, got %d", backend.GroupCalls) + } +} diff --git a/config.example.yaml b/config.example.yaml index ff5e847..0dd7b7d 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -11,7 +11,7 @@ fs: # Use a symlink to point to the real location of the repository instead of doing a loopback # Using symlinks is more performant and allow cloning to be asynchronous, but may cause compatibility issues with some applications - # use_symlinks: false + # use_symlinks: true # The git forge to use as the backend. # Must be one of "gitlab", "github", or "gitea" diff --git a/config/loader.go b/config/loader.go index 456e8d8..014ed71 100644 --- a/config/loader.go +++ b/config/loader.go @@ -90,7 +90,7 @@ func LoadConfig(configPath string) (*Config, error) { FS: FSConfig{ Mountpoint: "", MountOptions: "nodev,nosuid", - UseSymlinks: false, + UseSymlinks: true, Forge: "", }, Gitlab: GitlabClientConfig{ diff --git a/forges/gitea/client.go b/forges/gitea/client.go index b568579..350b47e 100644 --- a/forges/gitea/client.go +++ b/forges/gitea/client.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "log/slog" - "sync" "code.gitea.io/sdk/gitea" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type giteaClient struct { @@ -17,15 +16,8 @@ type giteaClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClient, error) { @@ -40,12 +32,7 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -53,45 +40,41 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - giteaClient.UserNames = append(giteaClient.UserNames, *¤tUser.UserName) + giteaClient.UserNames = append(giteaClient.UserNames, currentUser.UserName) } return giteaClient, nil } -func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GiteaClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - for _, userName := range c.GiteaClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GiteaClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GiteaClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *giteaClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/gitea/organization.go b/forges/gitea/organization.go index e0f5b65..6697326 100644 --- a/forges/gitea/organization.go +++ b/forges/gitea/organization.go @@ -3,51 +3,35 @@ package gitea import ( "context" "fmt" - "sync" + "time" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type Organization struct { - ID int64 - Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource + ID int64 + Name string + LastModified time.Time } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() - - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupName() string { + return o.Name } -func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() +func (o *Organization) GetGroupPath() string { + return o.Name +} - c.logger.Debug("Organization cache miss", "org_name", orgName) - } +func (o *Organization) GetLastModified() time.Time { + return o.LastModified +} - // If not found in cache, fetch organization infos from API +func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { giteaOrg, _, err := c.client.GetOrg(orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) @@ -55,51 +39,43 @@ func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*O newOrg := Organization{ ID: giteaOrg.ID, Name: giteaOrg.UserName, - - childRepositories: nil, } - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - return &newOrg, nil } -func (c *giteaClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the organization repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitea/repository.go b/forges/gitea/repository.go index d2c6cec..ff2c3cb 100644 --- a/forges/gitea/repository.go +++ b/forges/gitea/repository.go @@ -1,8 +1,8 @@ package gitea import ( - "context" "path" + "time" "code.gitea.io/sdk/gitea" "github.com/badjware/gitforgefs/config" @@ -10,7 +10,9 @@ import ( type Repository struct { ID int64 + Name string Path string + LastModified time.Time CloneURL string DefaultBranch string } @@ -19,6 +21,18 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { + return r.Path +} + +func (r *Repository) GetLastModified() time.Time { + return r.LastModified +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,13 +41,15 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *giteaClient) newRepositoryFromGiteaRepository(ctx context.Context, repository *gitea.Repository) *Repository { +func (c *giteaClient) newRepositoryFromGiteaRepository(repository *gitea.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && repository.Archived { return nil } r := Repository{ ID: repository.ID, - Path: repository.Name, + Name: repository.Name, + Path: repository.FullName, + LastModified: repository.Updated, DefaultBranch: repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -45,6 +61,7 @@ func (c *giteaClient) newRepositoryFromGiteaRepository(ctx context.Context, repo r.CloneURL = repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/gitea/user.go b/forges/gitea/user.go index d392355..4627329 100644 --- a/forges/gitea/user.go +++ b/forges/gitea/user.go @@ -3,51 +3,35 @@ package gitea import ( "context" "fmt" - "sync" + "time" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type User struct { - ID int64 - Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource + ID int64 + Name string + LastModified time.Time } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() - - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupName() string { + return u.Name } -func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() +func (u *User) GetGroupPath() string { + return u.Name +} - c.logger.Debug("User cache miss", "user_name", userName) - } +func (u *User) GetLastModified() time.Time { + return u.LastModified +} - // If not found in cache, fetch user infos from API +func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, error) { giteaUser, _, err := c.client.GetUserInfo(userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) @@ -55,51 +39,43 @@ func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, er newUser := User{ ID: giteaUser.ID, Name: giteaUser.UserName, - - childRepositories: nil, } - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - return &newUser, nil } -func (c *giteaClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the user repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/client.go b/forges/github/client.go index a95827f..5f08ba4 100644 --- a/forges/github/client.go +++ b/forges/github/client.go @@ -2,12 +2,10 @@ package github import ( "context" - "fmt" "log/slog" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) @@ -17,15 +15,8 @@ type githubClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubClient, error) { @@ -40,12 +31,7 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -59,39 +45,35 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl return gitHubClient, nil } -func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GithubClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - for _, userName := range c.GithubClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GithubClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GithubClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *githubClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *githubClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/github/organization.go b/forges/github/organization.go index bef3ce2..693f42a 100644 --- a/forges/github/organization.go +++ b/forges/github/organization.go @@ -3,103 +3,78 @@ package github import ( "context" "fmt" - "sync" + "time" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type Organization struct { - ID int64 - Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource + ID int64 + Name string + LastModified time.Time } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() - - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupName() string { + return o.Name } -func (c *githubClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() +func (o *Organization) GetGroupPath() string { + return o.Name +} - c.logger.Debug("Organization cache miss", "org_name", orgName) - } +func (o *Organization) GetLastModified() time.Time { + return o.LastModified +} - // If not found in cache, fetch organization infos from API +func (c *githubClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { githubOrg, _, err := c.client.Organizations.Get(ctx, orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) } - newOrg := Organization{ - ID: *githubOrg.ID, - Name: *githubOrg.Login, - - childRepositories: nil, - } - - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - - return &newOrg, nil + return &Organization{ + ID: *githubOrg.ID, + Name: *githubOrg.Login, + LastModified: githubOrg.UpdatedAt.Time, + }, nil } -func (c *githubClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - repositoryListOpt := &github.RepositoryListByOrgOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the organization repositories + repositoryListOpt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } - } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/repository.go b/forges/github/repository.go index ecfe720..ebafbca 100644 --- a/forges/github/repository.go +++ b/forges/github/repository.go @@ -1,8 +1,8 @@ package github import ( - "context" "path" + "time" "github.com/badjware/gitforgefs/config" "github.com/google/go-github/v63/github" @@ -10,7 +10,9 @@ import ( type Repository struct { ID int64 + Name string Path string + LastModified time.Time CloneURL string DefaultBranch string } @@ -19,6 +21,18 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { + return r.Path +} + +func (r *Repository) GetLastModified() time.Time { + return r.LastModified +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,13 +41,15 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *githubClient) newRepositoryFromGithubRepository(ctx context.Context, repository *github.Repository) *Repository { +func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && *repository.Archived { return nil } r := Repository{ ID: *repository.ID, - Path: *repository.Name, + Name: *repository.Name, + Path: *repository.FullName, + LastModified: repository.UpdatedAt.Time, DefaultBranch: *repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -45,6 +61,7 @@ func (c *githubClient) newRepositoryFromGithubRepository(ctx context.Context, re r.CloneURL = *repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && *repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/github/user.go b/forges/github/user.go index 5e16b90..2593f56 100644 --- a/forges/github/user.go +++ b/forges/github/user.go @@ -3,103 +3,78 @@ package github import ( "context" "fmt" - "sync" + "time" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type User struct { - ID int64 - Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource + ID int64 + Name string + LastModified time.Time } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() - - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupName() string { + return u.Name } -func (c *githubClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() +func (u *User) GetGroupPath() string { + return u.Name +} - c.logger.Debug("User cache miss", "user_name", userName) - } +func (u *User) GetLastModified() time.Time { + return u.LastModified +} - // If not found in cache, fetch user infos from API +func (c *githubClient) fetchUser(ctx context.Context, userName string) (*User, error) { githubUser, _, err := c.client.Users.Get(ctx, userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) } - newUser := User{ - ID: *githubUser.ID, - Name: *githubUser.Login, - - childRepositories: nil, - } - - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - - return &newUser, nil + return &User{ + ID: *githubUser.ID, + Name: *githubUser.Login, + LastModified: githubUser.UpdatedAt.Time, + }, nil } -func (c *githubClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - repositoryListOpt := &github.RepositoryListByUserOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the user repositories + repositoryListOpt := &github.RepositoryListByUserOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } - } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitlab/client.go b/forges/gitlab/client.go index 1fb42e9..52e4eeb 100644 --- a/forges/gitlab/client.go +++ b/forges/gitlab/client.go @@ -4,11 +4,9 @@ import ( "context" "fmt" "log/slog" - "slices" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) @@ -18,15 +16,8 @@ type gitlabClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - userIDs []int - - // API response cache - groupCacheMux sync.RWMutex - groupCache map[int]*Group - userCacheMux sync.RWMutex - userCache map[int]*User + // use a map without values for efficient lookups + users map[string]int } func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabClient, error) { @@ -44,12 +35,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl logger: logger, - rootContent: nil, - - userIDs: []int{}, - - groupCache: map[int]*Group{}, - userCache: map[int]*User{}, + users: make(map[string]int), } // Fetch current user and add it to the list @@ -57,7 +43,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, currentUser.ID) + gitlabClient.users[currentUser.Username] = currentUser.ID } // Fetch the configured users and add them to the list @@ -66,54 +52,39 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil || len(user) != 1 { logger.Warn("failed to fetch the user", "userName", userName, "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, user[0].ID) + gitlabClient.users[userName] = user[0].ID } } return gitlabClient, nil } -func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - // use cached values if available - if c.rootContent == nil { - rootGroupCache := make(map[string]fstree.GroupSource) +func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - // fetch root groups - for _, gid := range c.GroupIDs { - group, err := c.fetchGroup(ctx, gid) - if err != nil { - return nil, err - } - rootGroupCache[group.Name] = group + // fetch root groups + for _, gid := range c.GroupIDs { + group, err := c.fetchGroup(ctx, gid) + if err != nil { + return nil, err } - // fetch users - for _, uid := range c.userIDs { - user, err := c.fetchUser(ctx, uid) - if err != nil { - return nil, err - } - rootGroupCache[user.Name] = user + rootContent[group.Name] = group + } + // fetch users + for _, uid := range c.users { + user, err := c.fetchUser(ctx, uid) + if err != nil { + return nil, err } - - c.rootContent = rootGroupCache + rootContent[user.Name] = user } - return c.rootContent, nil + return rootContent, nil } -func (c *gitlabClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if slices.Contains[[]int, int](c.userIDs, int(gid)) { - // gid is a user - user, err := c.fetchUser(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchUserContent(ctx, user) +func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, int(source.GetGroupID())) } else { - // gid is a group - group, err := c.fetchGroup(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchGroupContent(ctx, group) + return c.fetchGroupContent(ctx, int(source.GetGroupID())) } } diff --git a/forges/gitlab/group.go b/forges/gitlab/group.go index 9ee48d5..8ada7fa 100644 --- a/forges/gitlab/group.go +++ b/forges/gitlab/group.go @@ -3,180 +3,112 @@ package gitlab import ( "context" "fmt" - "sync" + "time" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type Group struct { - ID int - Name string - - gitlabClient *gitlabClient - - mux sync.Mutex - - // hold group content - childGroups map[string]fstree.GroupSource - childProjects map[string]fstree.RepositorySource + ID int + Name string + Path string + LastModified time.Time } func (g *Group) GetGroupID() uint64 { return uint64(g.ID) } -func (g *Group) InvalidateContentCache() { - g.mux.Lock() - defer g.mux.Unlock() +func (g *Group) GetGroupName() string { + return g.Name +} - // clear child group from cache - g.gitlabClient.groupCacheMux.Lock() - for _, childGroup := range g.childGroups { - gid := int(childGroup.GetGroupID()) - delete(g.gitlabClient.groupCache, gid) - } - g.gitlabClient.groupCacheMux.Unlock() - g.childGroups = nil +func (g *Group) GetGroupPath() string { + return g.Path +} - // clear child repositories from cache - g.childGroups = nil +func (g *Group) GetLastModified() time.Time { + return g.LastModified } -func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { - // start by searching the cache - // TODO: cache invalidation? - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; fetching group", "gid", gid) +func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) *Group { + lastModified := time.Time{} + if gitlabGroup.CreatedAt != nil { + lastModified = *gitlabGroup.CreatedAt } + return &Group{ + ID: gitlabGroup.ID, + Name: gitlabGroup.Path, + Path: gitlabGroup.FullPath, + LastModified: lastModified, + } +} - // If not in cache, fetch group infos from API +func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { gitlabGroup, _, err := c.client.Groups.GetGroup(gid, &gitlab.GetGroupOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err) } c.logger.Debug("Fetched group", "gid", gid) - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, - } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil + return c.newGroupFromGitlabGroup(gitlabGroup), nil } -func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) (*Group, error) { - gid := gitlabGroup.ID - - // start by searching the cache - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; registering group", "gid", gid) +func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.RepositoryGroupContent, error) { + childGroups := make(map[string]types.RepositoryGroupSource) + childProjects := make(map[string]types.RepositorySource) + + // List subgroups in path + listGroupsOpt := &gitlab.ListSubGroupsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }, + AllAvailable: gitlab.Ptr(true), } - - // if not found in cache, convert and save to cache now - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, - } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil -} - -func (c *gitlabClient) fetchGroupContent(ctx context.Context, group *Group) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the group content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - group.mux.Lock() - defer group.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if group.childGroups == nil || group.childProjects == nil { - childGroups := make(map[string]fstree.GroupSource) - childProjects := make(map[string]fstree.RepositorySource) - - // List subgroups in path - listGroupsOpt := &gitlab.ListSubGroupsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }, - AllAvailable: gitlab.Ptr(true), + for { + gitlabGroups, response, err := c.client.Groups.ListSubGroups(gid, listGroupsOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch groups in gitlab: %v", err) } - for { - gitlabGroups, response, err := c.client.Groups.ListSubGroups(group.ID, listGroupsOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err) - } - for _, gitlabGroup := range gitlabGroups { - group, _ := c.newGroupFromGitlabGroup(gitlabGroup) - childGroups[group.Name] = group + for _, gitlabGroup := range gitlabGroups { + group := c.newGroupFromGitlabGroup(gitlabGroup) + if group != nil { + childGroups[group.GetGroupName()] = group } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - listGroupsOpt.Page = response.NextPage } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listGroupsOpt.Page = response.NextPage + } - // List projects in path - listProjectOpt := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break + // List projects in path + listProjectOpt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Groups.ListGroupProjects(gid, listProjectOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - group.childGroups = childGroups - group.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return group.childGroups, group.childProjects, nil + return types.RepositoryGroupContent{ + Groups: childGroups, + Repositories: childProjects, + }, nil } diff --git a/forges/gitlab/project.go b/forges/gitlab/project.go index 2587265..8b76293 100644 --- a/forges/gitlab/project.go +++ b/forges/gitlab/project.go @@ -1,8 +1,8 @@ package gitlab import ( - "context" "path" + "time" "github.com/badjware/gitforgefs/config" gitlab "gitlab.com/gitlab-org/api/client-go" @@ -10,7 +10,9 @@ import ( type Project struct { ID int + Name string Path string + LastModified time.Time CloneURL string DefaultBranch string } @@ -19,6 +21,18 @@ func (p *Project) GetRepositoryID() uint64 { return uint64(p.ID) } +func (p *Project) GetRepositoryName() string { + return p.Name +} + +func (p *Project) GetRepositoryPath() string { + return p.Path +} + +func (p *Project) GetLastModified() time.Time { + return p.LastModified +} + func (p *Project) GetCloneURL() string { return p.CloneURL } @@ -27,14 +41,20 @@ func (p *Project) GetDefaultBranch() string { return p.DefaultBranch } -func (c *gitlabClient) newProjectFromGitlabProject(ctx context.Context, project *gitlab.Project) *Project { +func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) *Project { // https://godoc.org/github.com/xanzy/go-gitlab#Project if c.ArchivedProjectHandling == config.ArchivedProjectIgnore && project.Archived { return nil } + lastModified := time.Time{} + if project.UpdatedAt != nil { + lastModified = *project.UpdatedAt + } p := Project{ ID: project.ID, - Path: project.Path, + Name: project.Path, + Path: project.PathWithNamespace, + LastModified: lastModified, DefaultBranch: project.DefaultBranch, } if p.DefaultBranch == "" { @@ -46,6 +66,7 @@ func (c *gitlabClient) newProjectFromGitlabProject(ctx context.Context, project p.CloneURL = project.HTTPURLToRepo } if c.ArchivedProjectHandling == config.ArchivedProjectHide && project.Archived { + p.Name = "." + p.Name p.Path = path.Join(path.Dir(p.Path), "."+path.Base(p.Path)) } return &p diff --git a/forges/gitlab/user.go b/forges/gitlab/user.go index 726e96b..d729031 100644 --- a/forges/gitlab/user.go +++ b/forges/gitlab/user.go @@ -3,105 +3,78 @@ package gitlab import ( "context" "fmt" - "sync" + "time" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type User struct { - ID int - Name string - - mux sync.Mutex - - // hold user content - childProjects map[string]fstree.RepositorySource + ID int + Name string + LastModified time.Time } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() +func (u *User) GetGroupName() string { + return u.Name +} - // clear child repositories from cache - u.childProjects = nil +func (u *User) GetGroupPath() string { + return u.Name } -func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { - // start by searching the cache - // TODO: cache invalidation? - c.userCacheMux.RLock() - user, found := c.userCache[uid] - c.userCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "uid", uid) - return user, nil - } else { - c.logger.Debug("User cache miss", "uid", uid) - } +func (u *User) GetLastModified() time.Time { + return u.LastModified +} - // If not found in cache, fetch group infos from API +func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { gitlabUser, _, err := c.client.Users.GetUser(uid, gitlab.GetUsersOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch user with id %v: %v", uid, err) } - newUser := User{ - ID: gitlabUser.ID, - Name: gitlabUser.Username, - - childProjects: nil, + lastModified := time.Time{} + if gitlabUser.CreatedAt != nil { + lastModified = *gitlabUser.CreatedAt } - - // save in cache - c.userCacheMux.Lock() - c.userCache[uid] = &newUser - c.userCacheMux.Unlock() - - return &newUser, nil + return &User{ + ID: gitlabUser.ID, + Name: gitlabUser.Username, + LastModified: lastModified, + }, nil } -func (c *gitlabClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the user content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - user.mux.Lock() - defer user.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if user.childProjects == nil { - childProjects := make(map[string]fstree.RepositorySource) - - // Fetch the user repositories - listProjectOpt := &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break +func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.RepositoryGroupContent, error) { + childProjects := make(map[string]types.RepositorySource) + + // Fetch the user repositories + listProjectOpt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Projects.ListUserProjects(uid, listProjectOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - user.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childProjects, nil + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: childProjects, + }, nil } diff --git a/fstree/group.go b/fstree/group.go index b58d874..9e16717 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -4,34 +4,29 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) -const ( - groupBaseInode = 1_000_000_000 -) - type groupNode struct { fs.Inode param *FSParam - source GroupSource + source types.RepositoryGroupSource staticNodes map[string]staticNode } -type GroupSource interface { - GetGroupID() uint64 - InvalidateContentCache() -} - // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSParam) (fs.InodeEmbedder, error) { +// Ensure we are implementing the NodeGetattrer interface +var _ = (fs.NodeGetattrer)((*groupNode)(nil)) + +func newGroupNodeFromSource(ctx context.Context, source types.RepositoryGroupSource, param *FSParam) (fs.InodeEmbedder, error) { node := &groupNode{ param: param, source: source, @@ -43,30 +38,27 @@ func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSPa } func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } - entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes)) - for groupName, group := range groups { + entries := make([]fuse.DirEntry, 0, len(content.Groups)+len(content.Repositories)+len(n.staticNodes)) + for groupName := range content.Groups { entries = append(entries, fuse.DirEntry{ Name: groupName, - Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, }) } - for repositoryName, repository := range repositories { + for repositoryName := range content.Repositories { entries = append(entries, fuse.DirEntry{ Name: repositoryName, - Ino: repository.GetRepositoryID() + repositoryBaseInode, Mode: fuse.S_IFLNK, }) } for name, staticNode := range n.staticNodes { entries = append(entries, fuse.DirEntry{ Name: name, - Ino: staticNode.Ino(), Mode: staticNode.Mode(), }) } @@ -74,15 +66,14 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } else { // Check if the map of groups contains it - group, found := groups[name] + group, found := content.Groups[name] if found { attrs := fs.StableAttr{ - Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, } groupNode, _ := newGroupNodeFromSource(ctx, group, n.param) @@ -90,11 +81,9 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } // Check if the map of projects contains it - repository, found := repositories[name] + repository, found := content.Repositories[name] if found { - attrs := fs.StableAttr{ - Ino: repository.GetRepositoryID() + repositoryBaseInode, - } + attrs := fs.StableAttr{} if n.param.UseSymlinks { attrs.Mode = fuse.S_IFLNK } else { @@ -113,7 +102,6 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) staticNode, ok := n.staticNodes[name] if ok { attrs := fs.StableAttr{ - Ino: staticNode.Ino(), Mode: staticNode.Mode(), } return n.NewInode(ctx, staticNode, attrs), 0 @@ -122,3 +110,8 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return nil, syscall.ENOENT } + +func (n *groupNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Mtime = uint64(n.source.GetLastModified().Unix()) + return 0 +} diff --git a/fstree/refresh.go b/fstree/refresh.go index 25671e9..157459b 100644 --- a/fstree/refresh.go +++ b/fstree/refresh.go @@ -4,15 +4,17 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) type refreshNode struct { fs.Inode - ino uint64 + ino uint64 + param *FSParam - source GroupSource + source types.RepositoryGroupSource } // Ensure we are implementing the NodeSetattrer interface @@ -21,17 +23,14 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil)) // Ensure we are implementing the NodeOpener interface var _ = (fs.NodeOpener)((*refreshNode)(nil)) -func newRefreshNode(source GroupSource, param *FSParam) *refreshNode { +func newRefreshNode(source types.RepositoryGroupSource, param *FSParam) *refreshNode { return &refreshNode{ ino: 0, source: source, + param: param, } } -func (n *refreshNode) Ino() uint64 { - return n.ino -} - func (n *refreshNode) Mode() uint32 { return fuse.S_IFREG } @@ -41,6 +40,6 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se } func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - n.source.InvalidateContentCache() + n.param.Backend.InvalidateCache(n.source) return nil, 0, 0 } diff --git a/fstree/repository.go b/fstree/repository.go index ce1f49f..0929981 100644 --- a/fstree/repository.go +++ b/fstree/repository.go @@ -8,31 +8,25 @@ import ( "syscall" "time" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" -) - -const ( - repositoryBaseInode = 2_000_000_000 + "github.com/hanwen/go-fuse/v2/fuse" ) type repositorySymlinkNode struct { fs.Inode param *FSParam - source RepositorySource -} - -type RepositorySource interface { - // GetName() string - GetRepositoryID() uint64 - GetCloneURL() string - GetDefaultBranch() string + source types.RepositorySource } -// Ensure we are implementing the NodeReaddirer interface +// Ensure we are implementing the NodeReadlinker interface var _ = (fs.NodeReadlinker)((*repositorySymlinkNode)(nil)) -func newRepositoryNodeFromSource(ctx context.Context, source RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { +// Ensure we are implementing the NodeGetattrer interface +var _ = (fs.NodeGetattrer)((*repositorySymlinkNode)(nil)) + +func newRepositoryNodeFromSource(ctx context.Context, source types.RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { if param.UseSymlinks { return &repositorySymlinkNode{ param: param, @@ -74,3 +68,8 @@ func (n *repositorySymlinkNode) Readlink(ctx context.Context) ([]byte, syscall.E } return []byte(localRepositoryPath), 0 } + +func (n *repositorySymlinkNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Mtime = uint64(n.source.GetLastModified().Unix()) + return 0 +} diff --git a/fstree/root.go b/fstree/root.go index c534e59..9737626 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -8,30 +8,21 @@ import ( "os/signal" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) type staticNode interface { fs.InodeEmbedder - Ino() uint64 Mode() uint32 } -type GitClient interface { - FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) -} - -type GitForge interface { - FetchRootGroupContent(ctx context.Context) (map[string]GroupSource, error) - FetchGroupContent(ctx context.Context, gid uint64) (map[string]GroupSource, map[string]RepositorySource, error) -} - type FSParam struct { UseSymlinks bool - GitClient GitClient - GitForge GitForge + GitClient types.GitClient + Backend types.GitForgeCacher logger *slog.Logger } @@ -71,7 +62,7 @@ func Start(logger *slog.Logger, mountpoint string, mountoptions []string, param } func (n *rootNode) OnAdd(ctx context.Context) { - rootGroups, err := n.param.GitForge.FetchRootGroupContent(ctx) + rootGroups, err := n.param.Backend.FetchRootGroupContent(ctx) if err != nil { panic(err) } @@ -82,7 +73,6 @@ func (n *rootNode) OnAdd(ctx context.Context) { ctx, groupNode, fs.StableAttr{ - Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, }, ) diff --git a/git/client.go b/git/client.go index 2532df0..bf060d7 100644 --- a/git/client.go +++ b/git/client.go @@ -10,8 +10,8 @@ import ( "strconv" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/queue" + "github.com/badjware/gitforgefs/types" "github.com/badjware/gitforgefs/utils" ) @@ -29,7 +29,7 @@ type gitClient struct { queue queue.TaskQueue } -func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error) { +func NewClient(logger *slog.Logger, p config.GitClientConfig) (types.GitClient, error) { // Create the client c := &gitClient{ GitClientConfig: p, @@ -62,7 +62,7 @@ func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error return c, nil } -func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source fstree.RepositorySource) (localRepoLoc string, err error) { +func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source types.RepositorySource) (localRepoLoc string, err error) { rid := source.GetRepositoryID() cloneUrl := source.GetCloneURL() defaultBranch := source.GetDefaultBranch() diff --git a/main.go b/main.go index a66a81d..365c17a 100644 --- a/main.go +++ b/main.go @@ -4,21 +4,26 @@ import ( "flag" "fmt" "log/slog" + "net/http" + _ "net/http/pprof" "os" "strings" + "github.com/badjware/gitforgefs/cache" "github.com/badjware/gitforgefs/config" "github.com/badjware/gitforgefs/forges/gitea" "github.com/badjware/gitforgefs/forges/github" "github.com/badjware/gitforgefs/forges/gitlab" "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/git" + "github.com/badjware/gitforgefs/types" ) func main() { configPath := flag.String("config", "config.yaml", "The config file") mountoptionsFlag := flag.String("o", "", "Filesystem mount options. See mount.fuse(8)") debug := flag.Bool("debug", false, "Enable debug logging") + debugPort := flag.Int("debug-port", 0, "Listen port for debug server. If 0, server is disabled") flag.Usage = func() { fmt.Println("USAGE:") @@ -45,6 +50,17 @@ func main() { Level: level, })) + // start pprof server if debug port is set + if *debugPort != 0 { + go func() { + logger.Info("Starting debug server", "port", *debugPort) + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", *debugPort), nil) + if err != nil { + logger.Error("debug server failed", "error", err) + } + }() + } + // Configure mountpoint mountpoint := loadedConfig.FS.Mountpoint if flag.NArg() == 1 { @@ -74,7 +90,8 @@ func main() { } gitClient, _ := git.NewClient(logger, *gitClientParam) - var gitForgeClient fstree.GitForge + // setup backend + var gitForgeClient types.GitForge if loadedConfig.FS.Forge == config.ForgeGitlab { // Create the gitlab client gitlabClientConfig, err := config.MakeGitlabConfig(loadedConfig) @@ -101,6 +118,9 @@ func main() { gitForgeClient, _ = gitea.NewClient(logger, *giteaClientConfig) } + // setup cache + cache := cache.NewForgeCache(gitForgeClient, logger) + // Start the filesystem err = fstree.Start( logger, @@ -109,7 +129,7 @@ func main() { &fstree.FSParam{ UseSymlinks: loadedConfig.FS.UseSymlinks, GitClient: gitClient, - GitForge: gitForgeClient, + Backend: cache, }, *debug, ) diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..b319965 --- /dev/null +++ b/types/types.go @@ -0,0 +1,45 @@ +package types + +import ( + "context" + "time" +) + +type GitClient interface { + FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) +} + +type GitForge interface { + FetchRootGroupContent(ctx context.Context) (map[string]RepositoryGroupSource, error) + FetchGroupContent(ctx context.Context, source RepositoryGroupSource) (RepositoryGroupContent, error) +} + +type GitForgeCacher interface { + GitForge + InvalidateCache(source RepositoryGroupSource) +} + +type RepositoryGroupSource interface { + GetGroupID() uint64 + GetGroupName() string + GetGroupPath() string + GetLastModified() time.Time +} + +type RepositorySource interface { + GetRepositoryID() uint64 + GetRepositoryName() string + GetRepositoryPath() string + GetLastModified() time.Time + GetCloneURL() string + GetDefaultBranch() string +} + +type RepositoryGroupContent struct { + // a map of the subgroups contained in this group, keyed by group name + // must not be nil + Groups map[string]RepositoryGroupSource + // a map of the repositories contained in this group, keyed by repository name + // must not be nil + Repositories map[string]RepositorySource +}