v1.1.1

.vscode/launch.json

@@ -10,8 +10,11 @@
             "request": "launch",
             "mode": "debug",
             "program": "${workspaceRoot}",
+            "buildFlags": "-race",
             "args": [
-                "-debug",
+                // "-debug",
+                "-debug-port",
+                "6060",
                 "-config",
                 "${workspaceRoot}/config.yaml"
             ]

CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.1.1
+* Fix a crash when `archived_project_handling` is set to "hidden" in gitlab
+* Refactored caching, laying the groundwork for automatic cache invalidation.
+
 # v1.1.0
 
 * Now default to using a loopback instead of symlinks. This should improve compatibility.

cache/cache.go

@@ -0,0 +1,90 @@
+package cache
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/badjware/gitforgefs/types"
+)
+
+type Cache struct {
+	backend types.GitForge
+	logger  *slog.Logger
+
+	rootContentLock   sync.RWMutex
+	cachedRootContent map[string]types.RepositoryGroupSource
+
+	// contentLock   sync.RWMutex
+	// cachedRepositoryGroupSource map[string]*CachedContent
+	cachedRepositoryGroupSource sync.Map
+}
+
+func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForgeCacher {
+	return &Cache{
+		backend: backend,
+		logger:  logger,
+	}
+}
+
+type CachedContent struct {
+	GetContent   func() (types.RepositoryGroupContent, error)
+	creationTime time.Time
+}
+
+func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) {
+	c.rootContentLock.RLock()
+	if c.cachedRootContent == nil {
+		c.rootContentLock.RUnlock()
+
+		// acquire write lock
+		c.rootContentLock.Lock()
+		defer c.rootContentLock.Unlock()
+
+		// check to make sure the data is still not there,
+		// since RWMutex is not upgradeable and another thread may have grabbed the lock in the meantime
+		if c.cachedRootContent == nil {
+			c.logger.Info("Fetching root content from backend")
+			content, err := c.backend.FetchRootGroupContent(ctx)
+			if err != nil {
+				return nil, err
+			}
+			c.cachedRootContent = content
+		}
+		return c.cachedRootContent, nil
+	}
+	c.rootContentLock.RUnlock()
+	return c.cachedRootContent, nil
+}
+
+func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) {
+	logger := c.logger.With("groupID", source.GetGroupID()).With("groupPath", source.GetGroupPath())
+
+	cachedContent := CachedContent{
+		GetContent: sync.OnceValues(func() (types.RepositoryGroupContent, error) {
+			logger.Info("Fetching content from backend")
+			return c.backend.FetchGroupContent(ctx, source)
+		}),
+		creationTime: time.Now(),
+	}
+	actual, loaded := c.cachedRepositoryGroupSource.LoadOrStore(source.GetGroupPath(), &cachedContent)
+	if loaded {
+		logger.Debug("Cache hit")
+		// If already loaded, return the existing cached content or wait for it to be available
+		return actual.(*CachedContent).GetContent()
+	} else {
+		logger.Info("Cache miss")
+		// Do the actual fetch in the background
+		content, err := cachedContent.GetContent()
+		if err != nil {
+			// If there was an error fetching the content, remove the cache entry to allow for retries
+			c.cachedRepositoryGroupSource.Delete(source.GetGroupPath())
+		}
+		return content, err
+	}
+}
+
+func (c *Cache) InvalidateCache(source types.RepositoryGroupSource) {
+	c.cachedRepositoryGroupSource.Delete(source.GetGroupPath())
+}

cache/cache_test.go

@@ -0,0 +1,142 @@
+package cache_test
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+
+	"github.com/badjware/gitforgefs/cache"
+	"github.com/badjware/gitforgefs/types"
+)
+
+type mockRepoGroupSource struct {
+	ID   uint64
+	Name string
+	Path string
+}
+
+func (m mockRepoGroupSource) GetGroupID() uint64   { return m.ID }
+func (m mockRepoGroupSource) GetGroupName() string { return m.Name }
+func (m mockRepoGroupSource) GetGroupPath() string { return m.Path }
+
+type mockBackend struct {
+	RootCalls  int
+	GroupCalls int
+
+	rootContentValue  map[string]types.RepositoryGroupSource
+	groupContentValue types.RepositoryGroupContent
+}
+
+func (m *mockBackend) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) {
+	m.RootCalls++
+	return m.rootContentValue, nil
+}
+
+func (m *mockBackend) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) {
+	m.GroupCalls++
+	return m.groupContentValue, nil
+}
+
+func newLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{}))
+}
+
+func TestFetchRootGroupContent(t *testing.T) {
+	backend := &mockBackend{
+		rootContentValue: map[string]types.RepositoryGroupSource{
+			"g": mockRepoGroupSource{ID: 1, Name: "g", Path: "g"},
+		},
+	}
+
+	logger := newLogger()
+	c := cache.NewForgeCache(backend, logger)
+
+	ctx := context.Background()
+
+	result1, err := c.FetchRootGroupContent(ctx)
+	if err != nil {
+		t.Fatalf("first FetchRootGroupContent failed: %v", err)
+	}
+	if content, ok := result1["g"]; !ok || content.GetGroupID() != 1 {
+		t.Fatalf("unexpected root content fetched: %v", result1)
+	}
+
+	result2, err := c.FetchRootGroupContent(ctx)
+	if err != nil {
+		t.Fatalf("second FetchRootGroupContent failed: %v", err)
+	}
+	if content, ok := result2["g"]; !ok || content.GetGroupID() != 1 {
+		t.Fatalf("unexpected root content fetched: %v", result2)
+	}
+
+	if backend.RootCalls != 1 {
+		t.Fatalf("expected backend.FetchRootGroupContent to be called once, got %d", backend.RootCalls)
+	}
+}
+
+func TestFetchGroupContent(t *testing.T) {
+	backend := &mockBackend{
+		groupContentValue: types.RepositoryGroupContent{
+			Groups:       map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}},
+			Repositories: map[string]types.RepositorySource{},
+		},
+	}
+
+	logger := newLogger()
+	c := cache.NewForgeCache(backend, logger)
+
+	src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"}
+
+	ctx := context.Background()
+
+	result1, err := c.FetchGroupContent(ctx, src)
+	if err != nil {
+		t.Fatalf("first FetchGroupContent failed: %v", err)
+	}
+	if content, ok := result1.Groups["group2"]; !ok || content.GetGroupID() != 2 {
+		t.Fatalf("unexpected group content fetched: %v", result1)
+	}
+
+	result2, err := c.FetchGroupContent(ctx, src)
+	if err != nil {
+		t.Fatalf("second FetchGroupContent failed: %v", err)
+	}
+	if content, ok := result2.Groups["group2"]; !ok || content.GetGroupID() != 2 {
+		t.Fatalf("unexpected group content fetched: %v", result2)
+	}
+
+	if backend.GroupCalls != 1 {
+		t.Fatalf("expected backend.FetchGroupContent to be called once, got %d", backend.GroupCalls)
+	}
+}
+
+func TestInvalidateCache(t *testing.T) {
+	backend := &mockBackend{
+		groupContentValue: types.RepositoryGroupContent{
+			Groups:       map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}},
+			Repositories: map[string]types.RepositorySource{},
+		},
+	}
+
+	logger := newLogger()
+	c := cache.NewForgeCache(backend, logger)
+
+	src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"}
+
+	ctx := context.Background()
+
+	if _, err := c.FetchGroupContent(ctx, src); err != nil {
+		t.Fatalf("first FetchGroupContent failed: %v", err)
+	}
+
+	c.InvalidateCache(src)
+
+	if _, err := c.FetchGroupContent(ctx, src); err != nil {
+		t.Fatalf("second FetchGroupContent failed: %v", err)
+	}
+
+	if backend.GroupCalls != 2 {
+		t.Fatalf("expected backend.FetchGroupContent to be called twice, got %d", backend.GroupCalls)
+	}
+}

config.example.yaml

@@ -11,7 +11,7 @@ fs:
 
   # Use a symlink to point to the real location of the repository instead of doing a loopback
   # Using symlinks is more performant and allow cloning to be asynchronous, but may cause compatibility issues with some applications
-  # use_symlinks: false
+  # use_symlinks: true
 
   # The git forge to use as the backend.
   # Must be one of "gitlab", "github", or "gitea"

config/loader.go

@@ -90,7 +90,7 @@ func LoadConfig(configPath string) (*Config, error) {
 		FS: FSConfig{
 			Mountpoint:   "",
 			MountOptions: "nodev,nosuid",
-			UseSymlinks:  false,
+			UseSymlinks:  true,
 			Forge:        "",
 		},
 		Gitlab: GitlabClientConfig{

forges/gitea/client.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"sync"
 
 	"code.gitea.io/sdk/gitea"
 	"github.com/badjware/gitforgefs/config"
-	"github.com/badjware/gitforgefs/fstree"
+	"github.com/badjware/gitforgefs/types"
 )
 
 type giteaClient struct {
@@ -17,15 +16,8 @@ type giteaClient struct {
 
 	logger *slog.Logger
 
-	rootContent map[string]fstree.GroupSource
-
-	// API response cache
-	organizationCacheMux    sync.RWMutex
-	organizationNameToIDMap map[string]int64
-	organizationCache       map[int64]*Organization
-	userCacheMux            sync.RWMutex
-	userNameToIDMap         map[string]int64
-	userCache               map[int64]*User
+	// use a map without values for efficient lookups
+	users map[string]struct{}
 }
 
 func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClient, error) {
@@ -40,58 +32,49 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie
 
 		logger: logger,
 
-		rootContent: nil,
-
-		organizationNameToIDMap: map[string]int64{},
-		organizationCache:       map[int64]*Organization{},
-		userNameToIDMap:         map[string]int64{},
-		userCache:               map[int64]*User{},
+		users: make(map[string]struct{}),
 	}
 
 	// Fetch current user and add it to the list
 	currentUser, _, err := client.GetMyUserInfo()
 	if err != nil {
 		logger.Warn("failed to fetch the current user:", "error", err.Error())
 	} else {
-		giteaClient.UserNames = append(giteaClient.UserNames, *&currentUser.UserName)
+		giteaClient.UserNames = append(giteaClient.UserNames, currentUser.UserName)
 	}
 
 	return giteaClient, nil
 }
 
-func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) {
-	if c.rootContent == nil {
-		rootContent := make(map[string]fstree.GroupSource)
-
-		for _, orgName := range c.GiteaClientConfig.OrgNames {
-			org, err := c.fetchOrganization(ctx, orgName)
-			if err != nil {
-				c.logger.Warn(err.Error())
-			} else {
-				rootContent[org.Name] = org
-			}
-		}
+func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) {
+	rootContent := make(map[string]types.RepositoryGroupSource)
 
-		for _, userName := range c.GiteaClientConfig.UserNames {
-			user, err := c.fetchUser(ctx, userName)
-			if err != nil {
-				c.logger.Warn(err.Error())
-			} else {
-				rootContent[user.Name] = user
-			}
+	for _, orgName := range c.GiteaClientConfig.OrgNames {
+		org, err := c.fetchOrganization(ctx, orgName)
+		if err != nil {
+			c.logger.Warn(err.Error())
+		} else {
+			rootContent[org.Name] = org
 		}
+	}
 
-		c.rootContent = rootContent
+	for _, userName := range c.GiteaClientConfig.UserNames {
+		user, err := c.fetchUser(ctx, userName)
+		if err != nil {
+			c.logger.Warn(err.Error())
+		} else {
+			rootContent[user.Name] = user
+			c.users[user.Name] = struct{}{}
+		}
 	}
-	return c.rootContent, nil
+
+	return rootContent, nil
 }
 
-func (c *giteaClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) {
-	if org, found := c.organizationCache[int64(gid)]; found {
-		return c.fetchOrganizationContent(ctx, org)
-	}
-	if user, found := c.userCache[int64(gid)]; found {
-		return c.fetchUserContent(ctx, user)
+func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) {
+	if _, found := c.users[source.GetGroupPath()]; found {
+		return c.fetchUserContent(ctx, source.GetGroupPath())
+	} else {
+		return c.fetchOrganizationContent(ctx, source.GetGroupPath())
 	}
-	return nil, nil, fmt.Errorf("invalid gid: %v", gid)
 }