From a9ad877f37ca43f44571aced33c6a1b431004cb9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:03:59 +0100 Subject: [PATCH 01/58] Nomad nodes discovery is now using shared logic for job allocations Client proxy previously used allocations job search but only for one jobs with jobs prefix name. This was not suitable for running in environment where it should be used for discover of both orchestrator and template manager instances. New filter logic search for template managers on build nodes and orchestrators on client default nodes. Now ENV variable for job prefix is no longer needed as its baked in discovery logic itself. Logic was also moved from client proxy package to shared as it will be later used for same job in API, that now uses edge api for local cluster template builders disocvery. This will be changed in following changes. --- iac/provider-gcp/nomad/jobs/edge.hcl | 7 +- iac/provider-gcp/nomad/jobs/orchestrator.hcl | 12 +++ .../nomad/jobs/template-manager.hcl | 2 + packages/client-proxy/internal/cfg/model.go | 5 +- .../internal/service-discovery/builder.go | 12 +-- .../nomad-service-discovery.go | 44 +++-------- packages/shared/go.mod | 3 +- packages/shared/go.sum | 4 + .../shared/pkg/clusters/discovery/nomad.go | 74 +++++++++++++++++++ 9 files changed, 111 insertions(+), 52 deletions(-) create mode 100644 packages/shared/pkg/clusters/discovery/nomad.go diff --git a/iac/provider-gcp/nomad/jobs/edge.hcl b/iac/provider-gcp/nomad/jobs/edge.hcl index bf303fb2c9..b96cdc2275 100644 --- a/iac/provider-gcp/nomad/jobs/edge.hcl +++ b/iac/provider-gcp/nomad/jobs/edge.hcl @@ -110,10 +110,9 @@ job "client-proxy" { PROXY_PORT = "${proxy_port}" ORCHESTRATOR_PORT = "${orchestrator_port}" - SD_ORCHESTRATOR_PROVIDER = "NOMAD" - SD_ORCHESTRATOR_NOMAD_ENDPOINT = "${nomad_endpoint}" - SD_ORCHESTRATOR_NOMAD_TOKEN = "${nomad_token}" - SD_ORCHESTRATOR_NOMAD_JOB_PREFIX = "template-manager" + SD_ORCHESTRATOR_PROVIDER = "NOMAD" + SD_ORCHESTRATOR_NOMAD_ENDPOINT = "${nomad_endpoint}" + SD_ORCHESTRATOR_NOMAD_TOKEN = "${nomad_token}" ENVIRONMENT = "${environment}" diff --git a/iac/provider-gcp/nomad/jobs/orchestrator.hcl b/iac/provider-gcp/nomad/jobs/orchestrator.hcl index 1d27d3ea2a..81d88422c3 100644 --- a/iac/provider-gcp/nomad/jobs/orchestrator.hcl +++ b/iac/provider-gcp/nomad/jobs/orchestrator.hcl @@ -5,6 +5,18 @@ job "orchestrator-${latest_orchestrator_job_id}" { priority = 90 group "client-orchestrator" { + // For future as we can remove static and allow multiple instances on one machine if needed. + // Also network allocation is used by Nomad service discovery on API and edge API to find jobs and register them. + network { + port "orchestrator" { + static = "${port}" + } + + port "orchestrator-proxy" { + static = "${proxy_port}" + } + } + service { name = "orchestrator" port = "${port}" diff --git a/iac/provider-gcp/nomad/jobs/template-manager.hcl b/iac/provider-gcp/nomad/jobs/template-manager.hcl index 021385960c..b4f6b9c16a 100644 --- a/iac/provider-gcp/nomad/jobs/template-manager.hcl +++ b/iac/provider-gcp/nomad/jobs/template-manager.hcl @@ -22,6 +22,8 @@ job "template-manager-system" { mode = "delay" } + // For future as we can remove static and allow multiple instances on one machine if needed. + // Also network allocation is used by Nomad service discovery on API and edge API to find jobs and register them. network { port "template-manager" { static = "${port}" diff --git a/packages/client-proxy/internal/cfg/model.go b/packages/client-proxy/internal/cfg/model.go index a53d6a5bc5..d1b59c6328 100644 --- a/packages/client-proxy/internal/cfg/model.go +++ b/packages/client-proxy/internal/cfg/model.go @@ -36,9 +36,8 @@ type ServiceDiscoveryConfig struct { StaticEndpoints []string `env:"STATIC"` // when Provider == "NOMAD" - NomadEndpoint string `env:"NOMAD_ENDPOINT"` - NomadToken string `env:"NOMAD_TOKEN"` - NomadJobPrefix string `env:"NOMAD_JOB_PREFIX"` + NomadEndpoint string `env:"NOMAD_ENDPOINT"` + NomadToken string `env:"NOMAD_TOKEN"` } func Parse() (Config, error) { diff --git a/packages/client-proxy/internal/service-discovery/builder.go b/packages/client-proxy/internal/service-discovery/builder.go index fcd737a54f..62cc70ed4d 100644 --- a/packages/client-proxy/internal/service-discovery/builder.go +++ b/packages/client-proxy/internal/service-discovery/builder.go @@ -87,9 +87,8 @@ func createK8sProvider(ctx context.Context, config cfg.ServiceDiscoveryConfig, p } var ( - ErrMissingNomadEndpoint = errors.New("missing nomad endpoint") - ErrMissingNomadToken = errors.New("missing nomad token") - ErrMissingNomadJobPrefix = errors.New("missing nomad job prefix") + ErrMissingNomadEndpoint = errors.New("missing nomad endpoint") + ErrMissingNomadToken = errors.New("missing nomad token") ) func createNomadProvider(ctx context.Context, config cfg.ServiceDiscoveryConfig, port uint16, logger logger.Logger) (ServiceDiscoveryAdapter, error) { @@ -103,12 +102,7 @@ func createNomadProvider(ctx context.Context, config cfg.ServiceDiscoveryConfig, return nil, ErrMissingNomadToken } - jobPrefix := config.NomadJobPrefix - if jobPrefix == "" { - return nil, ErrMissingNomadJobPrefix - } - - return NewNomadServiceDiscovery(ctx, logger, port, nomadEndpoint, nomadToken, jobPrefix) + return NewNomadServiceDiscovery(ctx, logger, port, nomadEndpoint, nomadToken) } var ErrMissingStaticEndpoints = errors.New("missing static endpoints") diff --git a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go index a12960ae32..8a9538b94d 100644 --- a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go @@ -8,6 +8,7 @@ import ( nomadapi "github.com/hashicorp/nomad/api" "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/shared/pkg/clusters/discovery" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/smap" ) @@ -21,24 +22,19 @@ type NomadServiceDiscovery struct { entries *smap.Map[ServiceDiscoveryItem] client *nomadapi.Client - port uint16 - filter string + port uint16 } -func NewNomadServiceDiscovery(ctx context.Context, logger logger.Logger, port uint16, nomadEndpoint string, nomadToken string, job string) (*NomadServiceDiscovery, error) { +func NewNomadServiceDiscovery(ctx context.Context, logger logger.Logger, port uint16, nomadEndpoint string, nomadToken string) (*NomadServiceDiscovery, error) { config := &nomadapi.Config{Address: nomadEndpoint, SecretID: nomadToken} client, err := nomadapi.NewClient(config) if err != nil { return nil, fmt.Errorf("failed to create Nomad client: %w", err) } - // We want to filter all jobs that are in running state and JobID contains (not equals as we are using suffixes sometimes) - filter := fmt.Sprintf("ClientStatus == \"running\" and JobID contains \"%s\"", job) - sd := &NomadServiceDiscovery{ logger: logger, client: client, - filter: filter, port: port, entries: smap.New[ServiceDiscoveryItem](), } @@ -82,40 +78,18 @@ func (sd *NomadServiceDiscovery) sync(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, nomadQueryRefreshInterval) defer cancel() - options := &nomadapi.QueryOptions{ - Filter: sd.filter, - - // https://developer.hashicorp.com/nomad/api-docs/allocations#resources - // Return allocation resources as part of the response - Params: map[string]string{"resources": "true"}, - } - - results, _, err := sd.client.Allocations().List(options.WithContext(ctx)) + alloc, err := discovery.ListOrchestratorAndTemplateBuilderAllocations(ctx, sd.client) if err != nil { - sd.logger.Error(ctx, "Failed to list Nomad allocations in service discovery", zap.Error(err)) + sd.logger.Error(ctx, "Failed to list orchestrator and template builders", zap.Error(err)) return } - found := make(map[string]string) - for _, v := range results { - if v.AllocatedResources == nil { - sd.logger.Warn(ctx, "No allocated resources found", zap.String("job", v.JobID), zap.String("alloc", v.ID)) - - continue - } - - nets := v.AllocatedResources.Shared.Networks - if len(nets) == 0 { - sd.logger.Warn(ctx, "No allocation networks found", zap.String("job", v.JobID), zap.String("alloc", v.ID)) - - continue - } - - net := nets[0] - key := fmt.Sprintf("%s:%d", net.IP, sd.port) + found := make(map[string]string, len(alloc)) + for _, v := range alloc { + key := fmt.Sprintf("%s:%d", v.AllocationIP, sd.port) item := ServiceDiscoveryItem{ - NodeIP: net.IP, + NodeIP: v.AllocationIP, NodePort: sd.port, } diff --git a/packages/shared/go.mod b/packages/shared/go.mod index e041e5b41a..274fcadd39 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -45,6 +45,7 @@ require ( github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c github.com/jellydator/ttlcache/v3 v3.4.0 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 @@ -191,6 +192,7 @@ require ( github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/hashicorp/consul/api v1.30.0 // indirect + github.com/hashicorp/cronexpr v1.1.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect @@ -203,7 +205,6 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/memberlist v0.5.0 // indirect - github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect diff --git a/packages/shared/go.sum b/packages/shared/go.sum index e8ed579444..4bd69c82c1 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -680,6 +680,8 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= @@ -827,6 +829,8 @@ github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtr github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= diff --git a/packages/shared/pkg/clusters/discovery/nomad.go b/packages/shared/pkg/clusters/discovery/nomad.go new file mode 100644 index 0000000000..e3087ea294 --- /dev/null +++ b/packages/shared/pkg/clusters/discovery/nomad.go @@ -0,0 +1,74 @@ +package discovery + +import ( + "context" + "fmt" + + nomadapi "github.com/hashicorp/nomad/api" + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +type Allocation struct { + NodeID string + AllocationID string + AllocationIP string +} + +const ( + templateManagersTaskGroup = "template-manager" + templateManagerJobPrefix = "template-manager" + + orchestratorsTaskGroup = "client-orchestrator" + orchestratorJobPrefix = "orchestrator" +) + +var filter = fmt.Sprintf( + "ClientStatus == \"running\" and ((TaskGroup == \"%s\" and JobID contains \"%s\") or (TaskGroup == \"%s\" and JobID contains \"%s\"))", + templateManagersTaskGroup, + templateManagerJobPrefix, + orchestratorsTaskGroup, + orchestratorJobPrefix, +) + +func ListOrchestratorAndTemplateBuilderAllocations(ctx context.Context, client *nomadapi.Client) ([]Allocation, error) { + options := &nomadapi.QueryOptions{ + // https://developer.hashicorp.com/nomad/api-docs/allocations#resources + // Return allocation resources as part of the response + Params: map[string]string{"resources": "true"}, + Filter: filter, + } + + results, _, err := client.Allocations().List(options.WithContext(ctx)) + if err != nil { + return nil, fmt.Errorf("failed to list Nomad allocations in service discovery: %w", err) + } + + result := make([]Allocation, 0) + for _, v := range results { + if v.AllocatedResources == nil { + logger.L().Warn(ctx, "No allocated resources found", zap.String("job", v.JobID), zap.String("alloc", v.ID)) + + continue + } + + nets := v.AllocatedResources.Shared.Networks + if len(nets) == 0 { + logger.L().Warn(ctx, "No allocation networks found", zap.String("job", v.JobID), zap.String("alloc", v.ID)) + + continue + } + + net := nets[0] + item := Allocation{ + NodeID: v.NodeID, + AllocationID: v.ID, + AllocationIP: net.IP, + } + + result = append(result, item) + } + + return result, nil +} From 4e697c9facaffed908764c6a7feb337687659c3b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:07:42 +0100 Subject: [PATCH 02/58] Renamed structs used for service discovery `ServiceDiscoveryItem` is now renamed to `DiscoveredInstance` as it better reflects value it holds and makes obvious that its ment just for internal cycle of discovery. --- .../internal/edge/pool/orchestrator-pool.go | 20 +++++++++---------- .../internal/service-discovery/adapter.go | 8 ++++---- .../dns-service-discovery.go | 12 +++++------ .../k8s-service-discovery.go | 14 ++++++------- .../nomad-service-discovery.go | 14 ++++++------- .../static-service-discovery.go | 8 ++++---- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go index c85210511d..54ef57caf5 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go +++ b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go @@ -19,7 +19,7 @@ import ( type OrchestratorsPool struct { discovery sd.ServiceDiscoveryAdapter instances *smap.Map[*OrchestratorInstance] - synchronization *synchronization.Synchronize[sd.ServiceDiscoveryItem, *OrchestratorInstance] + synchronization *synchronization.Synchronize[sd.DiscoveredInstance, *OrchestratorInstance] logger logger.Logger @@ -133,14 +133,14 @@ func (e *orchestratorInstancesSyncStore) getHost(ip string, port uint16) string return fmt.Sprintf("%s:%d", ip, port) } -func (e *orchestratorInstancesSyncStore) SourceList(ctx context.Context) ([]sd.ServiceDiscoveryItem, error) { - return e.pool.discovery.ListNodes(ctx) +func (e *orchestratorInstancesSyncStore) SourceList(ctx context.Context) ([]sd.DiscoveredInstance, error) { + return e.pool.discovery.ListInstances(ctx) } -func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd.ServiceDiscoveryItem, p *OrchestratorInstance) bool { +func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd.DiscoveredInstance, p *OrchestratorInstance) bool { itself := p.GetInfo() for _, item := range s { - itemHost := e.getHost(item.NodeIP, item.NodePort) + itemHost := e.getHost(item.InstanceIPAddress, item.InstancePort) if itemHost == itself.Host { return true } @@ -158,16 +158,16 @@ func (e *orchestratorInstancesSyncStore) PoolList(_ context.Context) []*Orchestr return items } -func (e *orchestratorInstancesSyncStore) PoolExists(_ context.Context, source sd.ServiceDiscoveryItem) bool { - host := e.getHost(source.NodeIP, source.NodePort) +func (e *orchestratorInstancesSyncStore) PoolExists(_ context.Context, source sd.DiscoveredInstance) bool { + host := e.getHost(source.InstanceIPAddress, source.InstancePort) _, found := e.pool.instances.Get(host) return found } -func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source sd.ServiceDiscoveryItem) { - host := e.getHost(source.NodeIP, source.NodePort) - o, err := NewOrchestratorInstance(e.pool.tracerProvider, e.pool.metricProvider, source.NodeIP, source.NodePort) +func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source sd.DiscoveredInstance) { + host := e.getHost(source.InstanceIPAddress, source.InstancePort) + o, err := NewOrchestratorInstance(e.pool.tracerProvider, e.pool.metricProvider, source.InstanceIPAddress, source.InstancePort) if err != nil { logger.L().Error(ctx, "failed to register new orchestrator instance", zap.String("host", host), zap.Error(err)) diff --git a/packages/client-proxy/internal/service-discovery/adapter.go b/packages/client-proxy/internal/service-discovery/adapter.go index eca096f963..ace972647b 100644 --- a/packages/client-proxy/internal/service-discovery/adapter.go +++ b/packages/client-proxy/internal/service-discovery/adapter.go @@ -4,11 +4,11 @@ import ( "context" ) -type ServiceDiscoveryItem struct { - NodeIP string `json:"node_ip"` - NodePort uint16 `json:"node_port"` +type DiscoveredInstance struct { + InstanceIPAddress string + InstancePort uint16 } type ServiceDiscoveryAdapter interface { - ListNodes(ctx context.Context) ([]ServiceDiscoveryItem, error) + ListInstances(ctx context.Context) ([]DiscoveredInstance, error) } diff --git a/packages/client-proxy/internal/service-discovery/dns-service-discovery.go b/packages/client-proxy/internal/service-discovery/dns-service-discovery.go index 37d88761a7..7ac17b0240 100644 --- a/packages/client-proxy/internal/service-discovery/dns-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/dns-service-discovery.go @@ -14,7 +14,7 @@ import ( type DnsServiceDiscovery struct { logger logger.Logger - entries *smap.Map[ServiceDiscoveryItem] + entries *smap.Map[DiscoveredInstance] resolver string hosts []string @@ -40,7 +40,7 @@ func NewDnsServiceDiscovery(ctx context.Context, logger logger.Logger, hosts []s resolver: resolver, servicePort: servicePort, - entries: smap.New[ServiceDiscoveryItem](), + entries: smap.New[DiscoveredInstance](), } go func() { sd.keepInSync(ctx) }() @@ -48,9 +48,9 @@ func NewDnsServiceDiscovery(ctx context.Context, logger logger.Logger, hosts []s return sd } -func (sd *DnsServiceDiscovery) ListNodes(_ context.Context) ([]ServiceDiscoveryItem, error) { +func (sd *DnsServiceDiscovery) ListInstances(_ context.Context) ([]DiscoveredInstance, error) { entries := sd.entries.Items() - items := make([]ServiceDiscoveryItem, 0) + items := make([]DiscoveredInstance, 0) for _, item := range entries { items = append(items, item) @@ -118,13 +118,13 @@ func (sd *DnsServiceDiscovery) sync(ctx context.Context) { for ip := range ips { key := fmt.Sprintf("%s:%d", ip, sd.servicePort) sd.entries.Insert( - key, ServiceDiscoveryItem{NodeIP: ip, NodePort: sd.servicePort}, + key, DiscoveredInstance{InstanceIPAddress: ip, InstancePort: sd.servicePort}, ) } // remove entries that are no longer in DNS response for key, item := range sd.entries.Items() { - if _, ok := ips[item.NodeIP]; !ok { + if _, ok := ips[item.InstanceIPAddress]; !ok { sd.entries.Remove(key) } } diff --git a/packages/client-proxy/internal/service-discovery/k8s-service-discovery.go b/packages/client-proxy/internal/service-discovery/k8s-service-discovery.go index f3b6438fdb..fa53a0545e 100644 --- a/packages/client-proxy/internal/service-discovery/k8s-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/k8s-service-discovery.go @@ -19,7 +19,7 @@ const ( type K8sServiceDiscovery struct { logger logger.Logger - entries *smap.Map[ServiceDiscoveryItem] + entries *smap.Map[DiscoveredInstance] client *kubernetes.Clientset filterLabels string @@ -40,7 +40,7 @@ func NewK8sServiceDiscovery(ctx context.Context, logger logger.Logger, client *k filterLabels: podLabels, filterNamespace: podNamespace, - entries: smap.New[ServiceDiscoveryItem](), + entries: smap.New[DiscoveredInstance](), } go func() { sd.keepInSync(ctx) }() @@ -48,9 +48,9 @@ func NewK8sServiceDiscovery(ctx context.Context, logger logger.Logger, client *k return sd } -func (sd *K8sServiceDiscovery) ListNodes(_ context.Context) ([]ServiceDiscoveryItem, error) { +func (sd *K8sServiceDiscovery) ListInstances(_ context.Context) ([]DiscoveredInstance, error) { entries := sd.entries.Items() - items := make([]ServiceDiscoveryItem, 0) + items := make([]DiscoveredInstance, 0) for _, item := range entries { items = append(items, item) @@ -99,9 +99,9 @@ func (sd *K8sServiceDiscovery) sync(ctx context.Context) { } key := fmt.Sprintf("%s:%d", ip, sd.port) - item := ServiceDiscoveryItem{ - NodeIP: ip, - NodePort: sd.port, + item := DiscoveredInstance{ + InstanceIPAddress: ip, + InstancePort: sd.port, } sd.entries.Insert(key, item) diff --git a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go index 8a9538b94d..b4da591beb 100644 --- a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go @@ -19,7 +19,7 @@ const ( type NomadServiceDiscovery struct { logger logger.Logger - entries *smap.Map[ServiceDiscoveryItem] + entries *smap.Map[DiscoveredInstance] client *nomadapi.Client port uint16 @@ -36,7 +36,7 @@ func NewNomadServiceDiscovery(ctx context.Context, logger logger.Logger, port ui logger: logger, client: client, port: port, - entries: smap.New[ServiceDiscoveryItem](), + entries: smap.New[DiscoveredInstance](), } go func() { sd.keepInSync(ctx) }() @@ -44,9 +44,9 @@ func NewNomadServiceDiscovery(ctx context.Context, logger logger.Logger, port ui return sd, nil } -func (sd *NomadServiceDiscovery) ListNodes(_ context.Context) ([]ServiceDiscoveryItem, error) { +func (sd *NomadServiceDiscovery) ListInstances(_ context.Context) ([]DiscoveredInstance, error) { entries := sd.entries.Items() - items := make([]ServiceDiscoveryItem, 0) + items := make([]DiscoveredInstance, 0) for _, item := range entries { items = append(items, item) @@ -88,9 +88,9 @@ func (sd *NomadServiceDiscovery) sync(ctx context.Context) { found := make(map[string]string, len(alloc)) for _, v := range alloc { key := fmt.Sprintf("%s:%d", v.AllocationIP, sd.port) - item := ServiceDiscoveryItem{ - NodeIP: v.AllocationIP, - NodePort: sd.port, + item := DiscoveredInstance{ + InstanceIPAddress: v.AllocationIP, + InstancePort: sd.port, } sd.entries.Insert(key, item) diff --git a/packages/client-proxy/internal/service-discovery/static-service-discovery.go b/packages/client-proxy/internal/service-discovery/static-service-discovery.go index a9eab23ba2..315a7ec311 100644 --- a/packages/client-proxy/internal/service-discovery/static-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/static-service-discovery.go @@ -3,21 +3,21 @@ package service_discovery import "context" type StaticServiceDiscovery struct { - items []ServiceDiscoveryItem + items []DiscoveredInstance } func NewStaticServiceDiscovery(results []string, port uint16) *StaticServiceDiscovery { - items := make([]ServiceDiscoveryItem, 0) + items := make([]DiscoveredInstance, 0) for _, result := range results { items = append( - items, ServiceDiscoveryItem{NodeIP: result, NodePort: port}, + items, DiscoveredInstance{InstanceIPAddress: result, InstancePort: port}, ) } return &StaticServiceDiscovery{items: items} } -func (s StaticServiceDiscovery) ListNodes(_ context.Context) ([]ServiceDiscoveryItem, error) { +func (s StaticServiceDiscovery) ListInstances(_ context.Context) ([]DiscoveredInstance, error) { return s.items, nil } From 7a22fdfc24a0bb66481c25fc7cf54260dd99d2f1 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:17:11 +0100 Subject: [PATCH 03/58] Rename structs handling instance info Lot of structs and variables is prefixed orchestrator even its already in package name so its not needed. Rename variables on structs/struct itself to start with lower letter to not export them to other packages if not really needed. Instance gRPC client is also made internal and we now have two function for exporting raw gRPC connection used in gRPC proxy and gRPC info service client that is used in few places. --- .../internal/edge-pass-through/proxy.go | 2 +- .../handlers/service-discovery-node-drain.go | 2 +- .../internal/edge/pool/orchestrator-pool.go | 38 +++++----- .../internal/edge/pool/orchestrator.go | 72 ++++++++++--------- 4 files changed, 59 insertions(+), 55 deletions(-) diff --git a/packages/client-proxy/internal/edge-pass-through/proxy.go b/packages/client-proxy/internal/edge-pass-through/proxy.go index bcc7b1115a..b4a464e1d1 100644 --- a/packages/client-proxy/internal/edge-pass-through/proxy.go +++ b/packages/client-proxy/internal/edge-pass-through/proxy.go @@ -83,7 +83,7 @@ func (s *NodePassThroughServer) director(ctx context.Context) (*grpc.ClientConn, return nil, nil, status.Error(codes.NotFound, "service instance not found") } - return serviceInstance.GetClient().Connection, md, nil + return serviceInstance.GetConnection(), md, nil } // Handler - following code implement a gRPC pass-through proxy that forwards requests to the appropriate node diff --git a/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go b/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go index 8b0409d66e..94a48546e0 100644 --- a/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go +++ b/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go @@ -73,7 +73,7 @@ func (a *APIStore) sendOrchestratorRequest(ctx context.Context, serviceInstanceI defer findCtxCancel() orchestratorStatus := ApiNodeToOrchestratorStateMapper[status] - _, err := o.GetClient().Info.ServiceStatusOverride( + _, err := o.GetClient().ServiceStatusOverride( findCtx, &orchestratorinfo.ServiceStatusChangeRequest{ServiceStatus: orchestratorStatus}, ) if err != nil { diff --git a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go index 54ef57caf5..65e0e68f1f 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go +++ b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go @@ -18,8 +18,8 @@ import ( type OrchestratorsPool struct { discovery sd.ServiceDiscoveryAdapter - instances *smap.Map[*OrchestratorInstance] - synchronization *synchronization.Synchronize[sd.DiscoveredInstance, *OrchestratorInstance] + instances *smap.Map[*Instance] + synchronization *synchronization.Synchronize[sd.DiscoveredInstance, *Instance] logger logger.Logger @@ -47,7 +47,7 @@ func NewOrchestratorsPool( ) *OrchestratorsPool { pool := &OrchestratorsPool{ discovery: discovery, - instances: smap.New[*OrchestratorInstance](), + instances: smap.New[*Instance](), logger: l, @@ -69,11 +69,11 @@ func NewOrchestratorsPool( return pool } -func (p *OrchestratorsPool) GetOrchestrators() map[string]*OrchestratorInstance { +func (p *OrchestratorsPool) GetOrchestrators() map[string]*Instance { return p.instances.Items() } -func (p *OrchestratorsPool) GetOrchestrator(instanceID string) (i *OrchestratorInstance, ok bool) { +func (p *OrchestratorsPool) GetOrchestrator(instanceID string) (i *Instance, ok bool) { orchestrators := p.GetOrchestrators() for _, i = range orchestrators { if i.GetInfo().ServiceInstanceID == instanceID { @@ -112,7 +112,7 @@ func (p *OrchestratorsPool) Close(ctx context.Context) error { for _, instance := range p.instances.Items() { err := instance.Close() if err != nil { - p.logger.Error(ctx, "Error closing orchestrator instance", zap.Error(err), logger.WithNodeID(instance.GetInfo().NodeID)) + p.logger.Error(ctx, "Error closing orchestrator Instance", zap.Error(err), logger.WithNodeID(instance.GetInfo().NodeID)) } } @@ -137,7 +137,7 @@ func (e *orchestratorInstancesSyncStore) SourceList(ctx context.Context) ([]sd.D return e.pool.discovery.ListInstances(ctx) } -func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd.DiscoveredInstance, p *OrchestratorInstance) bool { +func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd.DiscoveredInstance, p *Instance) bool { itself := p.GetInfo() for _, item := range s { itemHost := e.getHost(item.InstanceIPAddress, item.InstancePort) @@ -149,8 +149,8 @@ func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd. return false } -func (e *orchestratorInstancesSyncStore) PoolList(_ context.Context) []*OrchestratorInstance { - items := make([]*OrchestratorInstance, 0) +func (e *orchestratorInstancesSyncStore) PoolList(_ context.Context) []*Instance { + items := make([]*Instance, 0) for _, item := range e.pool.instances.Items() { items = append(items, item) } @@ -169,7 +169,7 @@ func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source host := e.getHost(source.InstanceIPAddress, source.InstancePort) o, err := NewOrchestratorInstance(e.pool.tracerProvider, e.pool.metricProvider, source.InstanceIPAddress, source.InstancePort) if err != nil { - logger.L().Error(ctx, "failed to register new orchestrator instance", zap.String("host", host), zap.Error(err)) + logger.L().Error(ctx, "failed to register new orchestrator Instance", zap.String("host", host), zap.Error(err)) return } @@ -177,11 +177,11 @@ func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source ctx, cancel := context.WithTimeout(ctx, orchestratorsInstanceSyncTimeout) defer cancel() - // Initial synchronization of the orchestrator instance - // We want to do it separately here so failed init will cause not adding the instance to the pool + // Initial synchronization of the orchestrator Instance + // We want to do it separately here so failed init will cause not adding the Instance to the pool err = o.sync(ctx) if err != nil { - logger.L().Error(ctx, "Failed to finish initial orchestrator instance sync", zap.Error(err), logger.WithNodeID(o.GetInfo().NodeID)) + logger.L().Error(ctx, "Failed to finish initial orchestrator Instance sync", zap.Error(err), logger.WithNodeID(o.GetInfo().NodeID)) return } @@ -189,25 +189,25 @@ func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source e.pool.instances.Insert(host, o) } -func (e *orchestratorInstancesSyncStore) PoolUpdate(ctx context.Context, item *OrchestratorInstance) { +func (e *orchestratorInstancesSyncStore) PoolUpdate(ctx context.Context, item *Instance) { ctx, cancel := context.WithTimeout(ctx, orchestratorsInstanceSyncTimeout) defer cancel() err := item.sync(ctx) if err != nil { - logger.L().Error(ctx, "Failed to sync orchestrator instance", zap.Error(err), logger.WithNodeID(item.GetInfo().NodeID)) + logger.L().Error(ctx, "Failed to sync orchestrator Instance", zap.Error(err), logger.WithNodeID(item.GetInfo().NodeID)) } } -func (e *orchestratorInstancesSyncStore) PoolRemove(ctx context.Context, item *OrchestratorInstance) { +func (e *orchestratorInstancesSyncStore) PoolRemove(ctx context.Context, item *Instance) { info := item.GetInfo() - logger.L().Info(ctx, "Orchestrator instance connection is not active anymore, closing.", logger.WithNodeID(info.NodeID)) + logger.L().Info(ctx, "Orchestrator Instance connection is not active anymore, closing.", logger.WithNodeID(info.NodeID)) err := item.Close() if err != nil { - logger.L().Error(ctx, "Error closing connection to orchestrator instance", zap.Error(err), logger.WithNodeID(info.NodeID)) + logger.L().Error(ctx, "Error closing connection to orchestrator Instance", zap.Error(err), logger.WithNodeID(info.NodeID)) } e.pool.instances.Remove(info.Host) - logger.L().Info(ctx, "Orchestrator instance connection has been deregistered.", logger.WithNodeID(info.NodeID)) + logger.L().Info(ctx, "Orchestrator Instance connection has been deregistered.", logger.WithNodeID(info.NodeID)) } diff --git a/packages/client-proxy/internal/edge/pool/orchestrator.go b/packages/client-proxy/internal/edge/pool/orchestrator.go index 7aef4780b7..09f21e0a1e 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator.go +++ b/packages/client-proxy/internal/edge/pool/orchestrator.go @@ -23,14 +23,14 @@ import ( type OrchestratorStatus string const ( - orchestratorSyncMaxRetries = 3 + instanceSyncMaxRetries = 3 OrchestratorStatusHealthy OrchestratorStatus = "healthy" OrchestratorStatusDraining OrchestratorStatus = "draining" OrchestratorStatusUnhealthy OrchestratorStatus = "unhealthy" ) -type OrchestratorInstanceInfo struct { +type InstanceInfo struct { NodeID string ServiceInstanceID string @@ -44,23 +44,23 @@ type OrchestratorInstanceInfo struct { Roles []e2bgrpcorchestratorinfo.ServiceInfoRole } -type OrchestratorInstance struct { - MetricVCpuUsed atomic.Uint32 - MetricMemoryUsedBytes atomic.Uint64 - MetricDiskUsedBytes atomic.Uint64 - MetricSandboxesRunning atomic.Uint32 +type Instance struct { + metricVCpuUsed atomic.Uint32 + metricMemoryUsedBytes atomic.Uint64 + metricDiskUsedBytes atomic.Uint64 + metricSandboxesRunning atomic.Uint32 - client *OrchestratorGRPCClient - info OrchestratorInstanceInfo + client *instanceGRPCClient + info InstanceInfo mutex sync.RWMutex } -type OrchestratorGRPCClient struct { - Info e2bgrpcorchestratorinfo.InfoServiceClient - Connection *grpc.ClientConn +type instanceGRPCClient struct { + info e2bgrpcorchestratorinfo.InfoServiceClient + connection *grpc.ClientConn } -func NewOrchestratorInstance(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, ip string, port uint16) (*OrchestratorInstance, error) { +func NewOrchestratorInstance(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, ip string, port uint16) (*Instance, error) { host := fmt.Sprintf("%s:%d", ip, port) client, err := newClient(tracerProvider, meterProvider, host) @@ -68,9 +68,9 @@ func NewOrchestratorInstance(tracerProvider trace.TracerProvider, meterProvider return nil, fmt.Errorf("failed to create GRPC client: %w", err) } - o := &OrchestratorInstance{ + o := &Instance{ client: client, - info: OrchestratorInstanceInfo{ + info: InstanceInfo{ Host: host, IP: ip, }, @@ -79,11 +79,11 @@ func NewOrchestratorInstance(tracerProvider trace.TracerProvider, meterProvider return o, nil } -func (o *OrchestratorInstance) sync(ctx context.Context) error { - for range orchestratorSyncMaxRetries { +func (o *Instance) sync(ctx context.Context) error { + for range instanceSyncMaxRetries { freshInfo := o.GetInfo() - status, err := o.client.Info.ServiceInfo(ctx, &emptypb.Empty{}) + status, err := o.client.info.ServiceInfo(ctx, &emptypb.Empty{}) if err != nil { logger.L().Error(ctx, "failed to check orchestrator health", logger.WithNodeID(freshInfo.NodeID), zap.Error(err)) @@ -99,10 +99,10 @@ func (o *OrchestratorInstance) sync(ctx context.Context) error { freshInfo.Roles = status.GetServiceRoles() o.setInfo(freshInfo) - o.MetricSandboxesRunning.Store(status.GetMetricSandboxesRunning()) - o.MetricMemoryUsedBytes.Store(status.GetMetricMemoryUsedBytes()) - o.MetricDiskUsedBytes.Store(status.GetMetricDiskAllocatedBytes()) - o.MetricVCpuUsed.Store(status.GetMetricCpuCount()) + o.metricSandboxesRunning.Store(status.GetMetricSandboxesRunning()) + o.metricMemoryUsedBytes.Store(status.GetMetricMemoryUsedBytes()) + o.metricDiskUsedBytes.Store(status.GetMetricDiskAllocatedBytes()) + o.metricVCpuUsed.Store(status.GetMetricCpuCount()) return nil } @@ -110,30 +110,34 @@ func (o *OrchestratorInstance) sync(ctx context.Context) error { return errors.New("failed to check orchestrator status") } -func (o *OrchestratorInstance) setStatus(status OrchestratorStatus) { +func (o *Instance) setStatus(status OrchestratorStatus) { o.mutex.Lock() defer o.mutex.Unlock() o.info.ServiceStatus = status } -func (o *OrchestratorInstance) setInfo(i OrchestratorInstanceInfo) { +func (o *Instance) setInfo(i InstanceInfo) { o.mutex.Lock() defer o.mutex.Unlock() o.info = i } -func (o *OrchestratorInstance) GetInfo() OrchestratorInstanceInfo { +func (o *Instance) GetInfo() InstanceInfo { o.mutex.RLock() defer o.mutex.RUnlock() return o.info } -func (o *OrchestratorInstance) GetClient() *OrchestratorGRPCClient { - return o.client +func (o *Instance) GetClient() e2bgrpcorchestratorinfo.InfoServiceClient { + return o.client.info } -func (o *OrchestratorInstance) Close() error { +func (o *Instance) GetConnection() *grpc.ClientConn { + return o.client.connection +} + +func (o *Instance) Close() error { // close sync context o.setStatus(OrchestratorStatusUnhealthy) @@ -163,7 +167,7 @@ func getMappedStatus(ctx context.Context, s e2bgrpcorchestratorinfo.ServiceInfoS return OrchestratorStatusUnhealthy } -func newClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*OrchestratorGRPCClient, error) { +func newClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*instanceGRPCClient, error) { conn, err := grpc.NewClient(host, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler( @@ -177,14 +181,14 @@ func newClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterPr return nil, fmt.Errorf("failed to create GRPC client: %w", err) } - return &OrchestratorGRPCClient{ - Info: e2bgrpcorchestratorinfo.NewInfoServiceClient(conn), - Connection: conn, + return &instanceGRPCClient{ + info: e2bgrpcorchestratorinfo.NewInfoServiceClient(conn), + connection: conn, }, nil } -func (a *OrchestratorGRPCClient) close() error { - err := a.Connection.Close() +func (a *instanceGRPCClient) close() error { + err := a.connection.Close() if err != nil { return fmt.Errorf("failed to close connection: %w", err) } From 9b8f562599beae51cfa7808e7163372327a57a14 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:20:17 +0100 Subject: [PATCH 04/58] Edge pool renaming and cleanup Moved instance client holding gRPC connection and gRPC info client into separated file to make it cleaner and easier to work with. Orchestrator filename is renamed to instance and `NewOrchestratorInstance` is renamed to `newInstance` as orchestrator is duplicated here (package name) and dont need to be used ourside of package. --- .../client-proxy/internal/edge/pool/client.go | 47 +++++++++++++++++++ .../pool/{orchestrator.go => instance.go} | 38 +-------------- .../internal/edge/pool/orchestrator-pool.go | 2 +- 3 files changed, 49 insertions(+), 38 deletions(-) create mode 100644 packages/client-proxy/internal/edge/pool/client.go rename packages/client-proxy/internal/edge/pool/{orchestrator.go => instance.go} (76%) diff --git a/packages/client-proxy/internal/edge/pool/client.go b/packages/client-proxy/internal/edge/pool/client.go new file mode 100644 index 0000000000..53b8385b5f --- /dev/null +++ b/packages/client-proxy/internal/edge/pool/client.go @@ -0,0 +1,47 @@ +package pool + +import ( + "fmt" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + e2bgrpcorchestratorinfo "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" +) + +type instanceGRPCClient struct { + info e2bgrpcorchestratorinfo.InfoServiceClient + connection *grpc.ClientConn +} + +func newClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*instanceGRPCClient, error) { + conn, err := grpc.NewClient(host, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler( + otelgrpc.NewClientHandler( + otelgrpc.WithTracerProvider(tracerProvider), + otelgrpc.WithMeterProvider(meterProvider), + ), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create GRPC client: %w", err) + } + + return &instanceGRPCClient{ + info: e2bgrpcorchestratorinfo.NewInfoServiceClient(conn), + connection: conn, + }, nil +} + +func (a *instanceGRPCClient) close() error { + err := a.connection.Close() + if err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + + return nil +} diff --git a/packages/client-proxy/internal/edge/pool/orchestrator.go b/packages/client-proxy/internal/edge/pool/instance.go similarity index 76% rename from packages/client-proxy/internal/edge/pool/orchestrator.go rename to packages/client-proxy/internal/edge/pool/instance.go index 09f21e0a1e..d0748e4051 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator.go +++ b/packages/client-proxy/internal/edge/pool/instance.go @@ -8,12 +8,10 @@ import ( "sync/atomic" "time" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/types/known/emptypb" e2bgrpcorchestratorinfo "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" @@ -55,12 +53,7 @@ type Instance struct { mutex sync.RWMutex } -type instanceGRPCClient struct { - info e2bgrpcorchestratorinfo.InfoServiceClient - connection *grpc.ClientConn -} - -func NewOrchestratorInstance(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, ip string, port uint16) (*Instance, error) { +func newInstance(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, ip string, port uint16) (*Instance, error) { host := fmt.Sprintf("%s:%d", ip, port) client, err := newClient(tracerProvider, meterProvider, host) @@ -166,32 +159,3 @@ func getMappedStatus(ctx context.Context, s e2bgrpcorchestratorinfo.ServiceInfoS return OrchestratorStatusUnhealthy } - -func newClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*instanceGRPCClient, error) { - conn, err := grpc.NewClient(host, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithStatsHandler( - otelgrpc.NewClientHandler( - otelgrpc.WithTracerProvider(tracerProvider), - otelgrpc.WithMeterProvider(meterProvider), - ), - ), - ) - if err != nil { - return nil, fmt.Errorf("failed to create GRPC client: %w", err) - } - - return &instanceGRPCClient{ - info: e2bgrpcorchestratorinfo.NewInfoServiceClient(conn), - connection: conn, - }, nil -} - -func (a *instanceGRPCClient) close() error { - err := a.connection.Close() - if err != nil { - return fmt.Errorf("failed to close connection: %w", err) - } - - return nil -} diff --git a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go index 65e0e68f1f..2adfc9780a 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go +++ b/packages/client-proxy/internal/edge/pool/orchestrator-pool.go @@ -167,7 +167,7 @@ func (e *orchestratorInstancesSyncStore) PoolExists(_ context.Context, source sd func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source sd.DiscoveredInstance) { host := e.getHost(source.InstanceIPAddress, source.InstancePort) - o, err := NewOrchestratorInstance(e.pool.tracerProvider, e.pool.metricProvider, source.InstanceIPAddress, source.InstancePort) + o, err := newInstance(e.pool.tracerProvider, e.pool.metricProvider, source.InstanceIPAddress, source.InstancePort) if err != nil { logger.L().Error(ctx, "failed to register new orchestrator Instance", zap.String("host", host), zap.Error(err)) From 32f70062fefde1bb754543241e211457e13cf71f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:25:19 +0100 Subject: [PATCH 05/58] Renamed orchestrators pool to instances pool Renamed as orchestrators pool can be little bit configusing in case when pool contains both orchestrators and tempalte builders. Now we are calling them instances. --- .../internal/edge-pass-through/proxy.go | 4 +- .../handlers/service-discovery-node-drain.go | 2 +- .../handlers/service-discovery-nodes-list.go | 2 +- .../service-discovery-orchestrators.go | 4 +- .../internal/edge/handlers/store.go | 10 ++-- ...orchestrator-pool.go => instances-sync.go} | 56 +++++++++---------- .../client-proxy/internal/edge/service.go | 6 +- packages/client-proxy/main.go | 8 +-- 8 files changed, 45 insertions(+), 47 deletions(-) rename packages/client-proxy/internal/edge/pool/{orchestrator-pool.go => instances-sync.go} (69%) diff --git a/packages/client-proxy/internal/edge-pass-through/proxy.go b/packages/client-proxy/internal/edge-pass-through/proxy.go index b4a464e1d1..f96c625028 100644 --- a/packages/client-proxy/internal/edge-pass-through/proxy.go +++ b/packages/client-proxy/internal/edge-pass-through/proxy.go @@ -20,7 +20,7 @@ import ( ) type NodePassThroughServer struct { - nodes *e2borchestrators.OrchestratorsPool + nodes *e2borchestrators.InstancesPool catalog catalog.SandboxesCatalog info *e2binfo.ServiceInfo @@ -36,7 +36,7 @@ const ( var clientStreamDescForProxying = &grpc.StreamDesc{ServerStreams: true, ClientStreams: true} func NewNodePassThroughServer( - nodes *e2borchestrators.OrchestratorsPool, + nodes *e2borchestrators.InstancesPool, info *e2binfo.ServiceInfo, authorization authorization.AuthorizationService, catalog catalog.SandboxesCatalog, diff --git a/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go b/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go index 94a48546e0..33461994b1 100644 --- a/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go +++ b/packages/client-proxy/internal/edge/handlers/service-discovery-node-drain.go @@ -62,7 +62,7 @@ func (a *APIStore) sendOrchestratorRequest(ctx context.Context, serviceInstanceI logger := a.logger.With(l.WithServiceInstanceID(serviceInstanceID)) // try to find orchestrator node first - o, ok := a.orchestratorPool.GetOrchestrator(serviceInstanceID) + o, ok := a.instancesPool.GetOrchestrator(serviceInstanceID) if !ok { return errors.New("orchestrator instance doesn't found") } diff --git a/packages/client-proxy/internal/edge/handlers/service-discovery-nodes-list.go b/packages/client-proxy/internal/edge/handlers/service-discovery-nodes-list.go index 6338fc9a80..859485bbfd 100644 --- a/packages/client-proxy/internal/edge/handlers/service-discovery-nodes-list.go +++ b/packages/client-proxy/internal/edge/handlers/service-discovery-nodes-list.go @@ -18,7 +18,7 @@ func (a *APIStore) V1ServiceDiscoveryNodes(c *gin.Context) { response := make([]api.ClusterNode, 0) // iterate orchestrator pool - for _, orchestrator := range a.orchestratorPool.GetOrchestrators() { + for _, orchestrator := range a.instancesPool.GetOrchestrators() { info := orchestrator.GetInfo() response = append( response, diff --git a/packages/client-proxy/internal/edge/handlers/service-discovery-orchestrators.go b/packages/client-proxy/internal/edge/handlers/service-discovery-orchestrators.go index ea75ea279c..bdda075604 100644 --- a/packages/client-proxy/internal/edge/handlers/service-discovery-orchestrators.go +++ b/packages/client-proxy/internal/edge/handlers/service-discovery-orchestrators.go @@ -22,8 +22,8 @@ func (a *APIStore) V1ServiceDiscoveryGetOrchestrators(c *gin.Context) { response := make([]api.ClusterOrchestratorNode, 0) - for _, node := range a.orchestratorPool.GetOrchestrators() { - info := node.GetInfo() + for _, instance := range a.instancesPool.GetOrchestrators() { + info := instance.GetInfo() response = append( response, api.ClusterOrchestratorNode{ diff --git a/packages/client-proxy/internal/edge/handlers/store.go b/packages/client-proxy/internal/edge/handlers/store.go index 1c35414386..773f94442a 100644 --- a/packages/client-proxy/internal/edge/handlers/store.go +++ b/packages/client-proxy/internal/edge/handlers/store.go @@ -14,7 +14,7 @@ import ( "github.com/e2b-dev/infra/packages/proxy/internal/edge/info" loggerprovider "github.com/e2b-dev/infra/packages/proxy/internal/edge/logger-provider" metricsprovider "github.com/e2b-dev/infra/packages/proxy/internal/edge/metrics-provider" - e2borchestrators "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool" + e2binstances "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool" "github.com/e2b-dev/infra/packages/shared/pkg/env" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -27,7 +27,7 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/client-proxy/interna type APIStore struct { logger logger.Logger info *info.ServiceInfo - orchestratorPool *e2borchestrators.OrchestratorsPool + instancesPool *e2binstances.InstancesPool sandboxes catalog.SandboxesCatalog queryLogsProvider loggerprovider.LogsQueryProvider querySandboxMetricsProvider clickhouse.SandboxQueriesProvider @@ -41,7 +41,7 @@ func NewStore( ctx context.Context, l logger.Logger, info *info.ServiceInfo, - orchestratorsPool *e2borchestrators.OrchestratorsPool, + instances *e2binstances.InstancesPool, catalog catalog.SandboxesCatalog, config cfg.Config, ) (*APIStore, error) { @@ -56,7 +56,7 @@ func NewStore( } store := &APIStore{ - orchestratorPool: orchestratorsPool, + instancesPool: instances, queryLogsProvider: queryLogsProvider, querySandboxMetricsProvider: querySandboxMetricsProvider, @@ -91,7 +91,7 @@ func NewStore( case <-ctx.Done(): return case <-ticker.C: - list := orchestratorsPool.GetOrchestrators() + list := instances.GetOrchestrators() if len(list) > 0 { logger.L().Info(ctx, "Marking API as healthy, at least one orchestrator is available") store.info.SetStatus(ctx, api.Healthy) diff --git a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go b/packages/client-proxy/internal/edge/pool/instances-sync.go similarity index 69% rename from packages/client-proxy/internal/edge/pool/orchestrator-pool.go rename to packages/client-proxy/internal/edge/pool/instances-sync.go index 2adfc9780a..cc6d55272a 100644 --- a/packages/client-proxy/internal/edge/pool/orchestrator-pool.go +++ b/packages/client-proxy/internal/edge/pool/instances-sync.go @@ -16,7 +16,7 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" ) -type OrchestratorsPool struct { +type InstancesPool struct { discovery sd.ServiceDiscoveryAdapter instances *smap.Map[*Instance] synchronization *synchronization.Synchronize[sd.DiscoveredInstance, *Instance] @@ -31,21 +31,21 @@ type OrchestratorsPool struct { } const ( - orchestratorsPoolInterval = 5 * time.Second - orchestratorsPoolRoundTimeout = 5 * time.Second - orchestratorsInstanceSyncTimeout = 5 * time.Second + instancesPoolInterval = 5 * time.Second + instancesPoolRoundTimeout = 5 * time.Second + instanceSyncTimeout = 5 * time.Second statusLogInterval = 1 * time.Minute ) -func NewOrchestratorsPool( +func NewInstancesPool( ctx context.Context, l logger.Logger, tracerProvider trace.TracerProvider, metricProvider metric.MeterProvider, discovery sd.ServiceDiscoveryAdapter, -) *OrchestratorsPool { - pool := &OrchestratorsPool{ +) *InstancesPool { + pool := &InstancesPool{ discovery: discovery, instances: smap.New[*Instance](), @@ -57,23 +57,21 @@ func NewOrchestratorsPool( close: make(chan struct{}), } - store := &orchestratorInstancesSyncStore{pool: pool} - pool.synchronization = synchronization.NewSynchronize("orchestrator-instances", "Orchestrator instances", store) + store := &instancesSyncStore{pool: pool} + pool.synchronization = synchronization.NewSynchronize("instances", "Instances", store) // Background synchronization of orchestrators pool - go func() { - pool.synchronization.Start(ctx, orchestratorsPoolInterval, orchestratorsPoolRoundTimeout, true) - }() - go func() { pool.statusLogSync(ctx) }() + go pool.synchronization.Start(ctx, instancesPoolInterval, instancesPoolRoundTimeout, true) + go pool.statusLogSync(ctx) return pool } -func (p *OrchestratorsPool) GetOrchestrators() map[string]*Instance { +func (p *InstancesPool) GetOrchestrators() map[string]*Instance { return p.instances.Items() } -func (p *OrchestratorsPool) GetOrchestrator(instanceID string) (i *Instance, ok bool) { +func (p *InstancesPool) GetOrchestrator(instanceID string) (i *Instance, ok bool) { orchestrators := p.GetOrchestrators() for _, i = range orchestrators { if i.GetInfo().ServiceInstanceID == instanceID { @@ -84,7 +82,7 @@ func (p *OrchestratorsPool) GetOrchestrator(instanceID string) (i *Instance, ok return nil, false } -func (p *OrchestratorsPool) statusLogSync(ctx context.Context) { +func (p *InstancesPool) statusLogSync(ctx context.Context) { ticker := time.NewTicker(statusLogInterval) defer ticker.Stop() @@ -105,7 +103,7 @@ func (p *OrchestratorsPool) statusLogSync(ctx context.Context) { } } -func (p *OrchestratorsPool) Close(ctx context.Context) error { +func (p *InstancesPool) Close(ctx context.Context) error { p.synchronization.Close() // Close all orchestrator instances in the pool @@ -125,19 +123,19 @@ func (p *OrchestratorsPool) Close(ctx context.Context) error { } // SynchronizationStore is an interface that defines methods for synchronizing the orchestrator instances inside the pool. -type orchestratorInstancesSyncStore struct { - pool *OrchestratorsPool +type instancesSyncStore struct { + pool *InstancesPool } -func (e *orchestratorInstancesSyncStore) getHost(ip string, port uint16) string { +func (e *instancesSyncStore) getHost(ip string, port uint16) string { return fmt.Sprintf("%s:%d", ip, port) } -func (e *orchestratorInstancesSyncStore) SourceList(ctx context.Context) ([]sd.DiscoveredInstance, error) { +func (e *instancesSyncStore) SourceList(ctx context.Context) ([]sd.DiscoveredInstance, error) { return e.pool.discovery.ListInstances(ctx) } -func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd.DiscoveredInstance, p *Instance) bool { +func (e *instancesSyncStore) SourceExists(_ context.Context, s []sd.DiscoveredInstance, p *Instance) bool { itself := p.GetInfo() for _, item := range s { itemHost := e.getHost(item.InstanceIPAddress, item.InstancePort) @@ -149,7 +147,7 @@ func (e *orchestratorInstancesSyncStore) SourceExists(_ context.Context, s []sd. return false } -func (e *orchestratorInstancesSyncStore) PoolList(_ context.Context) []*Instance { +func (e *instancesSyncStore) PoolList(_ context.Context) []*Instance { items := make([]*Instance, 0) for _, item := range e.pool.instances.Items() { items = append(items, item) @@ -158,14 +156,14 @@ func (e *orchestratorInstancesSyncStore) PoolList(_ context.Context) []*Instance return items } -func (e *orchestratorInstancesSyncStore) PoolExists(_ context.Context, source sd.DiscoveredInstance) bool { +func (e *instancesSyncStore) PoolExists(_ context.Context, source sd.DiscoveredInstance) bool { host := e.getHost(source.InstanceIPAddress, source.InstancePort) _, found := e.pool.instances.Get(host) return found } -func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source sd.DiscoveredInstance) { +func (e *instancesSyncStore) PoolInsert(ctx context.Context, source sd.DiscoveredInstance) { host := e.getHost(source.InstanceIPAddress, source.InstancePort) o, err := newInstance(e.pool.tracerProvider, e.pool.metricProvider, source.InstanceIPAddress, source.InstancePort) if err != nil { @@ -174,7 +172,7 @@ func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source return } - ctx, cancel := context.WithTimeout(ctx, orchestratorsInstanceSyncTimeout) + ctx, cancel := context.WithTimeout(ctx, instanceSyncTimeout) defer cancel() // Initial synchronization of the orchestrator Instance @@ -189,8 +187,8 @@ func (e *orchestratorInstancesSyncStore) PoolInsert(ctx context.Context, source e.pool.instances.Insert(host, o) } -func (e *orchestratorInstancesSyncStore) PoolUpdate(ctx context.Context, item *Instance) { - ctx, cancel := context.WithTimeout(ctx, orchestratorsInstanceSyncTimeout) +func (e *instancesSyncStore) PoolUpdate(ctx context.Context, item *Instance) { + ctx, cancel := context.WithTimeout(ctx, instanceSyncTimeout) defer cancel() err := item.sync(ctx) @@ -199,7 +197,7 @@ func (e *orchestratorInstancesSyncStore) PoolUpdate(ctx context.Context, item *I } } -func (e *orchestratorInstancesSyncStore) PoolRemove(ctx context.Context, item *Instance) { +func (e *instancesSyncStore) PoolRemove(ctx context.Context, item *Instance) { info := item.GetInfo() logger.L().Info(ctx, "Orchestrator Instance connection is not active anymore, closing.", logger.WithNodeID(info.NodeID)) diff --git a/packages/client-proxy/internal/edge/service.go b/packages/client-proxy/internal/edge/service.go index 927ca44220..6f001a0845 100644 --- a/packages/client-proxy/internal/edge/service.go +++ b/packages/client-proxy/internal/edge/service.go @@ -8,7 +8,7 @@ import ( "github.com/e2b-dev/infra/packages/proxy/internal/cfg" "github.com/e2b-dev/infra/packages/proxy/internal/edge/handlers" e2binfo "github.com/e2b-dev/infra/packages/proxy/internal/edge/info" - e2borchestrators "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool" + e2binstances "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool" "github.com/e2b-dev/infra/packages/shared/pkg/logger" catalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog" ) @@ -19,11 +19,11 @@ func NewEdgeAPIStore( ctx context.Context, l logger.Logger, info *e2binfo.ServiceInfo, - orchestrators *e2borchestrators.OrchestratorsPool, + instances *e2binstances.InstancesPool, catalog catalog.SandboxesCatalog, config cfg.Config, ) (*handlers.APIStore, error) { - store, err := handlers.NewStore(ctx, l, info, orchestrators, catalog, config) + store, err := handlers.NewStore(ctx, l, info, instances, catalog, config) if err != nil { return nil, err } diff --git a/packages/client-proxy/main.go b/packages/client-proxy/main.go index a1a33a9d04..209da04abf 100644 --- a/packages/client-proxy/main.go +++ b/packages/client-proxy/main.go @@ -146,7 +146,7 @@ func run() int { catalog = e2bcatalog.NewMemorySandboxesCatalog() } - orchestrators := e2borchestrators.NewOrchestratorsPool(ctx, l, tel.TracerProvider, tel.MeterProvider, orchestratorsSD) + instances := e2borchestrators.NewInstancesPool(ctx, l, tel.TracerProvider, tel.MeterProvider, orchestratorsSD) info := &e2binfo.ServiceInfo{ NodeID: nodeID, @@ -176,9 +176,9 @@ func run() int { authorizationManager := authorization.NewStaticTokenAuthorizationService(config.EdgeSecret) var closers []Closeable - closers = append(closers, orchestrators, featureFlagsClient, catalog) + closers = append(closers, instances, featureFlagsClient, catalog) - edgeApiStore, err := edge.NewEdgeAPIStore(ctx, l, info, orchestrators, catalog, config) + edgeApiStore, err := edge.NewEdgeAPIStore(ctx, l, info, instances, catalog, config) if err != nil { l.Error(ctx, "failed to create edge api store", zap.Error(err)) @@ -205,7 +205,7 @@ func run() int { // Edge Pass Through Proxy for direct communication with orchestrator nodes grpcListener := muxServer.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) // handler requests for gRPC pass through - grpcSrv := edgepassthrough.NewNodePassThroughServer(orchestrators, info, authorizationManager, catalog) + grpcSrv := edgepassthrough.NewNodePassThroughServer(instances, info, authorizationManager, catalog) // Edge REST API restHttpHandler := edge.NewGinServer(l, edgeApiStore, edgeApiSwagger, authorizationManager) From f95844158145d8fed2561251e187854d0b192039 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:26:44 +0100 Subject: [PATCH 06/58] Unused feature flag removed --- packages/shared/pkg/feature-flags/flags.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/shared/pkg/feature-flags/flags.go b/packages/shared/pkg/feature-flags/flags.go index 0f710a575c..5ea1c2f368 100644 --- a/packages/shared/pkg/feature-flags/flags.go +++ b/packages/shared/pkg/feature-flags/flags.go @@ -110,7 +110,6 @@ var ( BestOfKSampleSize = newIntFlag("best-of-k-sample-size", 3) // Default K=3 BestOfKMaxOvercommit = newIntFlag("best-of-k-max-overcommit", 400) // Default R=4 (stored as percentage, max over-commit ratio) BestOfKAlpha = newIntFlag("best-of-k-alpha", 50) // Default Alpha=0.5 (stored as percentage for int flag, current usage weight) - PubsubQueueChannelSize = newIntFlag("pubsub-queue-channel-size", 8*1024) // size of the channel buffer used to queue incoming sandbox events EnvdInitTimeoutMilliseconds = newIntFlag("envd-init-request-timeout-milliseconds", 50) // Timeout for envd init request in milliseconds // BuildCacheMaxUsagePercentage the maximum percentage of the cache disk storage From 2c9ea210e3b4edc71fa3102a4fe1b733e0f1a1be Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:29:33 +0100 Subject: [PATCH 07/58] Renamed execution id to sandbox execution id in sandbox info struct Removed to make it properly prefixed as other fields here. --- packages/api/internal/orchestrator/lifecycle.go | 2 +- packages/client-proxy/internal/edge-pass-through/events.go | 2 +- packages/shared/pkg/sandbox-catalog/catalog.go | 2 +- packages/shared/pkg/sandbox-catalog/catalog_memory.go | 2 +- packages/shared/pkg/sandbox-catalog/catalog_redis.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/api/internal/orchestrator/lifecycle.go b/packages/api/internal/orchestrator/lifecycle.go index febb687b5a..51689f0804 100644 --- a/packages/api/internal/orchestrator/lifecycle.go +++ b/packages/api/internal/orchestrator/lifecycle.go @@ -19,8 +19,8 @@ func (o *Orchestrator) addSandboxToRoutingTable(ctx context.Context, sandbox san info := e2bcatalog.SandboxInfo{ OrchestratorID: node.Metadata().ServiceInstanceID, OrchestratorIP: node.IPAddress, - ExecutionID: sandbox.ExecutionID, + SandboxExecutionID: sandbox.ExecutionID, SandboxStartedAt: sandbox.StartTime, SandboxMaxLengthInHours: int64(sandbox.MaxInstanceLength / time.Hour), } diff --git a/packages/client-proxy/internal/edge-pass-through/events.go b/packages/client-proxy/internal/edge-pass-through/events.go index 8756279917..2617fdf745 100644 --- a/packages/client-proxy/internal/edge-pass-through/events.go +++ b/packages/client-proxy/internal/edge-pass-through/events.go @@ -53,8 +53,8 @@ func (s *NodePassThroughServer) catalogCreateEventHandler(ctx context.Context, m &catalog.SandboxInfo{ OrchestratorID: c.OrchestratorID, OrchestratorIP: o.GetInfo().IP, - ExecutionID: c.ExecutionID, + SandboxExecutionID: c.ExecutionID, SandboxStartedAt: c.SandboxStartTime, SandboxMaxLengthInHours: c.SandboxMaxLengthInHours, }, diff --git a/packages/shared/pkg/sandbox-catalog/catalog.go b/packages/shared/pkg/sandbox-catalog/catalog.go index 0eff6f0267..3ae5e7f761 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog.go +++ b/packages/shared/pkg/sandbox-catalog/catalog.go @@ -11,8 +11,8 @@ import ( type SandboxInfo struct { OrchestratorID string `json:"orchestrator_id"` OrchestratorIP string `json:"orchestrator_ip"` // used only for cases where orchestrator is not registered in edge pool - ExecutionID string `json:"execution_id"` + SandboxExecutionID string `json:"execution_id"` SandboxStartedAt time.Time `json:"sandbox_started_at"` // when sandbox was started SandboxMaxLengthInHours int64 `json:"sandbox_max_length_in_hours"` // how long can sandbox can possibly run (in hours) } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_memory.go b/packages/shared/pkg/sandbox-catalog/catalog_memory.go index b922117bc4..8300eae6d2 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_memory.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_memory.go @@ -71,7 +71,7 @@ func (c *MemorySandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID stri } // Different execution is stored in the cache, we don't want to remove it - if item.Value().ExecutionID != executionID { + if item.Value().SandboxExecutionID != executionID { return nil } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_redis.go b/packages/shared/pkg/sandbox-catalog/catalog_redis.go index 7fc75986bc..2e7bd5af41 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_redis.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_redis.go @@ -118,7 +118,7 @@ func (c *RedisSandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID strin } // Different execution is stored in the cache, we don't want to remove it - if info.ExecutionID != executionID { + if info.SandboxExecutionID != executionID { return nil } From 293329f0d72595bfeb7d619a6a44c73fe825d74b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:34:48 +0100 Subject: [PATCH 08/58] Renamed package storing clusters As we are migrating away from using edge api for local cluster and going towards removing need to deploy edge api for OSS deployment renamed was needed here. --- packages/api/internal/{edge => clusters}/client.go | 2 +- packages/api/internal/{edge => clusters}/cluster.go | 4 ++-- .../api/internal/{edge => clusters}/cluster_instances.go | 2 +- packages/api/internal/{edge => clusters}/metrics.go | 2 +- packages/api/internal/{edge => clusters}/pool.go | 2 +- packages/api/internal/handlers/sandbox_metrics.go | 4 ++-- packages/api/internal/handlers/sandboxes_list_metrics.go | 4 ++-- packages/api/internal/handlers/store.go | 6 +++--- packages/api/internal/orchestrator/client.go | 4 ++-- packages/api/internal/orchestrator/nodemanager/node.go | 4 ++-- packages/api/internal/orchestrator/orchestrator.go | 6 +++--- packages/api/internal/template-manager/build_client.go | 4 ++-- packages/api/internal/template-manager/logs.go | 4 ++-- .../api/internal/template-manager/logs/edge_provider.go | 4 ++-- .../template-manager/logs/template_manager_provider.go | 4 ++-- .../api/internal/template-manager/template_manager.go | 8 ++++---- 16 files changed, 32 insertions(+), 32 deletions(-) rename packages/api/internal/{edge => clusters}/client.go (99%) rename packages/api/internal/{edge => clusters}/cluster.go (99%) rename packages/api/internal/{edge => clusters}/cluster_instances.go (99%) rename packages/api/internal/{edge => clusters}/metrics.go (99%) rename packages/api/internal/{edge => clusters}/pool.go (99%) diff --git a/packages/api/internal/edge/client.go b/packages/api/internal/clusters/client.go similarity index 99% rename from packages/api/internal/edge/client.go rename to packages/api/internal/clusters/client.go index 0177707198..deccf88b47 100644 --- a/packages/api/internal/edge/client.go +++ b/packages/api/internal/clusters/client.go @@ -1,4 +1,4 @@ -package edge +package clusters import ( "context" diff --git a/packages/api/internal/edge/cluster.go b/packages/api/internal/clusters/cluster.go similarity index 99% rename from packages/api/internal/edge/cluster.go rename to packages/api/internal/clusters/cluster.go index f7b0e4b0e3..a487cfc42e 100644 --- a/packages/api/internal/edge/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -1,4 +1,4 @@ -package edge +package clusters import ( "context" @@ -20,7 +20,7 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) -var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/edge") +var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/clusters") type Cluster struct { ID uuid.UUID diff --git a/packages/api/internal/edge/cluster_instances.go b/packages/api/internal/clusters/cluster_instances.go similarity index 99% rename from packages/api/internal/edge/cluster_instances.go rename to packages/api/internal/clusters/cluster_instances.go index a23a75b107..ae7da148f3 100644 --- a/packages/api/internal/edge/cluster_instances.go +++ b/packages/api/internal/clusters/cluster_instances.go @@ -1,4 +1,4 @@ -package edge +package clusters import ( "context" diff --git a/packages/api/internal/edge/metrics.go b/packages/api/internal/clusters/metrics.go similarity index 99% rename from packages/api/internal/edge/metrics.go rename to packages/api/internal/clusters/metrics.go index a5a813f67e..d6f72f7721 100644 --- a/packages/api/internal/edge/metrics.go +++ b/packages/api/internal/clusters/metrics.go @@ -1,4 +1,4 @@ -package edge +package clusters import ( "context" diff --git a/packages/api/internal/edge/pool.go b/packages/api/internal/clusters/pool.go similarity index 99% rename from packages/api/internal/edge/pool.go rename to packages/api/internal/clusters/pool.go index 33144234fe..53ed7a7475 100644 --- a/packages/api/internal/edge/pool.go +++ b/packages/api/internal/clusters/pool.go @@ -1,4 +1,4 @@ -package edge +package clusters import ( "context" diff --git a/packages/api/internal/handlers/sandbox_metrics.go b/packages/api/internal/handlers/sandbox_metrics.go index 393ab4a54e..0c14cecf1b 100644 --- a/packages/api/internal/handlers/sandbox_metrics.go +++ b/packages/api/internal/handlers/sandbox_metrics.go @@ -12,8 +12,8 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/db/types" - "github.com/e2b-dev/infra/packages/api/internal/edge" "github.com/e2b-dev/infra/packages/api/internal/utils" clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" clickhouseUtils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" @@ -58,7 +58,7 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(c *gin.Context, sandboxID string var metrics []api.SandboxMetric var apiErr *api.APIError if edgeProvidedMetrics { - metrics, apiErr = edge.GetClusterSandboxMetrics( + metrics, apiErr = clusters.GetClusterSandboxMetrics( ctx, a.clustersPool, sandboxID, diff --git a/packages/api/internal/handlers/sandboxes_list_metrics.go b/packages/api/internal/handlers/sandboxes_list_metrics.go index 345804b5bc..60ef89c8ea 100644 --- a/packages/api/internal/handlers/sandboxes_list_metrics.go +++ b/packages/api/internal/handlers/sandboxes_list_metrics.go @@ -13,8 +13,8 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/db/types" - "github.com/e2b-dev/infra/packages/api/internal/edge" "github.com/e2b-dev/infra/packages/api/internal/utils" featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -57,7 +57,7 @@ func (a *APIStore) getSandboxesMetrics( var metrics map[string]api.SandboxMetric var apiErr *api.APIError if edgeProvidedMetrics { - metrics, apiErr = edge.GetClusterSandboxListMetrics( + metrics, apiErr = clusters.GetClusterSandboxListMetrics( ctx, a.clustersPool, teamID.String(), diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index b73871127f..5a53d4ef3c 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -20,9 +20,9 @@ import ( authcache "github.com/e2b-dev/infra/packages/api/internal/cache/auth" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" "github.com/e2b-dev/infra/packages/api/internal/cfg" + "github.com/e2b-dev/infra/packages/api/internal/clusters" dbapi "github.com/e2b-dev/infra/packages/api/internal/db" "github.com/e2b-dev/infra/packages/api/internal/db/types" - "github.com/e2b-dev/infra/packages/api/internal/edge" "github.com/e2b-dev/infra/packages/api/internal/orchestrator" "github.com/e2b-dev/infra/packages/api/internal/sandbox" template_manager "github.com/e2b-dev/infra/packages/api/internal/template-manager" @@ -58,7 +58,7 @@ type APIStore struct { clickhouseStore clickhouse.Clickhouse accessTokenGenerator *sandbox.AccessTokenGenerator featureFlags *featureflags.Client - clustersPool *edge.Pool + clustersPool *clusters.Pool } func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) *APIStore { @@ -110,7 +110,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) } } - clustersPool, err := edge.NewPool(ctx, tel, sqlcDB, config) + clustersPool, err := clusters.NewPool(ctx, tel, sqlcDB, config) if err != nil { logger.L().Fatal(ctx, "initializing edge clusters pool failed", zap.Error(err)) } diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index 4be468c80c..10d77aeba9 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -9,7 +9,7 @@ import ( nomadapi "github.com/hashicorp/nomad/api" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" "github.com/e2b-dev/infra/packages/shared/pkg/consts" @@ -33,7 +33,7 @@ func (o *Orchestrator) connectToNode(ctx context.Context, discovered nodemanager return nil } -func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *edge.Cluster, i *edge.ClusterInstance) { +func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.ClusterInstance) { // this way we don't need to worry about multiple clusters with the same node ID in shared pool clusterGRPC := cluster.GetGRPC(i.ServiceInstanceID) diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index 06027d1a7c..88b8bf3083 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -15,7 +15,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/consts" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -114,7 +114,7 @@ func New( return n, nil } -func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *edge.ClusterInstance) (*Node, error) { +func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.ClusterInstance) (*Node, error) { nodeStatus, ok := OrchestratorToApiNodeStateMapper[i.GetStatus()] if !ok { logger.L().Error(ctx, "Unknown service info status", zap.String("status", i.GetStatus().String()), logger.WithNodeID(i.NodeID)) diff --git a/packages/api/internal/orchestrator/orchestrator.go b/packages/api/internal/orchestrator/orchestrator.go index f9379ae4c7..55083f267a 100644 --- a/packages/api/internal/orchestrator/orchestrator.go +++ b/packages/api/internal/orchestrator/orchestrator.go @@ -14,7 +14,7 @@ import ( analyticscollector "github.com/e2b-dev/infra/packages/api/internal/analytics_collector" "github.com/e2b-dev/infra/packages/api/internal/cfg" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/metrics" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/evictor" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" @@ -49,7 +49,7 @@ type Orchestrator struct { routingCatalog e2bcatalog.SandboxesCatalog sqlcDB *sqlcdb.Client tel *telemetry.Client - clusters *edge.Pool + clusters *clusters.Pool metricsRegistration metric.Registration createdSandboxesCounter metric.Int64Counter teamMetricsObserver *metrics.TeamObserver @@ -66,7 +66,7 @@ func New( posthogClient *analyticscollector.PosthogClient, redisClient redis.UniversalClient, sqlcDB *sqlcdb.Client, - clusters *edge.Pool, + clusters *clusters.Pool, featureFlags *featureflags.Client, accessTokenGenerator *sandbox.AccessTokenGenerator, ) (*Orchestrator, error) { diff --git a/packages/api/internal/template-manager/build_client.go b/packages/api/internal/template-manager/build_client.go index 9951020a16..eb59736cf8 100644 --- a/packages/api/internal/template-manager/build_client.go +++ b/packages/api/internal/template-manager/build_client.go @@ -1,9 +1,9 @@ package template_manager import ( - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" ) type BuildClient struct { - GRPC *edge.ClusterGRPC + GRPC *clusters.ClusterGRPC } diff --git a/packages/api/internal/template-manager/logs.go b/packages/api/internal/template-manager/logs.go index 1ee2cda9f2..21df2f51f3 100644 --- a/packages/api/internal/template-manager/logs.go +++ b/packages/api/internal/template-manager/logs.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" buildlogs "github.com/e2b-dev/infra/packages/api/internal/template-manager/logs" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" @@ -21,7 +21,7 @@ const ( maxTimeRangeDuration = 7 * 24 * time.Hour ) -func GetBuildLogs(ctx context.Context, cluster *edge.Cluster, nodeID *string, templateID, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) []logs.LogEntry { +func GetBuildLogs(ctx context.Context, cluster *clusters.Cluster, nodeID *string, templateID, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) []logs.LogEntry { ctx, span := tracer.Start(ctx, "get build-logs") defer span.End() l := logger.L().With(logger.WithTemplateID(templateID), logger.WithBuildID(buildID)) diff --git a/packages/api/internal/template-manager/logs/edge_provider.go b/packages/api/internal/template-manager/logs/edge_provider.go index 2c14a89162..6bb5bc5d21 100644 --- a/packages/api/internal/template-manager/logs/edge_provider.go +++ b/packages/api/internal/template-manager/logs/edge_provider.go @@ -9,7 +9,7 @@ import ( "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" @@ -17,7 +17,7 @@ import ( ) type EdgeProvider struct { - HTTP *edge.ClusterHTTP + HTTP *clusters.ClusterHTTP } func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { diff --git a/packages/api/internal/template-manager/logs/template_manager_provider.go b/packages/api/internal/template-manager/logs/template_manager_provider.go index 1ae1fd7b61..6d2ecc2e7b 100644 --- a/packages/api/internal/template-manager/logs/template_manager_provider.go +++ b/packages/api/internal/template-manager/logs/template_manager_provider.go @@ -9,7 +9,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" @@ -18,7 +18,7 @@ import ( ) type TemplateManagerProvider struct { - GRPC *edge.ClusterGRPC + GRPC *clusters.ClusterGRPC } func (t *TemplateManagerProvider) GetLogs(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 5810773197..bfe8684416 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -14,7 +14,7 @@ import ( "google.golang.org/grpc/metadata" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" - "github.com/e2b-dev/infra/packages/api/internal/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/utils" sqlcdb "github.com/e2b-dev/infra/packages/db/client" "github.com/e2b-dev/infra/packages/db/queries" @@ -33,7 +33,7 @@ type processingBuilds struct { } type TemplateManager struct { - edgePool *edge.Pool + edgePool *clusters.Pool lock sync.Mutex processing map[uuid.UUID]processingBuilds @@ -58,7 +58,7 @@ const ( func New( sqlcDB *sqlcdb.Client, - edgePool *edge.Pool, + edgePool *clusters.Pool, buildCache *templatecache.TemplatesBuildCache, templateCache *templatecache.TemplateCache, featureFlags *featureflags.Client, @@ -108,7 +108,7 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { } } -func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterID uuid.UUID) (*edge.ClusterInstance, error) { +func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterID uuid.UUID) (*clusters.ClusterInstance, error) { cluster, ok := tm.edgePool.GetClusterById(clusterID) if !ok { return nil, fmt.Errorf("cluster with ID '%s' not found", clusterID) From ae7a01f39747648b25fd3b9ef87b91d9629ff345 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 12:44:10 +0100 Subject: [PATCH 09/58] Introduced cluster resources provider This is interface ment for unify all calls to resources we want to query from cluster (sandbox logs, metrics, build logs etc). This commit introduces basic implementation and getter on cluster that will be later used on all places where we need to query sandbox/build logs and/or metrics. Later this same interface will be used for local cluster to query local Loki and ClickHouse to provide same data. This way we should have unified interface. --- packages/api/internal/clusters/cluster.go | 19 +- packages/api/internal/clusters/resources.go | 37 ++++ .../api/internal/clusters/resources_impl.go | 173 ++++++++++++++++++ 3 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 packages/api/internal/clusters/resources.go create mode 100644 packages/api/internal/clusters/resources_impl.go diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index a487cfc42e..b660d866d8 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -23,14 +23,14 @@ import ( var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/clusters") type Cluster struct { - ID uuid.UUID - - httpClient *api.ClientWithResponses - grpcClient *grpclient.GRPCClient + ID uuid.UUID + SandboxDomain *string + httpClient *api.ClientWithResponses + grpcClient *grpclient.GRPCClient instances *smap.Map[*ClusterInstance] synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *ClusterInstance] - SandboxDomain *string + resources ClusterResource } type ClusterGRPC struct { @@ -80,11 +80,14 @@ func NewCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end return nil, fmt.Errorf("failed to create grpc client: %w", err) } + instances := smap.New[*ClusterInstance]() + c := &Cluster{ ID: clusterID, SandboxDomain: sandboxDomain, - instances: smap.New[*ClusterInstance](), + resources: newRemoteClusterResourceProvider(instances, httpClient), + instances: instances, httpClient: httpClient, grpcClient: grpcClient, } @@ -180,3 +183,7 @@ func (c *Cluster) GetOrchestrators() []*ClusterInstance { func (c *Cluster) GetHttpClient() *api.ClientWithResponses { return c.httpClient } + +func (c *Cluster) GetResources() ClusterResource { + return c.resources +} diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go new file mode 100644 index 0000000000..f305d58574 --- /dev/null +++ b/packages/api/internal/clusters/resources.go @@ -0,0 +1,37 @@ +package clusters + +import ( + "context" + "time" + + "github.com/e2b-dev/infra/packages/api/internal/api" + "github.com/e2b-dev/infra/packages/shared/pkg/logs" +) + +type ClusterResource interface { + GetSandboxMetrics(ctx context.Context, teamID string, sandboxID string, qStart *int64, qEnd *int64) ([]api.SandboxMetric, error) + GetSandboxesMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, error) + GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, limit *int32) (api.SandboxLogs, error) + GetBuildLogs(ctx context.Context, nodeID *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) +} + +const ( + maxTimeRangeDuration = 7 * 24 * time.Hour +) + +func logQueryWindow(cursor *time.Time, direction api.LogsDirection) (time.Time, time.Time) { + start, end := time.Now().Add(-maxTimeRangeDuration), time.Now() + if cursor == nil { + return start, end + } + + if direction == api.LogsDirectionForward { + start = *cursor + end = start.Add(maxTimeRangeDuration) + } else { + end = *cursor + start = end.Add(-maxTimeRangeDuration) + } + + return start, end +} diff --git a/packages/api/internal/clusters/resources_impl.go b/packages/api/internal/clusters/resources_impl.go new file mode 100644 index 0000000000..349af93469 --- /dev/null +++ b/packages/api/internal/clusters/resources_impl.go @@ -0,0 +1,173 @@ +package clusters + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/api/internal/api" + edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/logs" + "github.com/e2b-dev/infra/packages/shared/pkg/smap" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" +) + +type ClusterResourceProviderImpl struct { + instances *smap.Map[*ClusterInstance] + client *edgeapi.ClientWithResponses +} + +func newRemoteClusterResourceProvider(instances *smap.Map[*ClusterInstance], client *edgeapi.ClientWithResponses) ClusterResource { + return &ClusterResourceProviderImpl{ + instances: instances, + client: client, + } +} + +func (r *ClusterResourceProviderImpl) GetSandboxMetrics(ctx context.Context, teamID string, sandboxID string, qStart *int64, qEnd *int64) ([]api.SandboxMetric, error) { + req := &edgeapi.V1SandboxMetricsParams{ + TeamID: teamID, + Start: qStart, + End: qEnd, + } + + res, err := r.client.V1SandboxMetricsWithResponse(ctx, sandboxID, req) + if err != nil { + return nil, err + } + + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) + } + + raw := *res.JSON200 + items := make([]api.SandboxMetric, len(raw)) + for i, m := range raw { + items[i] = api.SandboxMetric{ + Timestamp: m.Timestamp, + TimestampUnix: m.TimestampUnix, + CpuUsedPct: m.CpuUsedPct, + CpuCount: m.CpuCount, + MemTotal: m.MemTotal, + MemUsed: m.MemUsed, + DiskTotal: m.DiskTotal, + DiskUsed: m.DiskUsed, + } + } + + return items, nil +} + +func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, error) { + res, err := r.client.V1SandboxesMetricsWithResponse(ctx, &edgeapi.V1SandboxesMetricsParams{TeamID: teamID, SandboxIds: sandboxIDs}) + if err != nil { + return nil, err + } + + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) + } + + raw := *res.JSON200 + items := make(map[string]api.SandboxMetric, len(raw.Sandboxes)) + for sbxID, v := range raw.Sandboxes { + items[sbxID] = api.SandboxMetric{ + Timestamp: v.Timestamp, + TimestampUnix: v.TimestampUnix, + CpuUsedPct: v.CpuUsedPct, + CpuCount: v.CpuCount, + MemTotal: v.MemTotal, + MemUsed: v.MemUsed, + DiskTotal: v.DiskTotal, + DiskUsed: v.DiskUsed, + } + } + + return map[string]api.SandboxMetric{}, nil +} + +func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, limit *int32) (api.SandboxLogs, error) { + res, err := r.client.V1SandboxLogsWithResponse(ctx, sandboxID, &edgeapi.V1SandboxLogsParams{TeamID: teamID, Start: start, Limit: limit}) + if err != nil { + return api.SandboxLogs{}, err + } + + if res.StatusCode() != http.StatusOK { + return api.SandboxLogs{}, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) + } + + raw := *res.JSON200 + l := make([]api.SandboxLog, len(raw.Logs)) + for i, row := range raw.Logs { + l[i] = api.SandboxLog{ + Line: row.Line, + Timestamp: row.Timestamp, + } + } + + le := make([]api.SandboxLogEntry, len(raw.LogEntries)) + for i, row := range raw.LogEntries { + le[i] = api.SandboxLogEntry{ + Timestamp: row.Timestamp, + Level: api.LogLevel(row.Level), + Message: row.Message, + Fields: row.Fields, + } + } + + return api.SandboxLogs{Logs: l, LogEntries: le}, nil +} + +func (r *ClusterResourceProviderImpl) GetBuildLogs(ctx context.Context, _ *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) { + start, end := logQueryWindow(cursor, direction) + res, err := r.client.V1TemplateBuildLogsWithResponse( + ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{ + TemplateID: templateID, + Offset: &offset, + Limit: &limit, + Level: logToEdgeLevel(level), + // TODO: remove this once the API spec is not required to have orchestratorID (https://linear.app/e2b/issue/ENG-3352) + OrchestratorID: utils.ToPtr("unused"), + Start: utils.ToPtr(start.UnixMilli()), + End: utils.ToPtr(end.UnixMilli()), + Direction: utils.ToPtr(edgeapi.V1TemplateBuildLogsParamsDirection(direction)), + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to get build logs in template manager: %w", err) + } + + if res.StatusCode() != 200 || res.JSON200 == nil { + logger.L().Error(ctx, "failed to get build logs in template manager", zap.String("body", string(res.Body))) + + return nil, errors.New("failed to get build logs in template manager") + } + + raw := *res.JSON200 + l := make([]logs.LogEntry, len(raw.LogEntries)) + for i, entry := range raw.LogEntries { + l[i] = logs.LogEntry{ + Timestamp: entry.Timestamp, + Message: entry.Message, + Level: logs.StringToLevel(string(entry.Level)), + Fields: entry.Fields, + } + } + + return l, nil +} + +func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { + if level == nil { + return nil + } + + value := edgeapi.LogLevel(logs.LevelToString(*level)) + + return &value +} From 718f50f97b45f0a5528de84913cf7085aedbd40a Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 13:37:51 +0100 Subject: [PATCH 10/58] Use cluster resource provider for getting sbx and build resources This change removes local/remove cluster specific backends for query sandbox logs, metrics and build logs and replaces it with cluster resource backend that is now unified. --- packages/api/internal/clusters/metrics.go | 128 ------------------ .../api/internal/handlers/sandbox_logs.go | 64 +-------- .../api/internal/handlers/sandbox_metrics.go | 119 ++-------------- .../handlers/sandboxes_list_metrics.go | 60 ++------ .../internal/handlers/template_build_logs.go | 19 +-- .../handlers/template_build_status.go | 3 +- .../internal/template-manager/build_client.go | 9 -- .../template-manager/create_template.go | 8 +- .../api/internal/template-manager/logs.go | 74 ---------- .../template-manager/logs/edge_provider.go | 68 ---------- .../template-manager/logs/provider.go | 13 -- .../logs/template_manager_provider.go | 73 ---------- .../template-manager/template_manager.go | 41 +++--- .../upload_template_layer_files.go | 10 +- packages/shared/pkg/feature-flags/flags.go | 13 +- 15 files changed, 80 insertions(+), 622 deletions(-) delete mode 100644 packages/api/internal/clusters/metrics.go delete mode 100644 packages/api/internal/template-manager/build_client.go delete mode 100644 packages/api/internal/template-manager/logs.go delete mode 100644 packages/api/internal/template-manager/logs/edge_provider.go delete mode 100644 packages/api/internal/template-manager/logs/provider.go delete mode 100644 packages/api/internal/template-manager/logs/template_manager_provider.go diff --git a/packages/api/internal/clusters/metrics.go b/packages/api/internal/clusters/metrics.go deleted file mode 100644 index d6f72f7721..0000000000 --- a/packages/api/internal/clusters/metrics.go +++ /dev/null @@ -1,128 +0,0 @@ -package clusters - -import ( - "context" - "fmt" - "net/http" - - "github.com/google/uuid" - - "github.com/e2b-dev/infra/packages/api/internal/api" - apiedge "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" -) - -func GetClusterSandboxMetrics(ctx context.Context, pool *Pool, sandboxID string, teamID string, clusterID uuid.UUID, qStart *int64, qEnd *int64) ([]api.SandboxMetric, *api.APIError) { - cluster, ok := pool.GetClusterById(clusterID) - if !ok { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: fmt.Sprintf("Error getting cluster '%s'", clusterID), - Err: fmt.Errorf("cluster with ID '%s' not found", clusterID), - } - } - - res, err := cluster.GetHttpClient().V1SandboxMetricsWithResponse( - ctx, sandboxID, &apiedge.V1SandboxMetricsParams{ - TeamID: teamID, - Start: qStart, - End: qEnd, - }, - ) - if err != nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: fmt.Sprintf("Error getting metrics for sandbox '%s'", sandboxID), - Err: fmt.Errorf("error getting metrics for sandbox '%s': %w", sandboxID, err), - } - } - - if res.StatusCode() != http.StatusOK { - return nil, &api.APIError{ - Code: res.StatusCode(), - ClientMsg: fmt.Sprintf("Error getting metrics for sandbox '%s'", sandboxID), - Err: fmt.Errorf("unexpected response for sandbox - HTTP status '%d'", res.StatusCode()), - } - } - - if res.JSON200 == nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: fmt.Sprintf("Error getting metrics for sandbox '%s'", sandboxID), - Err: fmt.Errorf("no metrics returned for sandbox '%s'", sandboxID), - } - } - - // Transform edge types (snake_case) to API types (camelCase) - apiMetrics := make([]api.SandboxMetric, len(*res.JSON200)) - for i, m := range *res.JSON200 { - apiMetrics[i] = api.SandboxMetric{ - Timestamp: m.Timestamp, - TimestampUnix: m.TimestampUnix, - CpuUsedPct: m.CpuUsedPct, - CpuCount: m.CpuCount, - MemTotal: m.MemTotal, - MemUsed: m.MemUsed, - DiskTotal: m.DiskTotal, - DiskUsed: m.DiskUsed, - } - } - - return apiMetrics, nil -} - -func GetClusterSandboxListMetrics(ctx context.Context, pool *Pool, teamID string, clusterID uuid.UUID, sandboxIDs []string) (map[string]api.SandboxMetric, *api.APIError) { - cluster, ok := pool.GetClusterById(clusterID) - if !ok { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: fmt.Sprintf("Error getting cluster '%s'", clusterID), - Err: fmt.Errorf("cluster with ID '%s' not found", clusterID), - } - } - - res, err := cluster.GetHttpClient().V1SandboxesMetricsWithResponse( - ctx, &apiedge.V1SandboxesMetricsParams{ - TeamID: teamID, - SandboxIds: sandboxIDs, - }, - ) - if err != nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: "Error getting metrics for sandbox list", - Err: fmt.Errorf("error getting metrics for sandbox list: %w", err), - } - } - - if res.StatusCode() != http.StatusOK { - return nil, &api.APIError{ - Code: res.StatusCode(), - ClientMsg: "Error getting metrics for sandbox list", - Err: fmt.Errorf("unexpected response for sandbox list - HTTP status '%d'", res.StatusCode()), - } - } - - if res.JSON200 == nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: "Error getting metrics for sandbox list", - Err: fmt.Errorf("no metrics returned for sandbox list"), - } - } - - apiMetrics := make(map[string]api.SandboxMetric) - for key, m := range res.JSON200.Sandboxes { - apiMetrics[key] = api.SandboxMetric{ - Timestamp: m.Timestamp, - TimestampUnix: m.TimestampUnix, - CpuUsedPct: m.CpuUsedPct, - CpuCount: m.CpuCount, - MemTotal: m.MemTotal, - MemUsed: m.MemUsed, - DiskTotal: m.DiskTotal, - DiskUsed: m.DiskUsed, - } - } - - return apiMetrics, nil -} diff --git a/packages/api/internal/handlers/sandbox_logs.go b/packages/api/internal/handlers/sandbox_logs.go index ce1d795544..16ec37b683 100644 --- a/packages/api/internal/handlers/sandbox_logs.go +++ b/packages/api/internal/handlers/sandbox_logs.go @@ -1,19 +1,16 @@ package handlers import ( - "context" "fmt" "net/http" "github.com/gin-gonic/gin" - "github.com/google/uuid" "go.opentelemetry.io/otel/attribute" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" "github.com/e2b-dev/infra/packages/api/internal/db/types" "github.com/e2b-dev/infra/packages/api/internal/utils" - apiedge "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -28,69 +25,22 @@ func (a *APIStore) GetSandboxesSandboxIDLogs(c *gin.Context, sandboxID string, p telemetry.WithTeamID(team.ID.String()), ) - // Sandboxes living in a cluster - sbxLogs, err := a.getClusterSandboxLogs(ctx, sandboxID, team.ID.String(), utils.WithClusterFallback(team.ClusterID), params.Limit, params.Start) - if err != nil { - a.sendAPIStoreError(c, int(err.Code), err.Message) - - return - } - - c.JSON(http.StatusOK, sbxLogs) -} - -func (a *APIStore) getClusterSandboxLogs(ctx context.Context, sandboxID string, teamID string, clusterID uuid.UUID, qLimit *int32, qStart *int64) (*api.SandboxLogs, *api.Error) { + clusterID := utils.WithClusterFallback(team.ClusterID) cluster, ok := a.clustersPool.GetClusterById(clusterID) if !ok { telemetry.ReportCriticalError(ctx, "error getting cluster by ID", fmt.Errorf("cluster with ID '%s' not found", clusterID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error getting cluster '%s'", clusterID)) - return nil, &api.Error{ - Code: http.StatusInternalServerError, - Message: fmt.Sprintf("Error getting cluster '%s'", clusterID), - } + return } - res, err := cluster.GetHttpClient().V1SandboxLogsWithResponse( - ctx, sandboxID, &apiedge.V1SandboxLogsParams{ - TeamID: teamID, - Start: qStart, - Limit: qLimit, - }, - ) + logs, err := cluster.GetResources().GetSandboxLogs(ctx, team.ID.String(), sandboxID, params.Start, params.Limit) if err != nil { telemetry.ReportCriticalError(ctx, "error when returning logs for sandbox", err) + a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error returning logs for sandbox '%s'", sandboxID)) - return nil, &api.Error{ - Code: http.StatusInternalServerError, - Message: fmt.Sprintf("Error returning logs for sandbox '%s'", sandboxID), - } - } - - if res.JSON200 == nil { - telemetry.ReportCriticalError(ctx, "error when returning logs for sandbox", fmt.Errorf("unexpected response for sandbox '%s': %s", sandboxID, string(res.Body))) - - return nil, &api.Error{ - Code: http.StatusInternalServerError, - Message: fmt.Sprintf("Error returning logs for sandbox '%s'", sandboxID), - } - } - - l := make([]api.SandboxLog, 0) - for _, row := range res.JSON200.Logs { - l = append(l, api.SandboxLog{Line: row.Line, Timestamp: row.Timestamp}) - } - - le := make([]api.SandboxLogEntry, 0) - for _, row := range res.JSON200.LogEntries { - le = append( - le, api.SandboxLogEntry{ - Timestamp: row.Timestamp, - Level: api.LogLevel(row.Level), - Message: row.Message, - Fields: row.Fields, - }, - ) + return } - return &api.SandboxLogs{Logs: l, LogEntries: le}, nil + c.JSON(http.StatusOK, logs) } diff --git a/packages/api/internal/handlers/sandbox_metrics.go b/packages/api/internal/handlers/sandbox_metrics.go index 0c14cecf1b..16c0ffe6eb 100644 --- a/packages/api/internal/handlers/sandbox_metrics.go +++ b/packages/api/internal/handlers/sandbox_metrics.go @@ -1,10 +1,7 @@ package handlers import ( - "context" - "fmt" "net/http" - "time" "github.com/gin-gonic/gin" "github.com/launchdarkly/go-sdk-common/v3/ldcontext" @@ -12,11 +9,8 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" - "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/db/types" "github.com/e2b-dev/infra/packages/api/internal/utils" - clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" - clickhouseUtils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) @@ -41,7 +35,6 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(c *gin.Context, sandboxID string ) metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlagName) - if !metricsReadFlag { logger.L().Debug(ctx, "sandbox metrics read feature flag is disabled") // If we are not reading from ClickHouse, we can return an empty map @@ -52,114 +45,22 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(c *gin.Context, sandboxID string return } - // TODO: Remove in [ENG-3377], once edge is migrated - edgeProvidedMetrics := a.featureFlags.BoolFlag(ctx, featureflags.EdgeProvidedSandboxMetricsFlagName) - - var metrics []api.SandboxMetric - var apiErr *api.APIError - if edgeProvidedMetrics { - metrics, apiErr = clusters.GetClusterSandboxMetrics( - ctx, - a.clustersPool, - sandboxID, - team.ID.String(), - utils.WithClusterFallback(team.ClusterID), - params.Start, - params.End, - ) - } else { - metrics, apiErr = a.getApiProvidedMetrics(ctx, team, sandboxID, params) - } - if apiErr != nil { - logger.L().Error(ctx, "error getting sandbox metrics", zap.Error(apiErr.Err)) - a.sendAPIStoreError(c, apiErr.Code, apiErr.ClientMsg) + clusterID := utils.WithClusterFallback(team.ClusterID) + cluster, found := a.clustersPool.GetClusterById(clusterID) + if !found { + logger.L().Error(ctx, "cluster not found for sandbox metrics", logger.WithClusterID(clusterID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "cluster not found for sandbox metrics") return } - c.JSON(http.StatusOK, metrics) -} - -// TODO: Remove in [ENG-3377], once edge is migrated -func (a *APIStore) getApiProvidedMetrics(ctx context.Context, team *types.Team, sandboxID string, params api.GetSandboxesSandboxIDMetricsParams) ([]api.SandboxMetric, *api.APIError) { - start, end, err := getSandboxStartEndTime(ctx, a.clickhouseStore, team.ID.String(), sandboxID, params) - if err != nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: "error getting metrics time range", - Err: fmt.Errorf("error getting metrics time range: %w", err), - } - } - - start, end, err = clickhouseUtils.ValidateRange(start, end) + metrics, err := cluster.GetResources().GetSandboxMetrics(ctx, team.ID.String(), sandboxID, params.Start, params.End) if err != nil { - return nil, &api.APIError{ - Code: http.StatusBadRequest, - ClientMsg: fmt.Sprintf("error validating time range: %s", err), - Err: err, - } - } - - // Calculate the step size - step := clickhouseUtils.CalculateStep(start, end) - - metrics, err := a.clickhouseStore.QuerySandboxMetrics(ctx, sandboxID, team.ID.String(), start, end, step) - if err != nil { - return nil, &api.APIError{ - Code: http.StatusInternalServerError, - ClientMsg: "error querying sandbox metrics", - Err: fmt.Errorf("error querying sandbox metrics: %w", err), - } - } + logger.L().Error(ctx, "error getting sandbox metrics from edge", zap.Error(err)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "error getting sandbox metrics from edge") - apiMetrics := make([]api.SandboxMetric, len(metrics)) - for i, m := range metrics { - apiMetrics[i] = api.SandboxMetric{ - Timestamp: m.Timestamp, - TimestampUnix: m.Timestamp.Unix(), - CpuUsedPct: float32(m.CPUUsedPercent), - CpuCount: int32(m.CPUCount), - MemTotal: int64(m.MemTotal), - MemUsed: int64(m.MemUsed), - DiskTotal: int64(m.DiskTotal), - DiskUsed: int64(m.DiskUsed), - } - } - - return apiMetrics, nil -} - -func getSandboxStartEndTime(ctx context.Context, clickhouseStore clickhouse.Clickhouse, teamID, sandboxID string, params api.GetSandboxesSandboxIDMetricsParams) (time.Time, time.Time, error) { - // Check if the sandbox exists - var start, end time.Time - if params.Start != nil { - start = time.Unix(*params.Start, 0) - } - - if params.End != nil { - end = time.Unix(*params.End, 0) - } - - if start.IsZero() || end.IsZero() { - sbxStart, sbxEnd, err := clickhouseStore.QuerySandboxTimeRange(ctx, sandboxID, teamID) - if err != nil { - logger.L().Error(ctx, "Error fetching sandbox time range from ClickHouse", - logger.WithSandboxID(sandboxID), - logger.WithTeamID(teamID), - zap.Error(err), - ) - - return time.Time{}, time.Time{}, fmt.Errorf("error querying sandbox time range: %w", err) - } - - if start.IsZero() { - start = sbxStart - } - - if end.IsZero() { - end = sbxEnd - } + return } - return start, end, nil + c.JSON(http.StatusOK, metrics) } diff --git a/packages/api/internal/handlers/sandboxes_list_metrics.go b/packages/api/internal/handlers/sandboxes_list_metrics.go index 60ef89c8ea..4c2d5fb28c 100644 --- a/packages/api/internal/handlers/sandboxes_list_metrics.go +++ b/packages/api/internal/handlers/sandboxes_list_metrics.go @@ -13,7 +13,6 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" - "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/api/internal/db/types" "github.com/e2b-dev/infra/packages/api/internal/utils" featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" @@ -26,7 +25,7 @@ const maxSandboxMetricsCount = 100 func (a *APIStore) getSandboxesMetrics( ctx context.Context, teamID uuid.UUID, - clusterID *uuid.UUID, + clusterID uuid.UUID, sandboxIDs []string, ) (map[string]api.SandboxMetric, *api.APIError) { ctx, span := tracer.Start(ctx, "fetch-sandboxes-metrics") @@ -41,9 +40,8 @@ func (a *APIStore) getSandboxesMetrics( attribute.Int("sandboxes.count", len(sandboxIDs)), ) - metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlagName) - // Get metrics for all sandboxes + metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlagName) if !metricsReadFlag { logger.L().Debug(ctx, "sandbox metrics read feature flag is disabled") // If we are not reading from ClickHouse, we can return an empty map @@ -51,35 +49,19 @@ func (a *APIStore) getSandboxesMetrics( return make(map[string]api.SandboxMetric), nil } - // TODO: Remove in [ENG-3377], once edge is migrated - edgeProvidedMetrics := a.featureFlags.BoolFlag(ctx, featureflags.EdgeProvidedSandboxMetricsFlagName) - - var metrics map[string]api.SandboxMetric - var apiErr *api.APIError - if edgeProvidedMetrics { - metrics, apiErr = clusters.GetClusterSandboxListMetrics( - ctx, - a.clustersPool, - teamID.String(), - utils.WithClusterFallback(clusterID), - sandboxIDs, - ) - } else { - metrics, apiErr = a.getApiProvidedSandboxListMetrics(ctx, teamID.String(), sandboxIDs) - } - if apiErr != nil { - return nil, apiErr + cluster, found := a.clustersPool.GetClusterById(clusterID) + if !found { + return nil, &api.APIError{ + Code: http.StatusInternalServerError, + ClientMsg: "cluster not found for sandbox metrics", + Err: fmt.Errorf("cluster not found for sandbox metrics, cluster id: %s", clusterID.String()), + } } - return metrics, nil -} - -// TODO: Remove in [ENG-3377], once edge is migrated -func (a *APIStore) getApiProvidedSandboxListMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, *api.APIError) { - metrics, err := a.clickhouseStore.QueryLatestMetrics(ctx, sandboxIDs, teamID) + metrics, err := cluster.GetResources().GetSandboxesMetrics(ctx, teamID.String(), sandboxIDs) if err != nil { - logger.L().Error(ctx, "Error fetching sandbox metrics from ClickHouse", - logger.WithTeamID(teamID), + logger.L().Error(ctx, "error fetching sandbox metrics from cluster resources provider", + logger.WithTeamID(teamID.String()), zap.Error(err), ) @@ -90,21 +72,7 @@ func (a *APIStore) getApiProvidedSandboxListMetrics(ctx context.Context, teamID } } - apiMetrics := make(map[string]api.SandboxMetric) - for _, m := range metrics { - apiMetrics[m.SandboxID] = api.SandboxMetric{ - Timestamp: m.Timestamp, - TimestampUnix: m.Timestamp.Unix(), - CpuUsedPct: float32(m.CPUUsedPercent), - CpuCount: int32(m.CPUCount), - MemTotal: int64(m.MemTotal), - MemUsed: int64(m.MemUsed), - DiskTotal: int64(m.DiskTotal), - DiskUsed: int64(m.DiskUsed), - } - } - - return apiMetrics, nil + return metrics, nil } func (a *APIStore) GetSandboxesMetrics(c *gin.Context, params api.GetSandboxesMetricsParams) { @@ -133,7 +101,7 @@ func (a *APIStore) GetSandboxesMetrics(c *gin.Context, params api.GetSandboxesMe Build(), ) - sandboxesWithMetrics, apiErr := a.getSandboxesMetrics(ctx, team.ID, team.ClusterID, params.SandboxIds) + sandboxesWithMetrics, apiErr := a.getSandboxesMetrics(ctx, team.ID, utils.WithClusterFallback(team.ClusterID), params.SandboxIds) if apiErr != nil { logger.L().Error(ctx, "error getting sandbox metrics", zap.Error(apiErr.Err)) a.sendAPIStoreError(c, apiErr.Code, apiErr.ClientMsg) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 8d0dea9ce6..40bb37dc96 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -11,7 +11,6 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" - template_manager "github.com/e2b-dev/infra/packages/api/internal/template-manager" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/db/types" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -92,14 +91,18 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDLogs(c *gin.Context, templ cursor = sharedUtils.ToPtr(time.UnixMilli(*params.Cursor)) } - logs := template_manager.GetBuildLogs(ctx, cluster, buildInfo.NodeID, templateID, buildID, 0, limit, apiToLogLevel(params.Level), cursor, direction, params.Source) + logs, err := cluster.GetResources().GetBuildLogs(ctx, buildInfo.NodeID, templateID, buildID, 0, limit, apiToLogLevel(params.Level), cursor, direction, params.Source) + if err != nil { + telemetry.ReportError(ctx, "error when getting build logs", err, telemetry.WithTemplateID(templateID), telemetry.WithBuildID(buildID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when getting build logs") + + return + } - logEntries := make([]api.BuildLogEntry, 0) - for _, entry := range logs { - logEntries = append(logEntries, getAPILogEntry(entry)) + logEntries := make([]api.BuildLogEntry, len(logs)) + for i, entry := range logs { + logEntries[i] = getAPILogEntry(entry) } - c.JSON(http.StatusOK, api.TemplateBuildLogsResponse{ - Logs: logEntries, - }) + c.JSON(http.StatusOK, api.TemplateBuildLogsResponse{Logs: logEntries}) } diff --git a/packages/api/internal/handlers/template_build_status.go b/packages/api/internal/handlers/template_build_status.go index 64099f5782..2587b78588 100644 --- a/packages/api/internal/handlers/template_build_status.go +++ b/packages/api/internal/handlers/template_build_status.go @@ -11,7 +11,6 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" - template_manager "github.com/e2b-dev/infra/packages/api/internal/template-manager" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/db/types" "github.com/e2b-dev/infra/packages/shared/pkg/logs" @@ -120,7 +119,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem limit = *params.Limit } - logs := template_manager.GetBuildLogs(ctx, cluster, buildInfo.NodeID, templateID, buildID, offset, limit, apiToLogLevel(params.Level), nil, api.LogsDirectionForward, nil) + logs, err := cluster.GetResources().GetBuildLogs(ctx, buildInfo.NodeID, templateID, buildID, offset, limit, apiToLogLevel(params.Level), nil, api.LogsDirectionForward, nil) for _, entry := range logs { if legacyLogs { lgs = append(lgs, fmt.Sprintf("[%s] %s\n", entry.Timestamp.Format(time.RFC3339), entry.Message)) diff --git a/packages/api/internal/template-manager/build_client.go b/packages/api/internal/template-manager/build_client.go deleted file mode 100644 index eb59736cf8..0000000000 --- a/packages/api/internal/template-manager/build_client.go +++ /dev/null @@ -1,9 +0,0 @@ -package template_manager - -import ( - "github.com/e2b-dev/infra/packages/api/internal/clusters" -) - -type BuildClient struct { - GRPC *clusters.ClusterGRPC -} diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index 6d4375ed74..c9124bfe6a 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -87,7 +87,7 @@ func (tm *TemplateManager) CreateTemplate( return fmt.Errorf("failed to get features for firecracker version '%s': %w", firecrackerVersion, err) } - cli, err := tm.GetClusterBuildClient(clusterID, nodeID) + client, err := tm.GetClusterBuildClient(clusterID, nodeID) if err != nil { return fmt.Errorf("failed to get builder: %w", err) } @@ -149,9 +149,9 @@ func (tm *TemplateManager) CreateTemplate( return nil } - reqCtx := metadata.NewOutgoingContext(ctx, cli.GRPC.Metadata) - _, err = cli.GRPC.Client.Template.TemplateCreate( - reqCtx, &templatemanagergrpc.TemplateCreateRequest{ + _, err = client.Client.Template.TemplateCreate( + metadata.NewOutgoingContext(ctx, client.Metadata), + &templatemanagergrpc.TemplateCreateRequest{ Template: template, CacheScope: ut.ToPtr(teamID.String()), Version: &version, diff --git a/packages/api/internal/template-manager/logs.go b/packages/api/internal/template-manager/logs.go deleted file mode 100644 index 21df2f51f3..0000000000 --- a/packages/api/internal/template-manager/logs.go +++ /dev/null @@ -1,74 +0,0 @@ -package template_manager - -import ( - "context" - "fmt" - "time" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - "go.uber.org/zap" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/clusters" - buildlogs "github.com/e2b-dev/infra/packages/api/internal/template-manager/logs" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/logs" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" -) - -const ( - maxTimeRangeDuration = 7 * 24 * time.Hour -) - -func GetBuildLogs(ctx context.Context, cluster *clusters.Cluster, nodeID *string, templateID, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) []logs.LogEntry { - ctx, span := tracer.Start(ctx, "get build-logs") - defer span.End() - l := logger.L().With(logger.WithTemplateID(templateID), logger.WithBuildID(buildID)) - - logProviders := make([]buildlogs.Provider, 0) - - if nodeID != nil && isSourceType(source, api.LogsSourceTemporary) { - instance, err := cluster.GetTemplateBuilderByNodeID(*nodeID) - if err == nil { - grpc := cluster.GetGRPC(instance.ServiceInstanceID) - logProviders = append(logProviders, &buildlogs.TemplateManagerProvider{GRPC: grpc}) - } else { - l.Debug(ctx, "falling back to edge provider, node not found", zap.Error(err), logger.WithNodeID(*nodeID)) - } - } - - if isSourceType(source, api.LogsSourcePersistent) { - logProviders = append(logProviders, &buildlogs.EdgeProvider{HTTP: cluster.GetHTTP()}) - } - - start, end := time.Now().Add(-maxTimeRangeDuration), time.Now() - if cursor != nil { - if direction == api.LogsDirectionForward { - start = *cursor - end = start.Add(maxTimeRangeDuration) - } else { - end = *cursor - start = end.Add(-maxTimeRangeDuration) - } - } - - for _, provider := range logProviders { - l, err := provider.GetLogs(ctx, templateID, buildID, offset, limit, level, start, end, direction) - if err != nil { - telemetry.ReportError(ctx, "soft error when getting logs for template build", err, telemetry.WithTemplateID(templateID), telemetry.WithBuildID(buildID), attribute.String("provider", fmt.Sprintf("%T", provider))) - - continue - } - - span.SetStatus(codes.Ok, "logs fetched for template build") - // Return the first non-error logs, the providers are ordered by most up-to-date data - return l - } - - return nil -} - -func isSourceType(source *api.LogsSource, sourceType api.LogsSource) bool { - return source == nil || *source == sourceType -} diff --git a/packages/api/internal/template-manager/logs/edge_provider.go b/packages/api/internal/template-manager/logs/edge_provider.go deleted file mode 100644 index 6bb5bc5d21..0000000000 --- a/packages/api/internal/template-manager/logs/edge_provider.go +++ /dev/null @@ -1,68 +0,0 @@ -package logs - -import ( - "context" - "errors" - "fmt" - "time" - - "go.uber.org/zap" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/clusters" - edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/logs" - "github.com/e2b-dev/infra/packages/shared/pkg/utils" -) - -type EdgeProvider struct { - HTTP *clusters.ClusterHTTP -} - -func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { - if level == nil { - return nil - } - - value := edgeapi.LogLevel(logs.LevelToString(*level)) - - return &value -} - -func (c *EdgeProvider) GetLogs(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { - res, err := c.HTTP.Client.V1TemplateBuildLogsWithResponse( - ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{ - TemplateID: templateID, - Offset: &offset, - Limit: &limit, - Level: logToEdgeLevel(level), - // TODO: remove this once the API spec is not required to have orchestratorID (https://linear.app/e2b/issue/ENG-3352) - OrchestratorID: utils.ToPtr("unused"), - Start: utils.ToPtr(start.UnixMilli()), - End: utils.ToPtr(end.UnixMilli()), - Direction: utils.ToPtr(edgeapi.V1TemplateBuildLogsParamsDirection(direction)), - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to get build logs in template manager: %w", err) - } - - if res.StatusCode() != 200 || res.JSON200 == nil { - logger.L().Error(ctx, "failed to get build logs in template manager", zap.String("body", string(res.Body))) - - return nil, errors.New("failed to get build logs in template manager") - } - - l := make([]logs.LogEntry, 0) - for _, entry := range res.JSON200.LogEntries { - l = append(l, logs.LogEntry{ - Timestamp: entry.Timestamp, - Message: entry.Message, - Level: logs.StringToLevel(string(entry.Level)), - Fields: entry.Fields, - }) - } - - return l, nil -} diff --git a/packages/api/internal/template-manager/logs/provider.go b/packages/api/internal/template-manager/logs/provider.go deleted file mode 100644 index bfa8c79bb8..0000000000 --- a/packages/api/internal/template-manager/logs/provider.go +++ /dev/null @@ -1,13 +0,0 @@ -package logs - -import ( - "context" - "time" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/shared/pkg/logs" -) - -type Provider interface { - GetLogs(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) -} diff --git a/packages/api/internal/template-manager/logs/template_manager_provider.go b/packages/api/internal/template-manager/logs/template_manager_provider.go deleted file mode 100644 index 6d2ecc2e7b..0000000000 --- a/packages/api/internal/template-manager/logs/template_manager_provider.go +++ /dev/null @@ -1,73 +0,0 @@ -package logs - -import ( - "context" - "time" - - "go.uber.org/zap" - "google.golang.org/grpc/metadata" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/clusters" - templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/logs" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "github.com/e2b-dev/infra/packages/shared/pkg/utils" -) - -type TemplateManagerProvider struct { - GRPC *clusters.ClusterGRPC -} - -func (t *TemplateManagerProvider) GetLogs(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { - reqCtx := metadata.NewOutgoingContext(ctx, t.GRPC.Metadata) - - var lvlReq *templatemanagergrpc.LogLevel - if level != nil { - lvlReq = templatemanagergrpc.LogLevel(*level).Enum() - } - res, err := t.GRPC.Client.Template.TemplateBuildStatus( - reqCtx, &templatemanagergrpc.TemplateStatusRequest{ - TemplateID: templateID, - BuildID: buildID, - Offset: &offset, - Limit: utils.ToPtr(uint32(limit)), - Level: lvlReq, - Start: timestamppb.New(start), - End: timestamppb.New(end), - Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), - }, - ) - if err != nil { - telemetry.ReportError(ctx, "error when returning logs for template build", err) - logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) - - return nil, err - } - - l := make([]logs.LogEntry, 0) - // Add an extra newline to each log entry to ensure proper formatting in the CLI - for _, entry := range res.GetLogEntries() { - l = append(l, logs.LogEntry{ - Timestamp: entry.GetTimestamp().AsTime(), - Message: entry.GetMessage(), - Level: logs.LogLevel(entry.GetLevel()), - Fields: entry.GetFields(), - }) - } - - return l, nil -} - -func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection { - switch direction { - case api.LogsDirectionForward: - return templatemanagergrpc.LogsDirection_Forward - case api.LogsDirectionBackward: - return templatemanagergrpc.LogsDirection_Backward - default: - return templatemanagergrpc.LogsDirection_Forward - } -} diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index bfe8684416..7f2f6b29e1 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -33,7 +33,7 @@ type processingBuilds struct { } type TemplateManager struct { - edgePool *clusters.Pool + clusters *clusters.Pool lock sync.Mutex processing map[uuid.UUID]processingBuilds @@ -67,7 +67,7 @@ func New( sqlcDB: sqlcDB, buildCache: buildCache, templateCache: templateCache, - edgePool: edgePool, + clusters: edgePool, featureFlags: featureFlags, lock: sync.Mutex{}, @@ -109,7 +109,7 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { } func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterID uuid.UUID) (*clusters.ClusterInstance, error) { - cluster, ok := tm.edgePool.GetClusterById(clusterID) + cluster, ok := tm.clusters.GetClusterById(clusterID) if !ok { return nil, fmt.Errorf("cluster with ID '%s' not found", clusterID) } @@ -122,8 +122,17 @@ func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterI return builder, nil } -func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID string) (*BuildClient, error) { - cluster, ok := tm.edgePool.GetClusterById(clusterID) +func (tm *TemplateManager) GetClusterResources(clusterID uuid.UUID) (clusters.ClusterResource, error) { + cluster, ok := tm.clusters.GetClusterById(clusterID) + if !ok { + return nil, errors.New("cluster not found") + } + + return cluster.GetResources(), nil +} + +func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID string) (*clusters.ClusterGRPC, error) { + cluster, ok := tm.clusters.GetClusterById(clusterID) if !ok { return nil, errors.New("cluster not found") } @@ -133,11 +142,7 @@ func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID str return nil, fmt.Errorf("failed to get builder by id '%s': %w", nodeID, err) } - grpc := cluster.GetGRPC(instance.ServiceInstanceID) - - return &BuildClient{ - GRPC: grpc, - }, nil + return cluster.GetGRPC(instance.ServiceInstanceID), nil } func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, templateID string, clusterID uuid.UUID, nodeID string) error { @@ -166,9 +171,9 @@ func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, t } } - reqCtx := metadata.NewOutgoingContext(ctx, client.GRPC.Metadata) - _, err = client.GRPC.Client.Template.TemplateBuildDelete( - reqCtx, &templatemanagergrpc.TemplateBuildDeleteRequest{ + _, err = client.Client.Template.TemplateBuildDelete( + metadata.NewOutgoingContext(ctx, client.Metadata), + &templatemanagergrpc.TemplateBuildDeleteRequest{ BuildID: buildID.String(), TemplateID: templateID, }, @@ -194,16 +199,14 @@ func (tm *TemplateManager) DeleteBuilds(ctx context.Context, builds []DeleteBuil } func (tm *TemplateManager) GetStatus(ctx context.Context, buildID uuid.UUID, templateID string, clusterID uuid.UUID, nodeID string) (*templatemanagergrpc.TemplateBuildStatusResponse, error) { - cli, err := tm.GetClusterBuildClient(clusterID, nodeID) + client, err := tm.GetClusterBuildClient(clusterID, nodeID) if err != nil { - return nil, fmt.Errorf("failed to get builder edgeHttpClient: %w", err) + return nil, fmt.Errorf("failed to get builder client: %w", err) } - reqCtx := metadata.NewOutgoingContext(ctx, cli.GRPC.Metadata) - // error unwrapping is done in the caller - return cli.GRPC.Client.Template.TemplateBuildStatus( - reqCtx, + return client.Client.Template.TemplateBuildStatus( + metadata.NewOutgoingContext(ctx, client.Metadata), &templatemanagergrpc.TemplateStatusRequest{ BuildID: buildID.String(), TemplateID: templateID, }, diff --git a/packages/api/internal/template-manager/upload_template_layer_files.go b/packages/api/internal/template-manager/upload_template_layer_files.go index ecfa59f93d..41cab88265 100644 --- a/packages/api/internal/template-manager/upload_template_layer_files.go +++ b/packages/api/internal/template-manager/upload_template_layer_files.go @@ -8,19 +8,19 @@ import ( "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/utils" - template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + templatemanager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" ut "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) -func (tm *TemplateManager) InitLayerFileUpload(ctx context.Context, clusterID uuid.UUID, nodeID string, teamID uuid.UUID, templateID string, hash string) (*template_manager.InitLayerFileUploadResponse, error) { +func (tm *TemplateManager) InitLayerFileUpload(ctx context.Context, clusterID uuid.UUID, nodeID string, teamID uuid.UUID, templateID string, hash string) (*templatemanager.InitLayerFileUploadResponse, error) { client, err := tm.GetClusterBuildClient(clusterID, nodeID) if err != nil { return nil, fmt.Errorf("failed to get build client for template '%s': %w", templateID, err) } - reqCtx := metadata.NewOutgoingContext(ctx, client.GRPC.Metadata) - resp, err := client.GRPC.Client.Template.InitLayerFileUpload( - reqCtx, &template_manager.InitLayerFileUploadRequest{ + reqCtx := metadata.NewOutgoingContext(ctx, client.Metadata) + resp, err := client.Client.Template.InitLayerFileUpload( + reqCtx, &templatemanager.InitLayerFileUploadRequest{ CacheScope: ut.ToPtr(teamID.String()), TemplateID: templateID, Hash: hash, diff --git a/packages/shared/pkg/feature-flags/flags.go b/packages/shared/pkg/feature-flags/flags.go index 5ea1c2f368..c37e4486f4 100644 --- a/packages/shared/pkg/feature-flags/flags.go +++ b/packages/shared/pkg/feature-flags/flags.go @@ -70,13 +70,12 @@ func newBoolFlag(name string, fallback bool) BoolFlag { } var ( - MetricsWriteFlagName = newBoolFlag("sandbox-metrics-write", env.IsDevelopment()) - MetricsReadFlagName = newBoolFlag("sandbox-metrics-read", env.IsDevelopment()) - SnapshotFeatureFlagName = newBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) - TemplateFeatureFlagName = newBoolFlag("use-nfs-for-templates", env.IsDevelopment()) - BestOfKCanFit = newBoolFlag("best-of-k-can-fit", true) - BestOfKTooManyStarting = newBoolFlag("best-of-k-too-many-starting", false) - EdgeProvidedSandboxMetricsFlagName = newBoolFlag("edge-provided-sandbox-metrics", false) + MetricsWriteFlagName = newBoolFlag("sandbox-metrics-write", env.IsDevelopment()) + MetricsReadFlagName = newBoolFlag("sandbox-metrics-read", env.IsDevelopment()) + SnapshotFeatureFlagName = newBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) + TemplateFeatureFlagName = newBoolFlag("use-nfs-for-templates", env.IsDevelopment()) + BestOfKCanFit = newBoolFlag("best-of-k-can-fit", true) + BestOfKTooManyStarting = newBoolFlag("best-of-k-too-many-starting", false) ) type IntFlag struct { From beeb0a43564c7bdd49517ab86f4499c6f87a805b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 13:44:00 +0100 Subject: [PATCH 11/58] Renamed cluster.ClusterInstance to cluster.Instance --- packages/api/internal/clusters/cluster.go | 20 +++++----- .../internal/clusters/cluster_instances.go | 40 +++++++++---------- .../api/internal/clusters/resources_impl.go | 4 +- packages/api/internal/orchestrator/client.go | 4 +- .../internal/orchestrator/nodemanager/node.go | 4 +- .../template-manager/template_manager.go | 4 +- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index b660d866d8..de57a35fb2 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -28,8 +28,8 @@ type Cluster struct { httpClient *api.ClientWithResponses grpcClient *grpclient.GRPCClient - instances *smap.Map[*ClusterInstance] - synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *ClusterInstance] + instances *smap.Map[*Instance] + synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *Instance] resources ClusterResource } @@ -80,7 +80,7 @@ func NewCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end return nil, fmt.Errorf("failed to create grpc client: %w", err) } - instances := smap.New[*ClusterInstance]() + instances := smap.New[*Instance]() c := &Cluster{ ID: clusterID, @@ -108,7 +108,7 @@ func (c *Cluster) Close() error { return err } -func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*ClusterInstance, error) { +func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { instance, found := c.instances.Get(nodeID) if !found { return nil, ErrTemplateBuilderNotFound @@ -121,9 +121,9 @@ func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*ClusterInstance, e return instance, nil } -func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*ClusterInstance, bool) { +func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*Instance, bool) { for _, instance := range c.instances.Items() { - if instance.ServiceInstanceID == serviceInstanceID { + if instance.InstanceID == serviceInstanceID { return instance, true } } @@ -131,12 +131,12 @@ func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*ClusterInst return nil, false } -func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*ClusterInstance, error) { +func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, error) { _, span := tracer.Start(ctx, "template-builder-get-available-instance") span.SetAttributes(telemetry.WithClusterID(c.ID)) defer span.End() - var instances []*ClusterInstance + var instances []*Instance for _, instance := range c.instances.Items() { instances = append(instances, instance) } @@ -169,8 +169,8 @@ func (c *Cluster) GetHTTP() *ClusterHTTP { return &ClusterHTTP{c.httpClient} } -func (c *Cluster) GetOrchestrators() []*ClusterInstance { - instances := make([]*ClusterInstance, 0) +func (c *Cluster) GetOrchestrators() []*Instance { + instances := make([]*Instance, 0) for _, i := range c.instances.Items() { if i.IsOrchestrator() { instances = append(instances, i) diff --git a/packages/api/internal/clusters/cluster_instances.go b/packages/api/internal/clusters/cluster_instances.go index ae7da148f3..11757cda8c 100644 --- a/packages/api/internal/clusters/cluster_instances.go +++ b/packages/api/internal/clusters/cluster_instances.go @@ -20,10 +20,10 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" ) -type ClusterInstance struct { - NodeID string +type Instance struct { + NodeID string + InstanceID string - ServiceInstanceID string ServiceVersion string ServiceVersionCommit string @@ -43,8 +43,8 @@ func (c *Cluster) startSync(ctx context.Context) { c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) } -func (c *Cluster) syncInstance(ctx context.Context, instance *ClusterInstance) { - grpc := c.GetGRPC(instance.ServiceInstanceID) +func (c *Cluster) syncInstance(ctx context.Context, instance *Instance) { + grpc := c.GetGRPC(instance.InstanceID) // we are taking service info directly from the instance to avoid timing delays in service discovery reqCtx := metadata.NewOutgoingContext(ctx, grpc.Metadata) @@ -56,7 +56,7 @@ func (c *Cluster) syncInstance(ctx context.Context, instance *ClusterInstance) { zap.Error(err), logger.WithClusterID(c.ID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.ServiceInstanceID), + logger.WithServiceInstanceID(instance.InstanceID), ) return @@ -70,32 +70,32 @@ func (c *Cluster) syncInstance(ctx context.Context, instance *ClusterInstance) { instance.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) } -func (n *ClusterInstance) GetStatus() infogrpc.ServiceInfoStatus { +func (n *Instance) GetStatus() infogrpc.ServiceInfoStatus { n.mutex.RLock() defer n.mutex.RUnlock() return n.status } -func (n *ClusterInstance) GetMachineInfo() machineinfo.MachineInfo { +func (n *Instance) GetMachineInfo() machineinfo.MachineInfo { n.mutex.RLock() defer n.mutex.RUnlock() return n.machineInfo } -func (n *ClusterInstance) hasRole(r infogrpc.ServiceInfoRole) bool { +func (n *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { n.mutex.RLock() defer n.mutex.RUnlock() return slices.Contains(n.roles, r) } -func (n *ClusterInstance) IsBuilder() bool { +func (n *Instance) IsBuilder() bool { return n.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) } -func (n *ClusterInstance) IsOrchestrator() bool { +func (n *Instance) IsOrchestrator() bool { return n.hasRole(infogrpc.ServiceInfoRole_Orchestrator) } @@ -122,11 +122,11 @@ func (d clusterSynchronizationStore) SourceList(ctx context.Context) ([]api.Clus return *res.JSON200, nil } -func (d clusterSynchronizationStore) SourceExists(_ context.Context, s []api.ClusterOrchestratorNode, p *ClusterInstance) bool { +func (d clusterSynchronizationStore) SourceExists(_ context.Context, s []api.ClusterOrchestratorNode, p *Instance) bool { for _, item := range s { // With comparing service instance ID we ensure when orchestrator on same node and node ID is still same // we will properly clean up old instance and later register as new one - if item.ServiceInstanceID == p.ServiceInstanceID { + if item.ServiceInstanceID == p.InstanceID { return true } } @@ -134,8 +134,8 @@ func (d clusterSynchronizationStore) SourceExists(_ context.Context, s []api.Clu return false } -func (d clusterSynchronizationStore) PoolList(_ context.Context) []*ClusterInstance { - mapped := make([]*ClusterInstance, 0) +func (d clusterSynchronizationStore) PoolList(_ context.Context) []*Instance { + mapped := make([]*Instance, 0) for _, item := range d.cluster.instances.Items() { mapped = append(mapped, item) } @@ -156,10 +156,10 @@ func (d clusterSynchronizationStore) PoolInsert(ctx context.Context, item api.Cl logger.WithServiceInstanceID(item.ServiceInstanceID), ) - instance := &ClusterInstance{ + instance := &Instance{ NodeID: item.NodeID, - ServiceInstanceID: item.ServiceInstanceID, + InstanceID: item.ServiceInstanceID, ServiceVersion: item.ServiceVersion, ServiceVersionCommit: item.ServiceVersionCommit, @@ -174,15 +174,15 @@ func (d clusterSynchronizationStore) PoolInsert(ctx context.Context, item api.Cl d.cluster.instances.Insert(item.NodeID, instance) } -func (d clusterSynchronizationStore) PoolUpdate(ctx context.Context, instance *ClusterInstance) { +func (d clusterSynchronizationStore) PoolUpdate(ctx context.Context, instance *Instance) { d.cluster.syncInstance(ctx, instance) } -func (d clusterSynchronizationStore) PoolRemove(ctx context.Context, instance *ClusterInstance) { +func (d clusterSynchronizationStore) PoolRemove(ctx context.Context, instance *Instance) { logger.L().Info(ctx, "Removing instance from cluster pool", logger.WithClusterID(d.cluster.ID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.ServiceInstanceID), + logger.WithServiceInstanceID(instance.InstanceID), ) d.cluster.instances.Remove(instance.NodeID) } diff --git a/packages/api/internal/clusters/resources_impl.go b/packages/api/internal/clusters/resources_impl.go index 349af93469..b5370702eb 100644 --- a/packages/api/internal/clusters/resources_impl.go +++ b/packages/api/internal/clusters/resources_impl.go @@ -18,11 +18,11 @@ import ( ) type ClusterResourceProviderImpl struct { - instances *smap.Map[*ClusterInstance] + instances *smap.Map[*Instance] client *edgeapi.ClientWithResponses } -func newRemoteClusterResourceProvider(instances *smap.Map[*ClusterInstance], client *edgeapi.ClientWithResponses) ClusterResource { +func newRemoteClusterResourceProvider(instances *smap.Map[*Instance], client *edgeapi.ClientWithResponses) ClusterResource { return &ClusterResourceProviderImpl{ instances: instances, client: client, diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index 10d77aeba9..73c0db1250 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -33,9 +33,9 @@ func (o *Orchestrator) connectToNode(ctx context.Context, discovered nodemanager return nil } -func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.ClusterInstance) { +func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.Instance) { // this way we don't need to worry about multiple clusters with the same node ID in shared pool - clusterGRPC := cluster.GetGRPC(i.ServiceInstanceID) + clusterGRPC := cluster.GetGRPC(i.InstanceID) orchestratorNode, err := nodemanager.NewClusterNode(ctx, clusterGRPC.Client, cluster.ID, cluster.SandboxDomain, i) if err != nil { diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index 88b8bf3083..3c6d43bd52 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -114,7 +114,7 @@ func New( return n, nil } -func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.ClusterInstance) (*Node, error) { +func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.Instance) (*Node, error) { nodeStatus, ok := OrchestratorToApiNodeStateMapper[i.GetStatus()] if !ok { logger.L().Error(ctx, "Unknown service info status", zap.String("status", i.GetStatus().String()), logger.WithNodeID(i.NodeID)) @@ -125,7 +125,7 @@ func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID go buildCache.Start() nodeMetadata := NodeMetadata{ - ServiceInstanceID: i.ServiceInstanceID, + ServiceInstanceID: i.InstanceID, Commit: i.ServiceVersionCommit, Version: i.ServiceVersion, } diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 7f2f6b29e1..30d2ace1a4 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -108,7 +108,7 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { } } -func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterID uuid.UUID) (*clusters.ClusterInstance, error) { +func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterID uuid.UUID) (*clusters.Instance, error) { cluster, ok := tm.clusters.GetClusterById(clusterID) if !ok { return nil, fmt.Errorf("cluster with ID '%s' not found", clusterID) @@ -142,7 +142,7 @@ func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID str return nil, fmt.Errorf("failed to get builder by id '%s': %w", nodeID, err) } - return cluster.GetGRPC(instance.ServiceInstanceID), nil + return cluster.GetGRPC(instance.InstanceID), nil } func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, templateID string, clusterID uuid.UUID, nodeID string) error { From f6222f9cdec132a3c02ea09b364f1e3c7d85df92 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 13:45:20 +0100 Subject: [PATCH 12/58] Cluster http client is no longer need to be exposed form cluster --- packages/api/internal/clusters/cluster.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index de57a35fb2..08b9211efc 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -38,10 +38,6 @@ type ClusterGRPC struct { Metadata metadata.MD } -type ClusterHTTP struct { - Client *api.ClientWithResponses -} - var ( ErrTemplateBuilderNotFound = errors.New("template builder not found") ErrAvailableTemplateBuilderNotFound = errors.New("available template builder not found") @@ -165,10 +161,6 @@ func (c *Cluster) GetGRPC(serviceInstanceID string) *ClusterGRPC { return &ClusterGRPC{c.grpcClient, metadata.New(map[string]string{consts.EdgeRpcServiceInstanceIDHeader: serviceInstanceID})} } -func (c *Cluster) GetHTTP() *ClusterHTTP { - return &ClusterHTTP{c.httpClient} -} - func (c *Cluster) GetOrchestrators() []*Instance { instances := make([]*Instance, 0) for _, i := range c.instances.Items() { @@ -180,10 +172,6 @@ func (c *Cluster) GetOrchestrators() []*Instance { return instances } -func (c *Cluster) GetHttpClient() *api.ClientWithResponses { - return c.httpClient -} - func (c *Cluster) GetResources() ClusterResource { return c.resources } From c205b18edebdbe58706b31ac9f03ed7141243648 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 13:51:21 +0100 Subject: [PATCH 13/58] Rename and rearrange clusters files Separated cluster, cluster sync, instance and instance sync logic to separated files to make it more easy to read and navigate. --- packages/api/internal/clusters/cluster.go | 44 +++++++++- .../clusters/{pool.go => clusters_sync.go} | 2 +- packages/api/internal/clusters/instance.go | 52 +++++++++++ .../{client.go => instance_client.go} | 0 ...cluster_instances.go => instances_sync.go} | 87 ------------------- 5 files changed, 96 insertions(+), 89 deletions(-) rename packages/api/internal/clusters/{pool.go => clusters_sync.go} (99%) create mode 100644 packages/api/internal/clusters/instance.go rename packages/api/internal/clusters/{client.go => instance_client.go} (100%) rename packages/api/internal/clusters/{cluster_instances.go => instances_sync.go} (58%) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 08b9211efc..3dac431a7d 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -6,15 +6,21 @@ import ( "fmt" "math/rand/v2" "net/http" + "time" "github.com/google/uuid" "go.opentelemetry.io/otel" + "go.uber.org/zap" "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/emptypb" grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" + "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -22,6 +28,11 @@ import ( var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/clusters") +const ( + instancesSyncInterval = 5 * time.Second + instancesSyncTimeout = 5 * time.Second +) + type Cluster struct { ID uuid.UUID SandboxDomain *string @@ -43,7 +54,7 @@ var ( ErrAvailableTemplateBuilderNotFound = errors.New("available template builder not found") ) -func NewCluster(ctx context.Context, tel *telemetry.Client, endpoint string, endpointTLS bool, secret string, clusterID uuid.UUID, sandboxDomain *string) (*Cluster, error) { +func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, endpointTLS bool, secret string, clusterID uuid.UUID, sandboxDomain *string) (*Cluster, error) { clientAuthMiddleware := func(c *api.Client) error { c.RequestEditors = append( c.RequestEditors, @@ -175,3 +186,34 @@ func (c *Cluster) GetOrchestrators() []*Instance { func (c *Cluster) GetResources() ClusterResource { return c.resources } + +func (c *Cluster) startSync(ctx context.Context) { + c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) +} + +func (c *Cluster) syncInstance(ctx context.Context, instance *Instance) { + grpc := c.GetGRPC(instance.InstanceID) + + // we are taking service info directly from the instance to avoid timing delays in service discovery + reqCtx := metadata.NewOutgoingContext(ctx, grpc.Metadata) + info, err := grpc.Client.Info.ServiceInfo(reqCtx, &emptypb.Empty{}) + + err = utils.UnwrapGRPCError(err) + if err != nil { + logger.L().Error(ctx, "Failed to get instance info", + zap.Error(err), + logger.WithClusterID(c.ID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) + + return + } + + instance.mutex.Lock() + defer instance.mutex.Unlock() + + instance.status = info.GetServiceStatus() + instance.roles = info.GetServiceRoles() + instance.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) +} diff --git a/packages/api/internal/clusters/pool.go b/packages/api/internal/clusters/clusters_sync.go similarity index 99% rename from packages/api/internal/clusters/pool.go rename to packages/api/internal/clusters/clusters_sync.go index 53ed7a7475..55136a97df 100644 --- a/packages/api/internal/clusters/pool.go +++ b/packages/api/internal/clusters/clusters_sync.go @@ -157,7 +157,7 @@ func (d poolSynchronizationStore) PoolInsert(ctx context.Context, cluster querie logger.L().Info(ctx, "Initializing newly discovered cluster", logger.WithClusterID(cluster.ID)) - c, err := NewCluster(context.WithoutCancel(ctx), + c, err := newCluster(context.WithoutCancel(ctx), d.pool.tel, cluster.Endpoint, cluster.EndpointTls, diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go new file mode 100644 index 0000000000..f7eef1e69e --- /dev/null +++ b/packages/api/internal/clusters/instance.go @@ -0,0 +1,52 @@ +package clusters + +import ( + "slices" + "sync" + + infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" + "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" +) + +type Instance struct { + NodeID string + InstanceID string + + ServiceVersion string + ServiceVersionCommit string + + roles []infogrpc.ServiceInfoRole + machineInfo machineinfo.MachineInfo + + status infogrpc.ServiceInfoStatus + mutex sync.RWMutex +} + +func (n *Instance) GetStatus() infogrpc.ServiceInfoStatus { + n.mutex.RLock() + defer n.mutex.RUnlock() + + return n.status +} + +func (n *Instance) GetMachineInfo() machineinfo.MachineInfo { + n.mutex.RLock() + defer n.mutex.RUnlock() + + return n.machineInfo +} + +func (n *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { + n.mutex.RLock() + defer n.mutex.RUnlock() + + return slices.Contains(n.roles, r) +} + +func (n *Instance) IsBuilder() bool { + return n.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) +} + +func (n *Instance) IsOrchestrator() bool { + return n.hasRole(infogrpc.ServiceInfoRole_Orchestrator) +} diff --git a/packages/api/internal/clusters/client.go b/packages/api/internal/clusters/instance_client.go similarity index 100% rename from packages/api/internal/clusters/client.go rename to packages/api/internal/clusters/instance_client.go diff --git a/packages/api/internal/clusters/cluster_instances.go b/packages/api/internal/clusters/instances_sync.go similarity index 58% rename from packages/api/internal/clusters/cluster_instances.go rename to packages/api/internal/clusters/instances_sync.go index 11757cda8c..0c20040f63 100644 --- a/packages/api/internal/clusters/cluster_instances.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -5,100 +5,13 @@ import ( "errors" "fmt" "net/http" - "slices" "sync" - "time" - "go.uber.org/zap" - "google.golang.org/grpc/metadata" - "google.golang.org/protobuf/types/known/emptypb" - - "github.com/e2b-dev/infra/packages/api/internal/utils" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" -) - -type Instance struct { - NodeID string - InstanceID string - - ServiceVersion string - ServiceVersionCommit string - - roles []infogrpc.ServiceInfoRole - machineInfo machineinfo.MachineInfo - - status infogrpc.ServiceInfoStatus - mutex sync.RWMutex -} - -const ( - instancesSyncInterval = 5 * time.Second - instancesSyncTimeout = 5 * time.Second ) -func (c *Cluster) startSync(ctx context.Context) { - c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) -} - -func (c *Cluster) syncInstance(ctx context.Context, instance *Instance) { - grpc := c.GetGRPC(instance.InstanceID) - - // we are taking service info directly from the instance to avoid timing delays in service discovery - reqCtx := metadata.NewOutgoingContext(ctx, grpc.Metadata) - info, err := grpc.Client.Info.ServiceInfo(reqCtx, &emptypb.Empty{}) - - err = utils.UnwrapGRPCError(err) - if err != nil { - logger.L().Error(ctx, "Failed to get instance info", - zap.Error(err), - logger.WithClusterID(c.ID), - logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), - ) - - return - } - - instance.mutex.Lock() - defer instance.mutex.Unlock() - - instance.status = info.GetServiceStatus() - instance.roles = info.GetServiceRoles() - instance.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) -} - -func (n *Instance) GetStatus() infogrpc.ServiceInfoStatus { - n.mutex.RLock() - defer n.mutex.RUnlock() - - return n.status -} - -func (n *Instance) GetMachineInfo() machineinfo.MachineInfo { - n.mutex.RLock() - defer n.mutex.RUnlock() - - return n.machineInfo -} - -func (n *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { - n.mutex.RLock() - defer n.mutex.RUnlock() - - return slices.Contains(n.roles, r) -} - -func (n *Instance) IsBuilder() bool { - return n.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) -} - -func (n *Instance) IsOrchestrator() bool { - return n.hasRole(infogrpc.ServiceInfoRole_Orchestrator) -} - // SynchronizationStore defines methods for synchronizing cluster instances type clusterSynchronizationStore struct { cluster *Cluster From 32cdfee02928d6126c1886b3148948045e5d9cd9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 14:06:38 +0100 Subject: [PATCH 14/58] Renamed `clusterSynchronizationStore` to `instancesSyncStore` --- packages/api/internal/clusters/cluster.go | 2 +- packages/api/internal/clusters/instances_sync.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 3dac431a7d..087fb30961 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -99,7 +99,7 @@ func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end grpcClient: grpcClient, } - store := clusterSynchronizationStore{cluster: c} + store := instancesSyncStore{cluster: c} c.synchronization = synchronization.NewSynchronize("cluster-instances", "Cluster instances", store) // periodically sync cluster instances diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 0c20040f63..4872e68993 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -13,11 +13,11 @@ import ( ) // SynchronizationStore defines methods for synchronizing cluster instances -type clusterSynchronizationStore struct { +type instancesSyncStore struct { cluster *Cluster } -func (d clusterSynchronizationStore) SourceList(ctx context.Context) ([]api.ClusterOrchestratorNode, error) { +func (d instancesSyncStore) SourceList(ctx context.Context) ([]api.ClusterOrchestratorNode, error) { // fetch cluster instances with use of service discovery res, err := d.cluster.httpClient.V1ServiceDiscoveryGetOrchestratorsWithResponse(ctx) if err != nil { @@ -35,7 +35,7 @@ func (d clusterSynchronizationStore) SourceList(ctx context.Context) ([]api.Clus return *res.JSON200, nil } -func (d clusterSynchronizationStore) SourceExists(_ context.Context, s []api.ClusterOrchestratorNode, p *Instance) bool { +func (d instancesSyncStore) SourceExists(_ context.Context, s []api.ClusterOrchestratorNode, p *Instance) bool { for _, item := range s { // With comparing service instance ID we ensure when orchestrator on same node and node ID is still same // we will properly clean up old instance and later register as new one @@ -47,7 +47,7 @@ func (d clusterSynchronizationStore) SourceExists(_ context.Context, s []api.Clu return false } -func (d clusterSynchronizationStore) PoolList(_ context.Context) []*Instance { +func (d instancesSyncStore) PoolList(_ context.Context) []*Instance { mapped := make([]*Instance, 0) for _, item := range d.cluster.instances.Items() { mapped = append(mapped, item) @@ -56,13 +56,13 @@ func (d clusterSynchronizationStore) PoolList(_ context.Context) []*Instance { return mapped } -func (d clusterSynchronizationStore) PoolExists(_ context.Context, s api.ClusterOrchestratorNode) bool { +func (d instancesSyncStore) PoolExists(_ context.Context, s api.ClusterOrchestratorNode) bool { _, found := d.cluster.instances.Get(s.NodeID) return found } -func (d clusterSynchronizationStore) PoolInsert(ctx context.Context, item api.ClusterOrchestratorNode) { +func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrchestratorNode) { logger.L().Info(ctx, "Adding instance into cluster pool", logger.WithClusterID(d.cluster.ID), logger.WithNodeID(item.NodeID), @@ -87,11 +87,11 @@ func (d clusterSynchronizationStore) PoolInsert(ctx context.Context, item api.Cl d.cluster.instances.Insert(item.NodeID, instance) } -func (d clusterSynchronizationStore) PoolUpdate(ctx context.Context, instance *Instance) { +func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) { d.cluster.syncInstance(ctx, instance) } -func (d clusterSynchronizationStore) PoolRemove(ctx context.Context, instance *Instance) { +func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) { logger.L().Info(ctx, "Removing instance from cluster pool", logger.WithClusterID(d.cluster.ID), logger.WithNodeID(instance.NodeID), From 819b4a0937c562da50b5ffe66ac2b246620ff1f5 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 14:50:23 +0100 Subject: [PATCH 15/58] Each instance now holds its own gRPC connection Previously we used shared gRPC connection that was re-used for cluster instances. This introduces issue where for each called we need to expose not just gRPC connection but also ctx with additional gRPC metadata or metadata itself so for each request proper service instance id metadata key will be added so gRPC proxy running in remote cluster will router request properly. Now as each instance holds its own gRPC connection we no longer need to do this. To minimize refactoring changes some methods that were returning context with metadata re now returning context received without additional changes. There is small changes for sandbox create/delete metadata event. Previously we merged metadata and added them, but now metadata (auth and service instance id) are stored in gRPC auth middleware and they will be duplicated. This is why we are now doing only append of new one (sandbox delete/create event). --- packages/api/internal/clusters/cluster.go | 59 ++------ packages/api/internal/clusters/instance.go | 127 +++++++++++++++--- .../api/internal/clusters/instance_client.go | 15 ++- .../api/internal/clusters/instances_sync.go | 56 ++++++-- packages/api/internal/orchestrator/client.go | 17 +-- .../internal/orchestrator/delete_instance.go | 5 +- .../orchestrator/nodemanager/metadata.go | 67 ++++----- .../internal/orchestrator/nodemanager/mock.go | 14 +- .../internal/orchestrator/nodemanager/node.go | 29 ++-- .../nodemanager/sandbox_create.go | 4 +- .../orchestrator/nodemanager/status.go | 2 +- .../internal/orchestrator/pause_instance.go | 15 +-- .../internal/orchestrator/update_instance.go | 16 +-- .../template-manager/template_manager.go | 2 +- 14 files changed, 245 insertions(+), 183 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 087fb30961..6e86ee09c3 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -10,17 +10,12 @@ import ( "github.com/google/uuid" "go.opentelemetry.io/otel" - "go.uber.org/zap" "google.golang.org/grpc/metadata" - "google.golang.org/protobuf/types/known/emptypb" grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" - "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" - "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -38,7 +33,6 @@ type Cluster struct { SandboxDomain *string httpClient *api.ClientWithResponses - grpcClient *grpclient.GRPCClient instances *smap.Map[*Instance] synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *Instance] resources ClusterResource @@ -81,13 +75,13 @@ func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end return nil, fmt.Errorf("failed to create http client: %w", err) } - grpcAuthorization := clientAuthorization{secret: secret, tls: endpointTLS} - grpcClient, err := createClusterClient(tel, grpcAuthorization, endpoint, endpointTLS) - if err != nil { - return nil, fmt.Errorf("failed to create grpc client: %w", err) - } - instances := smap.New[*Instance]() + instanceCreation := func(ctx context.Context, item api.ClusterOrchestratorNode) (*Instance, error) { + // For remote cluster we are doing connection to endpoint that works as gRPC proxy and handles auth and routing for us. + auth := &instanceAuthorization{secret: secret, tls: endpointTLS, serviceInstanceID: item.ServiceInstanceID} + + return newInstance(ctx, tel, auth, clusterID, item.NodeID, item.ServiceInstanceID, endpoint, endpointTLS) + } c := &Cluster{ ID: clusterID, @@ -96,10 +90,9 @@ func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end resources: newRemoteClusterResourceProvider(instances, httpClient), instances: instances, httpClient: httpClient, - grpcClient: grpcClient, } - store := instancesSyncStore{cluster: c} + store := instancesSyncStore{cluster: c, instanceCreation: instanceCreation} c.synchronization = synchronization.NewSynchronize("cluster-instances", "Cluster instances", store) // periodically sync cluster instances @@ -110,9 +103,8 @@ func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end func (c *Cluster) Close() error { c.synchronization.Close() - err := c.grpcClient.Close() - return err + return nil } func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { @@ -149,9 +141,7 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e } // Make sure we will always iterate in different order and when there is bigger amount of builders, we will not always pick the same one - rand.Shuffle(len(instances), func(i, j int) { - instances[i], instances[j] = instances[j], instances[i] - }) + rand.Shuffle(len(instances), func(i, j int) { instances[i], instances[j] = instances[j], instances[i] }) for _, instance := range instances { if instance.GetStatus() != infogrpc.ServiceInfoStatus_Healthy { @@ -168,10 +158,6 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e return nil, ErrAvailableTemplateBuilderNotFound } -func (c *Cluster) GetGRPC(serviceInstanceID string) *ClusterGRPC { - return &ClusterGRPC{c.grpcClient, metadata.New(map[string]string{consts.EdgeRpcServiceInstanceIDHeader: serviceInstanceID})} -} - func (c *Cluster) GetOrchestrators() []*Instance { instances := make([]*Instance, 0) for _, i := range c.instances.Items() { @@ -190,30 +176,3 @@ func (c *Cluster) GetResources() ClusterResource { func (c *Cluster) startSync(ctx context.Context) { c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) } - -func (c *Cluster) syncInstance(ctx context.Context, instance *Instance) { - grpc := c.GetGRPC(instance.InstanceID) - - // we are taking service info directly from the instance to avoid timing delays in service discovery - reqCtx := metadata.NewOutgoingContext(ctx, grpc.Metadata) - info, err := grpc.Client.Info.ServiceInfo(reqCtx, &emptypb.Empty{}) - - err = utils.UnwrapGRPCError(err) - if err != nil { - logger.L().Error(ctx, "Failed to get instance info", - zap.Error(err), - logger.WithClusterID(c.ID), - logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), - ) - - return - } - - instance.mutex.Lock() - defer instance.mutex.Unlock() - - instance.status = info.GetServiceStatus() - instance.roles = info.GetServiceRoles() - instance.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) -} diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index f7eef1e69e..a0f06c9889 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -1,14 +1,26 @@ package clusters import ( + "context" + "fmt" "slices" "sync" + "github.com/google/uuid" + "go.uber.org/zap" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/e2b-dev/infra/packages/api/internal/grpc" + "github.com/e2b-dev/infra/packages/api/internal/utils" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) type Instance struct { + ClusterID uuid.UUID NodeID string InstanceID string @@ -17,36 +29,119 @@ type Instance struct { roles []infogrpc.ServiceInfoRole machineInfo machineinfo.MachineInfo + grpc *grpc.GRPCClient status infogrpc.ServiceInfoStatus mutex sync.RWMutex } -func (n *Instance) GetStatus() infogrpc.ServiceInfoStatus { - n.mutex.RLock() - defer n.mutex.RUnlock() +func newInstance( + ctx context.Context, + tel *telemetry.Client, + clusterAuth *instanceAuthorization, + clusterID uuid.UUID, + nodeID string, + instanceID string, + connAddr string, + connTls bool, +) (*Instance, error) { + conn, err := createConnection(tel, clusterAuth, connAddr, connTls) + if err != nil { + return nil, fmt.Errorf("failed to create cluster instance grpc client: %w", err) + } + + // Create with default values that will be updated on sync before returning the instance, + // so we will never have uninitialized instance status or roles. + // + // For case with local cluster we will not receive instance ID from service discovery, but its not needed for proxy routing, + // so it can be empty and will be filled after first sync. + i := &Instance{ + NodeID: nodeID, + ClusterID: clusterID, + InstanceID: instanceID, + + grpc: conn, + mutex: sync.RWMutex{}, + } + + err = i.Sync(ctx) + if err != nil { + closeErr := conn.Close() + if closeErr != nil { + logger.L().Error( + ctx, "Failed to close gRPC connection after instance sync failure", + zap.Error(err), + logger.WithNodeID(i.NodeID), + logger.WithClusterID(i.ClusterID), + logger.WithServiceInstanceID(i.InstanceID), + ) + } + + return nil, err + } + + return i, nil +} + +// Sync function can be called on freshly initialized instance to populate its data +// In initial case its possible that service instance id needed for proper remote cluster routing is not yet set. +func (i *Instance) Sync(ctx context.Context) error { + info, err := i.grpc.Info.ServiceInfo(ctx, &emptypb.Empty{}) + err = utils.UnwrapGRPCError(err) + if err != nil { + return err + } + + i.mutex.Lock() + defer i.mutex.Unlock() + + i.status = info.GetServiceStatus() + i.roles = info.GetServiceRoles() + i.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) + + i.InstanceID = info.GetServiceId() + i.ServiceVersion = info.GetServiceVersion() + i.ServiceVersionCommit = info.GetServiceCommit() + + return nil +} + +func (i *Instance) GetStatus() infogrpc.ServiceInfoStatus { + i.mutex.RLock() + defer i.mutex.RUnlock() - return n.status + return i.status } -func (n *Instance) GetMachineInfo() machineinfo.MachineInfo { - n.mutex.RLock() - defer n.mutex.RUnlock() +func (i *Instance) GetMachineInfo() machineinfo.MachineInfo { + i.mutex.RLock() + defer i.mutex.RUnlock() - return n.machineInfo + return i.machineInfo } -func (n *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { - n.mutex.RLock() - defer n.mutex.RUnlock() +func (i *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { + i.mutex.RLock() + defer i.mutex.RUnlock() + + return slices.Contains(i.roles, r) +} + +func (i *Instance) IsBuilder() bool { + return i.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) +} - return slices.Contains(n.roles, r) +func (i *Instance) IsOrchestrator() bool { + return i.hasRole(infogrpc.ServiceInfoRole_Orchestrator) } -func (n *Instance) IsBuilder() bool { - return n.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) +func (i *Instance) GetConnection() *ClusterGRPC { + return &ClusterGRPC{ + Client: i.grpc, + Metadata: metadata.New(nil), + } } -func (n *Instance) IsOrchestrator() bool { - return n.hasRole(infogrpc.ServiceInfoRole_Orchestrator) +func (i *Instance) Close() error { + return i.grpc.Close() } diff --git a/packages/api/internal/clusters/instance_client.go b/packages/api/internal/clusters/instance_client.go index deccf88b47..755d0f3dec 100644 --- a/packages/api/internal/clusters/instance_client.go +++ b/packages/api/internal/clusters/instance_client.go @@ -20,20 +20,21 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) -type clientAuthorization struct { - secret string - tls bool +type instanceAuthorization struct { + secret string + serviceInstanceID string + tls bool } -func (a clientAuthorization) GetRequestMetadata(_ context.Context, _ ...string) (map[string]string, error) { - return map[string]string{consts.EdgeRpcAuthHeader: a.secret}, nil +func (a instanceAuthorization) GetRequestMetadata(_ context.Context, _ ...string) (map[string]string, error) { + return map[string]string{consts.EdgeRpcAuthHeader: a.secret, consts.EdgeRpcServiceInstanceIDHeader: a.serviceInstanceID}, nil } -func (a clientAuthorization) RequireTransportSecurity() bool { +func (a instanceAuthorization) RequireTransportSecurity() bool { return a.tls } -func createClusterClient(tel *telemetry.Client, auth clientAuthorization, endpoint string, endpointTLS bool) (*grpclient.GRPCClient, error) { +func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*grpclient.GRPCClient, error) { grpcOptions := []grpc.DialOption{ grpc.WithPerRPCCredentials(auth), grpc.WithStatsHandler( diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 4872e68993..f44108bd52 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -5,16 +5,17 @@ import ( "errors" "fmt" "net/http" - "sync" - infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" + "go.uber.org/zap" + api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) // SynchronizationStore defines methods for synchronizing cluster instances type instancesSyncStore struct { - cluster *Cluster + cluster *Cluster + instanceCreation func(ctx context.Context, item api.ClusterOrchestratorNode) (*Instance, error) } func (d instancesSyncStore) SourceList(ctx context.Context) ([]api.ClusterOrchestratorNode, error) { @@ -69,26 +70,53 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrch logger.WithServiceInstanceID(item.ServiceInstanceID), ) - instance := &Instance{ - NodeID: item.NodeID, + instance, err := d.instanceCreation(ctx, item) + if err != nil { + logger.L().Error(ctx, "Failed to create cluster instance during pool insert", + zap.Error(err), + logger.WithClusterID(d.cluster.ID), + logger.WithNodeID(item.NodeID), + logger.WithServiceInstanceID(item.ServiceInstanceID), + ) + + return + } - InstanceID: item.ServiceInstanceID, - ServiceVersion: item.ServiceVersion, - ServiceVersionCommit: item.ServiceVersionCommit, + err = instance.Sync(ctx) + if err != nil { + closeErr := instance.Close() + if closeErr != nil { + logger.L().Error(ctx, "Failed to close cluster instance after sync failure", + zap.Error(closeErr), + logger.WithClusterID(d.cluster.ID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) + } - // initial values before first sync - status: infogrpc.ServiceInfoStatus_Unhealthy, - roles: make([]infogrpc.ServiceInfoRole, 0), + logger.L().Error(ctx, "Failed to sync cluster instance during pool insert", + zap.Error(err), + logger.WithClusterID(d.cluster.ID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) - mutex: sync.RWMutex{}, + return } - d.cluster.syncInstance(ctx, instance) d.cluster.instances.Insert(item.NodeID, instance) } func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) { - d.cluster.syncInstance(ctx, instance) + err := instance.Sync(ctx) + if err != nil { + logger.L().Error(ctx, "Failed to sync cluster instance during pool update", + zap.Error(err), + logger.WithClusterID(d.cluster.ID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) + } } func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) { diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index 73c0db1250..9e9395c4c5 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -10,7 +10,6 @@ import ( "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/clusters" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" "github.com/e2b-dev/infra/packages/shared/pkg/consts" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -34,10 +33,7 @@ func (o *Orchestrator) connectToNode(ctx context.Context, discovered nodemanager } func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.Instance) { - // this way we don't need to worry about multiple clusters with the same node ID in shared pool - clusterGRPC := cluster.GetGRPC(i.InstanceID) - - orchestratorNode, err := nodemanager.NewClusterNode(ctx, clusterGRPC.Client, cluster.ID, cluster.SandboxDomain, i) + orchestratorNode, err := nodemanager.NewClusterNode(ctx, i.GetConnection().Client, cluster.ID, cluster.SandboxDomain, i) if err != nil { logger.L().Error(ctx, "Failed to create node", zap.Error(err)) @@ -66,17 +62,6 @@ func (o *Orchestrator) scopedNodeID(clusterID uuid.UUID, nodeID string) string { return fmt.Sprintf("%s-%s", clusterID.String(), nodeID) } -func (o *Orchestrator) GetClient(ctx context.Context, clusterID uuid.UUID, nodeID string) (*grpclient.GRPCClient, context.Context, error) { - n := o.GetNode(clusterID, nodeID) - if n == nil { - return nil, nil, fmt.Errorf("node '%s' not found in cluster '%s'", nodeID, clusterID) - } - - client, ctx := n.GetClient(ctx) - - return client, ctx, nil -} - func (o *Orchestrator) listNomadNodes(ctx context.Context) ([]nodemanager.NomadServiceDiscovery, error) { _, listSpan := tracer.Start(ctx, "list-nomad-nodes") defer listSpan.End() diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index 36f54f4221..a8f422eb85 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -105,8 +105,9 @@ func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sa case sandbox.StateActionKill: var err error req := &orchestrator.SandboxDeleteRequest{SandboxId: sbx.SandboxID} - client, ctx := node.GetClient(ctx) - _, err = client.Sandbox.Delete(node.GetSandboxDeleteCtx(ctx, sbx.SandboxID, sbx.ExecutionID), req) + + client, ctx := node.GetSandboxDeleteCtx(ctx, sbx.SandboxID, sbx.ExecutionID) + _, err = client.Sandbox.Delete(ctx, req) if err != nil { return fmt.Errorf("failed to delete sandbox '%s': %w", sbx.SandboxID, err) } diff --git a/packages/api/internal/orchestrator/nodemanager/metadata.go b/packages/api/internal/orchestrator/nodemanager/metadata.go index a498315c1a..a43bae4337 100644 --- a/packages/api/internal/orchestrator/nodemanager/metadata.go +++ b/packages/api/internal/orchestrator/nodemanager/metadata.go @@ -5,7 +5,7 @@ import ( "google.golang.org/grpc/metadata" - "github.com/e2b-dev/infra/packages/shared/pkg/consts" + "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/edge" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" ) @@ -33,44 +33,47 @@ func (n *Node) Metadata() NodeMetadata { return n.meta } -// Generates Metadata with the current service instance ID -// to ensure we always use the latest ID (e.g. after orchestrator restarts) -func (n *Node) getClientMetadata() metadata.MD { - return metadata.New(map[string]string{consts.EdgeRpcServiceInstanceIDHeader: n.Metadata().ServiceInstanceID}) -} +func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.SandboxCreateRequest) (*grpc.GRPCClient, context.Context) { + md := metadata.MD{} + + if !n.IsNomadManaged() { + md = edge.SerializeSandboxCatalogCreateEvent( + edge.SandboxCatalogCreateEvent{ + SandboxID: req.GetSandbox().GetSandboxId(), + SandboxMaxLengthInHours: req.GetSandbox().GetMaxSandboxLength(), + SandboxStartTime: req.GetStartTime().AsTime(), -func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.SandboxCreateRequest) context.Context { - // Skip local cluster. It should be okay to send it here, but we don't want to do it until we explicitly support it. - if n.IsNomadManaged() { - return ctx + ExecutionID: req.GetSandbox().GetExecutionId(), + OrchestratorID: n.Metadata().ServiceInstanceID, + }, + ) } - md := edge.SerializeSandboxCatalogCreateEvent( - edge.SandboxCatalogCreateEvent{ - SandboxID: req.GetSandbox().GetSandboxId(), - SandboxMaxLengthInHours: req.GetSandbox().GetMaxSandboxLength(), - SandboxStartTime: req.GetStartTime().AsTime(), + // Merge medata from connection (auth, routing with service instance id) and event metadata. + return n.connection, appendMetadataCtx(ctx, md) +} + +func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, executionID string) (*grpc.GRPCClient, context.Context) { + md := metadata.MD{} - ExecutionID: req.GetSandbox().GetExecutionId(), - OrchestratorID: n.Metadata().ServiceInstanceID, - }, - ) + if !n.IsNomadManaged() { + md = edge.SerializeSandboxCatalogDeleteEvent( + edge.SandboxCatalogDeleteEvent{ + SandboxID: sandboxID, + ExecutionID: executionID, + }, + ) + } - return metadata.NewOutgoingContext(ctx, metadata.Join(n.getClientMetadata(), md)) + // Merge medata from connection (auth, routing with service instance id) and event metadata. + return n.connection, appendMetadataCtx(ctx, md) } -func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, executionID string) context.Context { - // Skip local cluster. It should be okay to send it here, but we don't want to do it until we explicitly support it. - if n.IsNomadManaged() { - return ctx +func appendMetadataCtx(ctx context.Context, md metadata.MD) context.Context { + args := make([]string, 0, len(md)*2) + for k, v := range md { + args = append(args, k, v[0]) } - md := edge.SerializeSandboxCatalogDeleteEvent( - edge.SandboxCatalogDeleteEvent{ - SandboxID: sandboxID, - ExecutionID: executionID, - }, - ) - - return metadata.NewOutgoingContext(ctx, metadata.Join(n.getClientMetadata(), md)) + return metadata.AppendToOutgoingContext(ctx, args...) } diff --git a/packages/api/internal/orchestrator/nodemanager/mock.go b/packages/api/internal/orchestrator/nodemanager/mock.go index 4451b76dc0..46397f3de4 100644 --- a/packages/api/internal/orchestrator/nodemanager/mock.go +++ b/packages/api/internal/orchestrator/nodemanager/mock.go @@ -57,7 +57,7 @@ func (n *mockSandboxClientWithSleep) Create(_ context.Context, _ *orchestrator.S return &orchestrator.SandboxCreateResponse{}, nil } -// newMockGRPCClient creates a new mock gRPC client for testing +// newMockGRPCClient creates a new mock gRPC connection for testing func newMockGRPCClient() *grpclient.GRPCClient { // Create a dummy connection that will never be used conn, _ := grpc.NewClient("localhost:0", grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -74,7 +74,7 @@ type TestOptions func(node *TestNode) func WithSandboxSleepingClient(baseSandboxCreateTime time.Duration) TestOptions { return func(node *TestNode) { - node.client.Sandbox = &mockSandboxClientWithSleep{ + node.connection.Sandbox = &mockSandboxClientWithSleep{ baseSandboxCreateTime: baseSandboxCreateTime, } } @@ -102,7 +102,7 @@ func (n *mockSandboxClientWithError) Create(_ context.Context, _ *orchestrator.S func WithSandboxCreateError(err error) TestOptions { return func(node *TestNode) { - node.client.Sandbox = &mockSandboxClientWithError{ + node.connection.Sandbox = &mockSandboxClientWithError{ err: err, } } @@ -126,18 +126,18 @@ func (n *MockSandboxClientCustom) Create(_ context.Context, _ *orchestrator.Sand return &orchestrator.SandboxCreateResponse{}, nil } -// SetSandboxClient allows setting a custom sandbox client on a test node +// SetSandboxClient allows setting a custom sandbox connection on a test node func (n *TestNode) SetSandboxClient(client orchestrator.SandboxServiceClient) { - n.client.Sandbox = client + n.connection.Sandbox = client } // NewTestNode creates a properly initialized Node for testing purposes -// It uses a mock gRPC client and has simplified Status() method behavior +// It uses a mock gRPC connection and has simplified Status() method behavior func NewTestNode(id string, status api.NodeStatus, cpuAllocated int64, cpuCount uint32, options ...TestOptions) *TestNode { node := &Node{ ID: id, ClusterID: uuid.New(), - client: newMockGRPCClient(), + connection: newMockGRPCClient(), IPAddress: "127.0.0.1", SandboxDomain: nil, status: status, diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index 3c6d43bd52..6cfe3dbfbf 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -11,7 +11,6 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/emptypb" "github.com/e2b-dev/infra/packages/api/internal/api" @@ -41,8 +40,8 @@ type Node struct { IPAddress string SandboxDomain *string - client *grpclient.GRPCClient - status api.NodeStatus + connection *grpclient.GRPCClient + status api.NodeStatus metrics Metrics metricsMu sync.RWMutex @@ -97,9 +96,9 @@ func New( IPAddress: discoveredNode.IPAddress, SandboxDomain: nil, - client: client, - status: nodeStatus, - meta: nodeMetadata, + connection: client, + status: nodeStatus, + meta: nodeMetadata, buildCache: buildCache, PlacementMetrics: PlacementMetrics{ @@ -137,17 +136,17 @@ func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID // We can't connect directly to the node in the cluster IPAddress: "", SandboxDomain: sandboxDomain, - - client: client, - status: nodeStatus, - meta: nodeMetadata, - - buildCache: buildCache, PlacementMetrics: PlacementMetrics{ sandboxesInProgress: smap.New[SandboxResources](), createSuccess: atomic.Uint64{}, createFails: atomic.Uint64{}, }, + + connection: client, + status: nodeStatus, + meta: nodeMetadata, + + buildCache: buildCache, } nodeClient, ctx := n.GetClient(ctx) @@ -167,7 +166,7 @@ func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID func (n *Node) Close(ctx context.Context) error { if n.IsNomadManaged() { logger.L().Info(ctx, "Closing local node", logger.WithNodeID(n.ID)) - err := n.client.Close() + err := n.connection.Close() if err != nil { logger.L().Error(ctx, "Error closing connection to node", zap.Error(err), logger.WithNodeID(n.ID)) } @@ -180,9 +179,9 @@ func (n *Node) Close(ctx context.Context) error { return nil } -// Ensures that GRPC client request context always has the latest service instance ID +// Ensures that GRPC connection request context always has the latest service instance ID func (n *Node) GetClient(ctx context.Context) (*grpclient.GRPCClient, context.Context) { - return n.client, metadata.NewOutgoingContext(ctx, n.getClientMetadata()) + return n.connection, ctx } func (n *Node) IsNomadManaged() bool { diff --git a/packages/api/internal/orchestrator/nodemanager/sandbox_create.go b/packages/api/internal/orchestrator/nodemanager/sandbox_create.go index f74f141b69..91fa803b5d 100644 --- a/packages/api/internal/orchestrator/nodemanager/sandbox_create.go +++ b/packages/api/internal/orchestrator/nodemanager/sandbox_create.go @@ -7,8 +7,8 @@ import ( ) func (n *Node) SandboxCreate(ctx context.Context, sbxRequest *orchestrator.SandboxCreateRequest) error { - client, ctx := n.GetClient(ctx) - _, err := client.Sandbox.Create(n.GetSandboxCreateCtx(ctx, sbxRequest), sbxRequest) + client, ctx := n.GetSandboxCreateCtx(ctx, sbxRequest) + _, err := client.Sandbox.Create(ctx, sbxRequest) if err != nil { return err } diff --git a/packages/api/internal/orchestrator/nodemanager/status.go b/packages/api/internal/orchestrator/nodemanager/status.go index 8c45c0f66a..0b5e9db6b6 100644 --- a/packages/api/internal/orchestrator/nodemanager/status.go +++ b/packages/api/internal/orchestrator/nodemanager/status.go @@ -26,7 +26,7 @@ func (n *Node) Status() api.NodeStatus { return n.status } - switch n.client.Connection.GetState() { + switch n.connection.Connection.GetState() { case connectivity.Shutdown: return api.NodeStatusUnhealthy case connectivity.TransientFailure: diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index 0514057173..bab17a445b 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -70,7 +70,7 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, return err } - err = snapshotInstance(ctx, o, node, sbx, result.TemplateID, result.BuildID.String()) + err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String()) if errors.Is(err, PauseQueueExhaustedError{}) { telemetry.ReportCriticalError(ctx, "pause queue exhausted", err) @@ -100,18 +100,13 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, return nil } -func snapshotInstance(ctx context.Context, orch *Orchestrator, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string) error { +func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string) error { childCtx, childSpan := tracer.Start(ctx, "snapshot-instance") defer childSpan.End() - client, childCtx, err := orch.GetClient(childCtx, sbx.ClusterID, sbx.NodeID) - if err != nil { - return fmt.Errorf("failed to get client '%s': %w", sbx.NodeID, err) - } - - _, err = client.Sandbox.Pause( - node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID), - &orchestrator.SandboxPauseRequest{ + client, childCtx := node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID) + _, err := client.Sandbox.Pause( + childCtx, &orchestrator.SandboxPauseRequest{ SandboxId: sbx.SandboxID, TemplateId: templateID, BuildId: buildID, diff --git a/packages/api/internal/orchestrator/update_instance.go b/packages/api/internal/orchestrator/update_instance.go index 05b465fe14..f751432d15 100644 --- a/packages/api/internal/orchestrator/update_instance.go +++ b/packages/api/internal/orchestrator/update_instance.go @@ -24,20 +24,16 @@ func (o *Orchestrator) UpdateSandbox( clusterID uuid.UUID, nodeID string, ) error { - childCtx, childSpan := tracer.Start(ctx, "update-sandbox", + ctx, span := tracer.Start(ctx, "update-sandbox", trace.WithAttributes( attribute.String("instance.id", sandboxID), ), ) - defer childSpan.End() + defer span.End() - client, childCtx, err := o.GetClient(childCtx, clusterID, nodeID) - if err != nil { - return fmt.Errorf("failed to get client '%s': %w", nodeID, err) - } - - _, err = client.Sandbox.Update( - childCtx, &orchestrator.SandboxUpdateRequest{ + client, ctx := o.GetNode(clusterID, nodeID).GetClient(ctx) + _, err := client.Sandbox.Update( + ctx, &orchestrator.SandboxUpdateRequest{ SandboxId: sandboxID, EndTime: timestamppb.New(endTime), }, @@ -53,7 +49,7 @@ func (o *Orchestrator) UpdateSandbox( return fmt.Errorf("failed to update sandbox '%s': %w", sandboxID, err) } - telemetry.ReportEvent(childCtx, "Updated sandbox") + telemetry.ReportEvent(ctx, "Updated sandbox") return nil } diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 30d2ace1a4..5c418c4523 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -142,7 +142,7 @@ func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID str return nil, fmt.Errorf("failed to get builder by id '%s': %w", nodeID, err) } - return cluster.GetGRPC(instance.InstanceID), nil + return instance.GetConnection(), nil } func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, templateID string, clusterID uuid.UUID, nodeID string) error { From b36641f613a2f021819e5d45c46b574459796339 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 14:58:49 +0100 Subject: [PATCH 16/58] Removed not needed metadata ctx call with instance connection --- .../api/internal/{grpc => clusters}/client.go | 4 +-- packages/api/internal/clusters/cluster.go | 9 +----- packages/api/internal/clusters/instance.go | 13 +++----- .../api/internal/clusters/instance_client.go | 5 ++-- packages/api/internal/orchestrator/client.go | 2 +- .../orchestrator/nodemanager/builds.go | 4 +-- .../orchestrator/nodemanager/client.go | 6 ++-- .../orchestrator/nodemanager/metadata.go | 6 ++-- .../internal/orchestrator/nodemanager/mock.go | 30 ++++++++----------- .../internal/orchestrator/nodemanager/node.go | 7 ++--- .../template-manager/create_template.go | 6 ++-- .../template-manager/template_manager.go | 13 ++++---- .../upload_template_layer_files.go | 6 ++-- 13 files changed, 42 insertions(+), 69 deletions(-) rename packages/api/internal/{grpc => clusters}/client.go (88%) diff --git a/packages/api/internal/grpc/client.go b/packages/api/internal/clusters/client.go similarity index 88% rename from packages/api/internal/grpc/client.go rename to packages/api/internal/clusters/client.go index 4597375c2f..d4845e5851 100644 --- a/packages/api/internal/grpc/client.go +++ b/packages/api/internal/clusters/client.go @@ -1,4 +1,4 @@ -package grpc +package clusters import ( "fmt" @@ -21,7 +21,7 @@ type GRPCClient struct { func (a *GRPCClient) Close() error { err := a.Connection.Close() if err != nil { - return fmt.Errorf("failed to close connection: %w", err) + return fmt.Errorf("failed to close Connection: %w", err) } return nil diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 6e86ee09c3..32daea9b13 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -10,9 +10,7 @@ import ( "github.com/google/uuid" "go.opentelemetry.io/otel" - "google.golang.org/grpc/metadata" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" @@ -38,11 +36,6 @@ type Cluster struct { resources ClusterResource } -type ClusterGRPC struct { - Client *grpclient.GRPCClient - Metadata metadata.MD -} - var ( ErrTemplateBuilderNotFound = errors.New("template builder not found") ErrAvailableTemplateBuilderNotFound = errors.New("available template builder not found") @@ -77,7 +70,7 @@ func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, end instances := smap.New[*Instance]() instanceCreation := func(ctx context.Context, item api.ClusterOrchestratorNode) (*Instance, error) { - // For remote cluster we are doing connection to endpoint that works as gRPC proxy and handles auth and routing for us. + // For remote cluster we are doing Connection to endpoint that works as gRPC proxy and handles auth and routing for us. auth := &instanceAuthorization{secret: secret, tls: endpointTLS, serviceInstanceID: item.ServiceInstanceID} return newInstance(ctx, tel, auth, clusterID, item.NodeID, item.ServiceInstanceID, endpoint, endpointTLS) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index a0f06c9889..bdf529c5e1 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -8,10 +8,8 @@ import ( "github.com/google/uuid" "go.uber.org/zap" - "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/emptypb" - "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/api/internal/utils" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -29,7 +27,7 @@ type Instance struct { roles []infogrpc.ServiceInfoRole machineInfo machineinfo.MachineInfo - grpc *grpc.GRPCClient + grpc *GRPCClient status infogrpc.ServiceInfoStatus mutex sync.RWMutex @@ -69,7 +67,7 @@ func newInstance( closeErr := conn.Close() if closeErr != nil { logger.L().Error( - ctx, "Failed to close gRPC connection after instance sync failure", + ctx, "Failed to close gRPC Connection after instance sync failure", zap.Error(err), logger.WithNodeID(i.NodeID), logger.WithClusterID(i.ClusterID), @@ -135,11 +133,8 @@ func (i *Instance) IsOrchestrator() bool { return i.hasRole(infogrpc.ServiceInfoRole_Orchestrator) } -func (i *Instance) GetConnection() *ClusterGRPC { - return &ClusterGRPC{ - Client: i.grpc, - Metadata: metadata.New(nil), - } +func (i *Instance) GetConnection() *GRPCClient { + return i.grpc } func (i *Instance) Close() error { diff --git a/packages/api/internal/clusters/instance_client.go b/packages/api/internal/clusters/instance_client.go index 755d0f3dec..d454426559 100644 --- a/packages/api/internal/clusters/instance_client.go +++ b/packages/api/internal/clusters/instance_client.go @@ -12,7 +12,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/consts" orchestratorgrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" @@ -34,7 +33,7 @@ func (a instanceAuthorization) RequireTransportSecurity() bool { return a.tls } -func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*grpclient.GRPCClient, error) { +func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*GRPCClient, error) { grpcOptions := []grpc.DialOption{ grpc.WithPerRPCCredentials(auth), grpc.WithStatsHandler( @@ -65,7 +64,7 @@ func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoi return nil, fmt.Errorf("failed to create grpc client: %w", err) } - return &grpclient.GRPCClient{ + return &GRPCClient{ Info: infogrpc.NewInfoServiceClient(conn), Sandbox: orchestratorgrpc.NewSandboxServiceClient(conn), Template: templatemanagergrpc.NewTemplateServiceClient(conn), diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index 9e9395c4c5..ad340af9f0 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -33,7 +33,7 @@ func (o *Orchestrator) connectToNode(ctx context.Context, discovered nodemanager } func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.Instance) { - orchestratorNode, err := nodemanager.NewClusterNode(ctx, i.GetConnection().Client, cluster.ID, cluster.SandboxDomain, i) + orchestratorNode, err := nodemanager.NewClusterNode(ctx, i.GetConnection(), cluster.ID, cluster.SandboxDomain, i) if err != nil { logger.L().Error(ctx, "Failed to create node", zap.Error(err)) diff --git a/packages/api/internal/orchestrator/nodemanager/builds.go b/packages/api/internal/orchestrator/nodemanager/builds.go index 58cf9623fb..88470d8b26 100644 --- a/packages/api/internal/orchestrator/nodemanager/builds.go +++ b/packages/api/internal/orchestrator/nodemanager/builds.go @@ -34,9 +34,7 @@ func (n *Node) listCachedBuilds(ctx context.Context) ([]*orchestrator.CachedBuil childCtx, childSpan := tracer.Start(ctx, "list-cached-builds") defer childSpan.End() - client, childCtx := n.GetClient(childCtx) - res, err := client.Sandbox.ListCachedBuilds(childCtx, &empty.Empty{}) - + res, err := n.connection.Sandbox.ListCachedBuilds(childCtx, &empty.Empty{}) err = utils.UnwrapGRPCError(err) if err != nil { return nil, fmt.Errorf("failed to list sandboxes: %w", err) diff --git a/packages/api/internal/orchestrator/nodemanager/client.go b/packages/api/internal/orchestrator/nodemanager/client.go index 468c52d102..ab7bb24189 100644 --- a/packages/api/internal/orchestrator/nodemanager/client.go +++ b/packages/api/internal/orchestrator/nodemanager/client.go @@ -10,7 +10,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/e2b-dev/infra/packages/api/internal/api" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" orchestratorinfo "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" ) @@ -21,7 +21,7 @@ var OrchestratorToApiNodeStateMapper = map[orchestratorinfo.ServiceInfoStatus]ap orchestratorinfo.ServiceInfoStatus_Unhealthy: api.NodeStatusUnhealthy, } -func NewClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*grpclient.GRPCClient, error) { +func NewClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterProvider, host string) (*clusters.GRPCClient, error) { conn, err := grpc.NewClient(host, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler( @@ -38,5 +38,5 @@ func NewClient(tracerProvider trace.TracerProvider, meterProvider metric.MeterPr sandboxClient := orchestrator.NewSandboxServiceClient(conn) infoClient := orchestratorinfo.NewInfoServiceClient(conn) - return &grpclient.GRPCClient{Sandbox: sandboxClient, Info: infoClient, Connection: conn}, nil + return &clusters.GRPCClient{Sandbox: sandboxClient, Info: infoClient, Connection: conn}, nil } diff --git a/packages/api/internal/orchestrator/nodemanager/metadata.go b/packages/api/internal/orchestrator/nodemanager/metadata.go index a43bae4337..a4a0bfdd2c 100644 --- a/packages/api/internal/orchestrator/nodemanager/metadata.go +++ b/packages/api/internal/orchestrator/nodemanager/metadata.go @@ -5,7 +5,7 @@ import ( "google.golang.org/grpc/metadata" - "github.com/e2b-dev/infra/packages/api/internal/grpc" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/shared/pkg/edge" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" ) @@ -33,7 +33,7 @@ func (n *Node) Metadata() NodeMetadata { return n.meta } -func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.SandboxCreateRequest) (*grpc.GRPCClient, context.Context) { +func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.SandboxCreateRequest) (*clusters.GRPCClient, context.Context) { md := metadata.MD{} if !n.IsNomadManaged() { @@ -53,7 +53,7 @@ func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.Sandbo return n.connection, appendMetadataCtx(ctx, md) } -func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, executionID string) (*grpc.GRPCClient, context.Context) { +func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, executionID string) (*clusters.GRPCClient, context.Context) { md := metadata.MD{} if !n.IsNomadManaged() { diff --git a/packages/api/internal/orchestrator/nodemanager/mock.go b/packages/api/internal/orchestrator/nodemanager/mock.go index 46397f3de4..4fa8c6632e 100644 --- a/packages/api/internal/orchestrator/nodemanager/mock.go +++ b/packages/api/internal/orchestrator/nodemanager/mock.go @@ -8,10 +8,9 @@ import ( "github.com/google/uuid" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "github.com/e2b-dev/infra/packages/api/internal/api" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" + "github.com/e2b-dev/infra/packages/api/internal/clusters" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" templatemanager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" @@ -58,15 +57,11 @@ func (n *mockSandboxClientWithSleep) Create(_ context.Context, _ *orchestrator.S } // newMockGRPCClient creates a new mock gRPC connection for testing -func newMockGRPCClient() *grpclient.GRPCClient { - // Create a dummy connection that will never be used - conn, _ := grpc.NewClient("localhost:0", grpc.WithTransportCredentials(insecure.NewCredentials())) - - return &grpclient.GRPCClient{ - Info: &mockInfoClient{}, - Sandbox: &mockSandboxClient{}, - Template: &mockTemplateClient{}, - Connection: conn, +func newMockGRPCClient() *clusters.GRPCClient { + return &clusters.GRPCClient{ + Info: &mockInfoClient{}, + Sandbox: &mockSandboxClient{}, + Template: &mockTemplateClient{}, } } @@ -137,19 +132,20 @@ func NewTestNode(id string, status api.NodeStatus, cpuAllocated int64, cpuCount node := &Node{ ID: id, ClusterID: uuid.New(), - connection: newMockGRPCClient(), IPAddress: "127.0.0.1", SandboxDomain: nil, - status: status, - metrics: Metrics{ - CpuAllocated: uint32(cpuAllocated), - CpuCount: cpuCount, - }, PlacementMetrics: PlacementMetrics{ sandboxesInProgress: smap.New[SandboxResources](), createSuccess: atomic.Uint64{}, createFails: atomic.Uint64{}, }, + + connection: newMockGRPCClient(), + status: status, + metrics: Metrics{ + CpuAllocated: uint32(cpuAllocated), + CpuCount: cpuCount, + }, } for _, option := range options { diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index 6cfe3dbfbf..b4b8af7184 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -15,7 +15,6 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/clusters" - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/consts" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" @@ -40,7 +39,7 @@ type Node struct { IPAddress string SandboxDomain *string - connection *grpclient.GRPCClient + connection *clusters.GRPCClient status api.NodeStatus metrics Metrics @@ -113,7 +112,7 @@ func New( return n, nil } -func NewClusterNode(ctx context.Context, client *grpclient.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.Instance) (*Node, error) { +func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.Instance) (*Node, error) { nodeStatus, ok := OrchestratorToApiNodeStateMapper[i.GetStatus()] if !ok { logger.L().Error(ctx, "Unknown service info status", zap.String("status", i.GetStatus().String()), logger.WithNodeID(i.NodeID)) @@ -180,7 +179,7 @@ func (n *Node) Close(ctx context.Context) error { } // Ensures that GRPC connection request context always has the latest service instance ID -func (n *Node) GetClient(ctx context.Context) (*grpclient.GRPCClient, context.Context) { +func (n *Node) GetClient(ctx context.Context) (*clusters.GRPCClient, context.Context) { return n.connection, ctx } diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index c9124bfe6a..9401952881 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/sandbox" @@ -149,9 +148,8 @@ func (tm *TemplateManager) CreateTemplate( return nil } - _, err = client.Client.Template.TemplateCreate( - metadata.NewOutgoingContext(ctx, client.Metadata), - &templatemanagergrpc.TemplateCreateRequest{ + _, err = client.Template.TemplateCreate( + ctx, &templatemanagergrpc.TemplateCreateRequest{ Template: template, CacheScope: ut.ToPtr(teamID.String()), Version: &version, diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 5c418c4523..6a17096ab9 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -11,7 +11,6 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - "google.golang.org/grpc/metadata" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" "github.com/e2b-dev/infra/packages/api/internal/clusters" @@ -131,7 +130,7 @@ func (tm *TemplateManager) GetClusterResources(clusterID uuid.UUID) (clusters.Cl return cluster.GetResources(), nil } -func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID string) (*clusters.ClusterGRPC, error) { +func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID string) (*clusters.GRPCClient, error) { cluster, ok := tm.clusters.GetClusterById(clusterID) if !ok { return nil, errors.New("cluster not found") @@ -171,9 +170,8 @@ func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, t } } - _, err = client.Client.Template.TemplateBuildDelete( - metadata.NewOutgoingContext(ctx, client.Metadata), - &templatemanagergrpc.TemplateBuildDeleteRequest{ + _, err = client.Template.TemplateBuildDelete( + ctx, &templatemanagergrpc.TemplateBuildDeleteRequest{ BuildID: buildID.String(), TemplateID: templateID, }, @@ -205,9 +203,8 @@ func (tm *TemplateManager) GetStatus(ctx context.Context, buildID uuid.UUID, tem } // error unwrapping is done in the caller - return client.Client.Template.TemplateBuildStatus( - metadata.NewOutgoingContext(ctx, client.Metadata), - &templatemanagergrpc.TemplateStatusRequest{ + return client.Template.TemplateBuildStatus( + ctx, &templatemanagergrpc.TemplateStatusRequest{ BuildID: buildID.String(), TemplateID: templateID, }, ) diff --git a/packages/api/internal/template-manager/upload_template_layer_files.go b/packages/api/internal/template-manager/upload_template_layer_files.go index 41cab88265..020682e1c8 100644 --- a/packages/api/internal/template-manager/upload_template_layer_files.go +++ b/packages/api/internal/template-manager/upload_template_layer_files.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/google/uuid" - "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/utils" templatemanager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" @@ -18,9 +17,8 @@ func (tm *TemplateManager) InitLayerFileUpload(ctx context.Context, clusterID uu return nil, fmt.Errorf("failed to get build client for template '%s': %w", templateID, err) } - reqCtx := metadata.NewOutgoingContext(ctx, client.Metadata) - resp, err := client.Client.Template.InitLayerFileUpload( - reqCtx, &templatemanager.InitLayerFileUploadRequest{ + resp, err := client.Template.InitLayerFileUpload( + ctx, &templatemanager.InitLayerFileUploadRequest{ CacheScope: ut.ToPtr(teamID.String()), TemplateID: templateID, Hash: hash, From 728cac82bbc64d6bc48465fd9ffcaff9452d8f59 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 15:34:26 +0100 Subject: [PATCH 17/58] Nomad allocations discovery with different query filters As now client proxy service discovery should discovery both template builders and orchestrators but api implementation for local cluster should discovery just template builders as orchestrators still discovered old way - introduction of query filter option is needed. --- .../api/internal/clusters/discovery/local.go | 1 + .../nomad-service-discovery.go | 2 +- .../shared/pkg/clusters/discovery/nomad.go | 28 +++++++++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 packages/api/internal/clusters/discovery/local.go diff --git a/packages/api/internal/clusters/discovery/local.go b/packages/api/internal/clusters/discovery/local.go new file mode 100644 index 0000000000..5844159aeb --- /dev/null +++ b/packages/api/internal/clusters/discovery/local.go @@ -0,0 +1 @@ +package discovery diff --git a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go index b4da591beb..806a461b2e 100644 --- a/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/nomad-service-discovery.go @@ -78,7 +78,7 @@ func (sd *NomadServiceDiscovery) sync(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, nomadQueryRefreshInterval) defer cancel() - alloc, err := discovery.ListOrchestratorAndTemplateBuilderAllocations(ctx, sd.client) + alloc, err := discovery.ListOrchestratorAndTemplateBuilderAllocations(ctx, sd.client, discovery.FilterTemplateBuildersAndOrchestrators) if err != nil { sd.logger.Error(ctx, "Failed to list orchestrator and template builders", zap.Error(err)) diff --git a/packages/shared/pkg/clusters/discovery/nomad.go b/packages/shared/pkg/clusters/discovery/nomad.go index e3087ea294..dffee0ba89 100644 --- a/packages/shared/pkg/clusters/discovery/nomad.go +++ b/packages/shared/pkg/clusters/discovery/nomad.go @@ -24,20 +24,32 @@ const ( orchestratorJobPrefix = "orchestrator" ) -var filter = fmt.Sprintf( - "ClientStatus == \"running\" and ((TaskGroup == \"%s\" and JobID contains \"%s\") or (TaskGroup == \"%s\" and JobID contains \"%s\"))", - templateManagersTaskGroup, - templateManagerJobPrefix, - orchestratorsTaskGroup, - orchestratorJobPrefix, +type NomadQueryFilter string + +var FilterTemplateBuilders = NomadQueryFilter( + fmt.Sprintf( + "ClientStatus == \"running\" and TaskGroup == \"%s\" and JobID contains \"%s\"", + templateManagersTaskGroup, + templateManagerJobPrefix, + ), ) -func ListOrchestratorAndTemplateBuilderAllocations(ctx context.Context, client *nomadapi.Client) ([]Allocation, error) { +var FilterTemplateBuildersAndOrchestrators = NomadQueryFilter( + fmt.Sprintf( + "ClientStatus == \"running\" and ((TaskGroup == \"%s\" and JobID contains \"%s\") or (TaskGroup == \"%s\" and JobID contains \"%s\"))", + templateManagersTaskGroup, + templateManagerJobPrefix, + orchestratorsTaskGroup, + orchestratorJobPrefix, + ), +) + +func ListOrchestratorAndTemplateBuilderAllocations(ctx context.Context, client *nomadapi.Client, filter NomadQueryFilter) ([]Allocation, error) { options := &nomadapi.QueryOptions{ // https://developer.hashicorp.com/nomad/api-docs/allocations#resources // Return allocation resources as part of the response Params: map[string]string{"resources": "true"}, - Filter: filter, + Filter: string(filter), } results, _, err := client.Allocations().List(options.WithContext(ctx)) From 704be350449e3ded40db86f7b9122f4ced341ff3 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 15:36:26 +0100 Subject: [PATCH 18/58] Cluster local and remote discovery impls New implementation for cluster instances discovery that creates difference between local nodes (discovers only template managers) via Nomad and remote discovery that is using edge rest api to receive info about running instances. We are now creating two implementations of clusters, each with different service discovery impl. --- packages/api/internal/clusters/cluster.go | 121 +++++++++++++----- .../api/internal/clusters/clusters_sync.go | 98 ++++++++------ .../internal/clusters/discovery/discovery.go | 27 ++++ .../api/internal/clusters/discovery/local.go | 59 +++++++++ .../api/internal/clusters/discovery/remote.go | 56 ++++++++ packages/api/internal/clusters/instance.go | 15 ++- .../api/internal/clusters/instances_sync.go | 70 +++++----- packages/api/internal/handlers/sandbox_get.go | 2 +- .../api/internal/handlers/sandbox_logs.go | 2 +- .../api/internal/handlers/sandbox_metrics.go | 2 +- .../handlers/sandboxes_list_metrics.go | 2 +- packages/api/internal/handlers/store.go | 12 +- .../internal/handlers/template_build_logs.go | 2 +- .../handlers/template_build_status.go | 2 +- packages/api/internal/orchestrator/client.go | 2 +- packages/shared/pkg/consts/sandboxes.go | 9 +- 16 files changed, 353 insertions(+), 128 deletions(-) create mode 100644 packages/api/internal/clusters/discovery/discovery.go create mode 100644 packages/api/internal/clusters/discovery/remote.go diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 32daea9b13..cb190ed70c 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -9,8 +9,10 @@ import ( "time" "github.com/google/uuid" + nomadapi "github.com/hashicorp/nomad/api" "go.opentelemetry.io/otel" + "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" @@ -30,9 +32,8 @@ type Cluster struct { ID uuid.UUID SandboxDomain *string - httpClient *api.ClientWithResponses instances *smap.Map[*Instance] - synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *Instance] + synchronization *synchronization.Synchronize[discovery.Item, *Instance] resources ClusterResource } @@ -41,55 +42,113 @@ var ( ErrAvailableTemplateBuilderNotFound = errors.New("available template builder not found") ) -func newCluster(ctx context.Context, tel *telemetry.Client, endpoint string, endpointTLS bool, secret string, clusterID uuid.UUID, sandboxDomain *string) (*Cluster, error) { - clientAuthMiddleware := func(c *api.Client) error { - c.RequestEditors = append( - c.RequestEditors, - func(_ context.Context, req *http.Request) error { - req.Header.Set(consts.EdgeApiAuthHeader, secret) +func newLocalCluster( + ctx context.Context, + tel *telemetry.Client, + nomad *nomadapi.Client, + endpoint string, + secret string, +) (*Cluster, error) { + clusterID := consts.LocalClusterID + + httpClient, err := api.NewClientWithResponses( + fmt.Sprintf("http://%s", endpoint), + func(c *api.Client) error { + c.RequestEditors = append( + c.RequestEditors, + func(_ context.Context, req *http.Request) error { + req.Header.Set(consts.EdgeApiAuthHeader, secret) + + return nil + }, + ) + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create http client: %w", err) + } - return nil - }, - ) + instances := smap.New[*Instance]() + instanceCreation := func(ctx context.Context, item discovery.Item) (*Instance, error) { + // For local cluster we are doing direct connection to instance IP and API port and without additional cluster auth. + return newInstance(ctx, tel, nil, clusterID, item, fmt.Sprintf("%s:%d", item.LocalIPAddress, item.LocalInstanceApiPort), false) + } + + storeDiscovery := discovery.NewLocalDiscovery(clusterID, nomad) + store := instancesSyncStore{clusterID: clusterID, instances: instances, discovery: storeDiscovery, instanceCreation: instanceCreation} + + c := &Cluster{ + ID: clusterID, + SandboxDomain: nil, - return nil + instances: instances, + resources: newRemoteClusterResourceProvider(instances, httpClient), + synchronization: synchronization.NewSynchronize("cluster-instances", "Cluster instances", store), } - // generate the full endpoint URL - var endpointBaseUrl string + // Periodically sync cluster instances + go c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) + + return c, nil +} + +func newRemoteCluster( + ctx context.Context, + tel *telemetry.Client, + endpoint string, + endpointTLS bool, + secret string, + clusterID uuid.UUID, + sandboxDomain *string, +) (*Cluster, error) { + endpointBaseUrl := fmt.Sprintf("http://%s", endpoint) if endpointTLS { endpointBaseUrl = fmt.Sprintf("https://%s", endpoint) - } else { - endpointBaseUrl = fmt.Sprintf("http://%s", endpoint) } - httpClient, err := api.NewClientWithResponses(endpointBaseUrl, clientAuthMiddleware) + httpClient, err := api.NewClientWithResponses( + endpointBaseUrl, + func(c *api.Client) error { + c.RequestEditors = append( + c.RequestEditors, + func(_ context.Context, req *http.Request) error { + req.Header.Set(consts.EdgeApiAuthHeader, secret) + + return nil + }, + ) + + return nil + }, + ) if err != nil { return nil, fmt.Errorf("failed to create http client: %w", err) } instances := smap.New[*Instance]() - instanceCreation := func(ctx context.Context, item api.ClusterOrchestratorNode) (*Instance, error) { - // For remote cluster we are doing Connection to endpoint that works as gRPC proxy and handles auth and routing for us. - auth := &instanceAuthorization{secret: secret, tls: endpointTLS, serviceInstanceID: item.ServiceInstanceID} + instanceCreation := func(ctx context.Context, item discovery.Item) (*Instance, error) { + // For remote cluster we are doing connection to endpoint that works as gRPC proxy and handles auth and routing for us. + auth := &instanceAuthorization{secret: secret, tls: endpointTLS, serviceInstanceID: item.InstanceID} - return newInstance(ctx, tel, auth, clusterID, item.NodeID, item.ServiceInstanceID, endpoint, endpointTLS) + return newInstance(ctx, tel, auth, clusterID, item, endpoint, endpointTLS) } + storeDiscovery := discovery.NewRemoteServiceDiscovery(clusterID, httpClient) + store := instancesSyncStore{clusterID: clusterID, instances: instances, instanceCreation: instanceCreation, discovery: storeDiscovery} + c := &Cluster{ ID: clusterID, SandboxDomain: sandboxDomain, - resources: newRemoteClusterResourceProvider(instances, httpClient), - instances: instances, - httpClient: httpClient, + instances: instances, + resources: newRemoteClusterResourceProvider(instances, httpClient), + synchronization: synchronization.NewSynchronize("cluster-instances", "Cluster instances", store), } - store := instancesSyncStore{cluster: c, instanceCreation: instanceCreation} - c.synchronization = synchronization.NewSynchronize("cluster-instances", "Cluster instances", store) - - // periodically sync cluster instances - go c.startSync(ctx) + // Periodically sync cluster instances + go c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) return c, nil } @@ -165,7 +224,3 @@ func (c *Cluster) GetOrchestrators() []*Instance { func (c *Cluster) GetResources() ClusterResource { return c.resources } - -func (c *Cluster) startSync(ctx context.Context) { - c.synchronization.Start(ctx, instancesSyncInterval, instancesSyncTimeout, true) -} diff --git a/packages/api/internal/clusters/clusters_sync.go b/packages/api/internal/clusters/clusters_sync.go index 55136a97df..0f171f4829 100644 --- a/packages/api/internal/clusters/clusters_sync.go +++ b/packages/api/internal/clusters/clusters_sync.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + nomadapi "github.com/hashicorp/nomad/api" "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/cfg" @@ -49,26 +50,24 @@ func localClusterConfig(clusterEndpoint, clusterToken string) (*queries.Cluster, }, nil } -func NewPool(ctx context.Context, tel *telemetry.Client, db *client.Client, config cfg.Config) (*Pool, error) { - p := &Pool{ - db: db, - tel: tel, - clusters: smap.New[*Cluster](), - } - - // Shutdown function to gracefully close the pool - go func() { - <-ctx.Done() - p.Close(ctx) - }() +func NewPool(ctx context.Context, tel *telemetry.Client, db *client.Client, nomad *nomadapi.Client, config cfg.Config) (*Pool, error) { + clusters := smap.New[*Cluster]() localCluster, err := localClusterConfig(config.LocalClusterEndpoint, config.LocalClusterToken) if err != nil { return nil, err } - store := poolSynchronizationStore{pool: p, localCluster: localCluster} - p.synchronization = synchronization.NewSynchronize("clusters-pool", "Clusters pool", store) + p := &Pool{ + db: db, + tel: tel, + clusters: clusters, + synchronization: synchronization.NewSynchronize( + "clusters-pool", + "Clusters pool", + clustersSyncStore{db: db, tel: tel, clusters: clusters, local: localCluster, nomad: nomad}, + ), + } // Periodically sync clusters with the database go p.synchronization.Start(ctx, poolSyncInterval, poolSyncTimeout, true) @@ -103,13 +102,16 @@ func (p *Pool) Close(ctx context.Context) { } // SynchronizationStore is an interface that defines methods for synchronizing the clusters pool with the database -type poolSynchronizationStore struct { - pool *Pool - localCluster *queries.Cluster +type clustersSyncStore struct { + db *client.Client + tel *telemetry.Client + clusters *smap.Map[*Cluster] + local *queries.Cluster + nomad *nomadapi.Client } -func (d poolSynchronizationStore) SourceList(ctx context.Context) ([]queries.Cluster, error) { - db, err := d.pool.db.GetActiveClusters(ctx) +func (d clustersSyncStore) SourceList(ctx context.Context) ([]queries.Cluster, error) { + db, err := d.db.GetActiveClusters(ctx) if err != nil { return nil, err } @@ -119,15 +121,15 @@ func (d poolSynchronizationStore) SourceList(ctx context.Context) ([]queries.Clu entries = append(entries, row.Cluster) } - // Append local cluster if registered - if d.localCluster != nil { - entries = append(entries, *d.localCluster) + // Append local cluster if provided + if d.local != nil { + entries = append(entries, *d.local) } return entries, nil } -func (d poolSynchronizationStore) SourceExists(_ context.Context, s []queries.Cluster, p *Cluster) bool { +func (d clustersSyncStore) SourceExists(_ context.Context, s []queries.Cluster, p *Cluster) bool { for _, item := range s { if item.ID == p.ID { return true @@ -137,49 +139,61 @@ func (d poolSynchronizationStore) SourceExists(_ context.Context, s []queries.Cl return false } -func (d poolSynchronizationStore) PoolList(_ context.Context) []*Cluster { +func (d clustersSyncStore) PoolList(_ context.Context) []*Cluster { items := make([]*Cluster, 0) - for _, item := range d.pool.clusters.Items() { + for _, item := range d.clusters.Items() { items = append(items, item) } return items } -func (d poolSynchronizationStore) PoolExists(_ context.Context, cluster queries.Cluster) bool { - _, found := d.pool.clusters.Get(cluster.ID.String()) +func (d clustersSyncStore) PoolExists(_ context.Context, cluster queries.Cluster) bool { + _, found := d.clusters.Get(cluster.ID.String()) return found } -func (d poolSynchronizationStore) PoolInsert(ctx context.Context, cluster queries.Cluster) { +func (d clustersSyncStore) PoolInsert(ctx context.Context, cluster queries.Cluster) { clusterID := cluster.ID.String() logger.L().Info(ctx, "Initializing newly discovered cluster", logger.WithClusterID(cluster.ID)) - c, err := newCluster(context.WithoutCancel(ctx), - d.pool.tel, - cluster.Endpoint, - cluster.EndpointTls, - cluster.Token, - cluster.ID, - cluster.SandboxProxyDomain, - ) + var c *Cluster + var err error + + // Local cluster + if cluster.ID == consts.LocalClusterID { + c, err = newLocalCluster(context.WithoutCancel(ctx), d.tel, d.nomad, d.local.Endpoint, d.local.Token) + if err != nil { + logger.L().Error(ctx, "Initializing local cluster failed", zap.Error(err), logger.WithClusterID(cluster.ID)) + + return + } + + d.clusters.Insert(clusterID, c) + logger.L().Info(ctx, "Local cluster initialized successfully", logger.WithClusterID(cluster.ID)) + + return + } + + // Remote cluster + c, err = newRemoteCluster(context.WithoutCancel(ctx), d.tel, cluster.Endpoint, cluster.EndpointTls, cluster.Token, cluster.ID, cluster.SandboxProxyDomain) if err != nil { - logger.L().Error(ctx, "Initializing cluster failed", zap.Error(err), logger.WithClusterID(c.ID)) + logger.L().Error(ctx, "Initializing remote cluster failed", zap.Error(err), logger.WithClusterID(cluster.ID)) return } - logger.L().Info(ctx, "Cluster initialized successfully", logger.WithClusterID(c.ID)) - d.pool.clusters.Insert(clusterID, c) + d.clusters.Insert(clusterID, c) + logger.L().Info(ctx, "Remote cluster initialized successfully", logger.WithClusterID(cluster.ID)) } -func (d poolSynchronizationStore) PoolUpdate(_ context.Context, _ *Cluster) { +func (d clustersSyncStore) PoolUpdate(_ context.Context, _ *Cluster) { // Clusters pool currently does not do something special during synchronization } -func (d poolSynchronizationStore) PoolRemove(ctx context.Context, cluster *Cluster) { +func (d clustersSyncStore) PoolRemove(ctx context.Context, cluster *Cluster) { logger.L().Info(ctx, "Removing cluster from pool", logger.WithClusterID(cluster.ID)) err := cluster.Close() @@ -187,5 +201,5 @@ func (d poolSynchronizationStore) PoolRemove(ctx context.Context, cluster *Clust logger.L().Error(ctx, "Error during removing cluster from pool", zap.Error(err), logger.WithClusterID(cluster.ID)) } - d.pool.clusters.Remove(cluster.ID.String()) + d.clusters.Remove(cluster.ID.String()) } diff --git a/packages/api/internal/clusters/discovery/discovery.go b/packages/api/internal/clusters/discovery/discovery.go new file mode 100644 index 0000000000..eee6c98f59 --- /dev/null +++ b/packages/api/internal/clusters/discovery/discovery.go @@ -0,0 +1,27 @@ +package discovery + +import ( + "context" + + "go.opentelemetry.io/otel" +) + +var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/clusters/discovery") + +type Item struct { + // Identifier that uniquely identifies the instance so it will not be registered multiple times. + UniqueIdentifier string + NodeID string + + // Instance ID that changes on each restart, available only for edge-backend service discovery. + InstanceID string + + // Following fields are available only for local cluster. + // For remote clusters gRPC proxy is used and these fields are not needed. + LocalIPAddress string + LocalInstanceApiPort uint16 +} + +type Discovery interface { + Query(ctx context.Context) ([]Item, error) +} diff --git a/packages/api/internal/clusters/discovery/local.go b/packages/api/internal/clusters/discovery/local.go index 5844159aeb..7b285442a5 100644 --- a/packages/api/internal/clusters/discovery/local.go +++ b/packages/api/internal/clusters/discovery/local.go @@ -1 +1,60 @@ package discovery + +import ( + "context" + "fmt" + + "github.com/google/uuid" + nomadapi "github.com/hashicorp/nomad/api" + "go.opentelemetry.io/otel/trace" + + "github.com/e2b-dev/infra/packages/shared/pkg/clusters/discovery" + "github.com/e2b-dev/infra/packages/shared/pkg/consts" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" +) + +type LocalServiceDiscovery struct { + nomad *nomadapi.Client + clusterID uuid.UUID +} + +func NewLocalDiscovery(clusterID uuid.UUID, nomad *nomadapi.Client) Discovery { + return &LocalServiceDiscovery{ + nomad: nomad, + clusterID: clusterID, + } +} + +func (sd *LocalServiceDiscovery) Query(ctx context.Context) ([]Item, error) { + _, span := tracer.Start(ctx, "query-local-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) + defer span.End() + + // For now, we want to search only for template builders as local orchestrators are still discovered + // old way via Nomad discovery directly inside node manager flow. To minimize changes, we keep it this way for now. + alloc, err := discovery.ListOrchestratorAndTemplateBuilderAllocations(ctx, sd.nomad, discovery.FilterTemplateBuilders) + if err != nil { + span.RecordError(err) + + return nil, fmt.Errorf("failed to list Nomad allocations in service discovery: %w", err) + } + + result := make([]Item, len(alloc)) + for i, v := range alloc { + item := Item{ + UniqueIdentifier: v.AllocationID, + NodeID: v.NodeID, + + // For local discovery it's not supported here, but it will be fetched during service sync + InstanceID: "unknown", + + // For now, we assume ports that are used for gRPC api and proxy are static, + // in future we should be able to take port numbers from Nomad API and map them accordingly here. + LocalIPAddress: v.AllocationIP, + LocalInstanceApiPort: consts.OrchestratorApiPort, + } + + result[i] = item + } + + return result, nil +} diff --git a/packages/api/internal/clusters/discovery/remote.go b/packages/api/internal/clusters/discovery/remote.go new file mode 100644 index 0000000000..95fc22ab52 --- /dev/null +++ b/packages/api/internal/clusters/discovery/remote.go @@ -0,0 +1,56 @@ +package discovery + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/google/uuid" + "go.opentelemetry.io/otel/trace" + + api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" +) + +type RemoteServiceDiscovery struct { + clusterID uuid.UUID + client *api.ClientWithResponses +} + +func NewRemoteServiceDiscovery(clusterID uuid.UUID, client *api.ClientWithResponses) Discovery { + return &RemoteServiceDiscovery{ + clusterID: clusterID, + client: client, + } +} + +func (sd *RemoteServiceDiscovery) Query(ctx context.Context) ([]Item, error) { + _, span := tracer.Start(ctx, "query-remote-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) + defer span.End() + + res, err := sd.client.V1ServiceDiscoveryGetOrchestratorsWithResponse(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get cluster instances from service discovery: %w", err) + } + + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("failed to get nodes from edge api: %s", res.Status()) + } + + if res.JSON200 == nil { + return nil, errors.New("request to get nodes returned nil response") + } + + nodes := *res.JSON200 + result := make([]Item, len(nodes)) + for i, n := range nodes { + result[i] = Item{ + UniqueIdentifier: n.ServiceInstanceID, + NodeID: n.NodeID, + InstanceID: n.ServiceInstanceID, + } + } + + return result, nil +} diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index bdf529c5e1..0691541487 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -10,6 +10,7 @@ import ( "go.uber.org/zap" "google.golang.org/protobuf/types/known/emptypb" + "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" "github.com/e2b-dev/infra/packages/api/internal/utils" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -18,6 +19,10 @@ import ( ) type Instance struct { + // Identifier that uniquely identifies the instance so it will not be registered multiple times. + // Depending on service discovery used, it can be combination of different parameters, what service discovery gives us. + UniqueIdentifier string + ClusterID uuid.UUID NodeID string InstanceID string @@ -38,8 +43,7 @@ func newInstance( tel *telemetry.Client, clusterAuth *instanceAuthorization, clusterID uuid.UUID, - nodeID string, - instanceID string, + sd discovery.Item, connAddr string, connTls bool, ) (*Instance, error) { @@ -54,9 +58,10 @@ func newInstance( // For case with local cluster we will not receive instance ID from service discovery, but its not needed for proxy routing, // so it can be empty and will be filled after first sync. i := &Instance{ - NodeID: nodeID, - ClusterID: clusterID, - InstanceID: instanceID, + UniqueIdentifier: sd.UniqueIdentifier, + NodeID: sd.NodeID, + InstanceID: sd.InstanceID, + ClusterID: clusterID, grpc: conn, mutex: sync.RWMutex{}, diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index f44108bd52..5cfcf2f970 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -2,45 +2,46 @@ package clusters import ( "context" - "errors" "fmt" - "net/http" + "github.com/google/uuid" "go.uber.org/zap" - api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" + "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/smap" ) // SynchronizationStore defines methods for synchronizing cluster instances type instancesSyncStore struct { - cluster *Cluster - instanceCreation func(ctx context.Context, item api.ClusterOrchestratorNode) (*Instance, error) + clusterID uuid.UUID + + discovery discovery.Discovery + instances *smap.Map[*Instance] + instanceCreation func(ctx context.Context, item discovery.Item) (*Instance, error) } -func (d instancesSyncStore) SourceList(ctx context.Context) ([]api.ClusterOrchestratorNode, error) { - // fetch cluster instances with use of service discovery - res, err := d.cluster.httpClient.V1ServiceDiscoveryGetOrchestratorsWithResponse(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get cluster instances from service discovery: %w", err) - } +func (d instancesSyncStore) SourceList(ctx context.Context) ([]discovery.Item, error) { + // Disable discovery for local environments + if env.IsLocal() { + logger.L().Debug(ctx, "Service discovery is disabled in local environment") - if res.StatusCode() != http.StatusOK { - return nil, fmt.Errorf("failed to get builders from api: %s", res.Status()) + return []discovery.Item{}, nil } - if res.JSON200 == nil { - return nil, errors.New("request to get builders returned nil response") + items, err := d.discovery.Query(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get cluster instances from service discovery: %w", err) } - return *res.JSON200, nil + return items, nil } -func (d instancesSyncStore) SourceExists(_ context.Context, s []api.ClusterOrchestratorNode, p *Instance) bool { +func (d instancesSyncStore) SourceExists(_ context.Context, s []discovery.Item, p *Instance) bool { for _, item := range s { - // With comparing service instance ID we ensure when orchestrator on same node and node ID is still same - // we will properly clean up old instance and later register as new one - if item.ServiceInstanceID == p.InstanceID { + // With comparing unique identifier that should ensure we are not re-adding same instance again + if item.UniqueIdentifier == p.UniqueIdentifier { return true } } @@ -50,33 +51,33 @@ func (d instancesSyncStore) SourceExists(_ context.Context, s []api.ClusterOrche func (d instancesSyncStore) PoolList(_ context.Context) []*Instance { mapped := make([]*Instance, 0) - for _, item := range d.cluster.instances.Items() { + for _, item := range d.instances.Items() { mapped = append(mapped, item) } return mapped } -func (d instancesSyncStore) PoolExists(_ context.Context, s api.ClusterOrchestratorNode) bool { - _, found := d.cluster.instances.Get(s.NodeID) +func (d instancesSyncStore) PoolExists(_ context.Context, s discovery.Item) bool { + _, found := d.instances.Get(s.NodeID) return found } -func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrchestratorNode) { +func (d instancesSyncStore) PoolInsert(ctx context.Context, item discovery.Item) { logger.L().Info(ctx, "Adding instance into cluster pool", - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(item.NodeID), - logger.WithServiceInstanceID(item.ServiceInstanceID), + logger.WithServiceInstanceID(item.InstanceID), ) instance, err := d.instanceCreation(ctx, item) if err != nil { logger.L().Error(ctx, "Failed to create cluster instance during pool insert", zap.Error(err), - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(item.NodeID), - logger.WithServiceInstanceID(item.ServiceInstanceID), + logger.WithServiceInstanceID(item.InstanceID), ) return @@ -88,7 +89,7 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrch if closeErr != nil { logger.L().Error(ctx, "Failed to close cluster instance after sync failure", zap.Error(closeErr), - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), logger.WithServiceInstanceID(instance.InstanceID), ) @@ -96,7 +97,7 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrch logger.L().Error(ctx, "Failed to sync cluster instance during pool insert", zap.Error(err), - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), logger.WithServiceInstanceID(instance.InstanceID), ) @@ -104,7 +105,7 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item api.ClusterOrch return } - d.cluster.instances.Insert(item.NodeID, instance) + d.instances.Insert(item.NodeID, instance) } func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) { @@ -112,7 +113,7 @@ func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) if err != nil { logger.L().Error(ctx, "Failed to sync cluster instance during pool update", zap.Error(err), - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), logger.WithServiceInstanceID(instance.InstanceID), ) @@ -121,9 +122,10 @@ func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) { logger.L().Info(ctx, "Removing instance from cluster pool", - logger.WithClusterID(d.cluster.ID), + logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), logger.WithServiceInstanceID(instance.InstanceID), ) - d.cluster.instances.Remove(instance.NodeID) + + d.instances.Remove(instance.NodeID) } diff --git a/packages/api/internal/handlers/sandbox_get.go b/packages/api/internal/handlers/sandbox_get.go index 7d7136e1fb..e6586e535e 100644 --- a/packages/api/internal/handlers/sandbox_get.go +++ b/packages/api/internal/handlers/sandbox_get.go @@ -29,7 +29,7 @@ func (a *APIStore) GetSandboxesSandboxID(c *gin.Context, id string) { var sbxDomain *string if team.ClusterID != nil { - cluster, ok := a.clustersPool.GetClusterById(*team.ClusterID) + cluster, ok := a.clusters.GetClusterById(*team.ClusterID) if !ok { telemetry.ReportCriticalError(ctx, fmt.Sprintf("cluster with ID '%s' not found", *team.ClusterID), nil) a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("cluster with id %s not found", *team.ClusterID)) diff --git a/packages/api/internal/handlers/sandbox_logs.go b/packages/api/internal/handlers/sandbox_logs.go index 16ec37b683..39c1ff693f 100644 --- a/packages/api/internal/handlers/sandbox_logs.go +++ b/packages/api/internal/handlers/sandbox_logs.go @@ -26,7 +26,7 @@ func (a *APIStore) GetSandboxesSandboxIDLogs(c *gin.Context, sandboxID string, p ) clusterID := utils.WithClusterFallback(team.ClusterID) - cluster, ok := a.clustersPool.GetClusterById(clusterID) + cluster, ok := a.clusters.GetClusterById(clusterID) if !ok { telemetry.ReportCriticalError(ctx, "error getting cluster by ID", fmt.Errorf("cluster with ID '%s' not found", clusterID)) a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error getting cluster '%s'", clusterID)) diff --git a/packages/api/internal/handlers/sandbox_metrics.go b/packages/api/internal/handlers/sandbox_metrics.go index 16c0ffe6eb..c345ac44dd 100644 --- a/packages/api/internal/handlers/sandbox_metrics.go +++ b/packages/api/internal/handlers/sandbox_metrics.go @@ -46,7 +46,7 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(c *gin.Context, sandboxID string } clusterID := utils.WithClusterFallback(team.ClusterID) - cluster, found := a.clustersPool.GetClusterById(clusterID) + cluster, found := a.clusters.GetClusterById(clusterID) if !found { logger.L().Error(ctx, "cluster not found for sandbox metrics", logger.WithClusterID(clusterID)) a.sendAPIStoreError(c, http.StatusInternalServerError, "cluster not found for sandbox metrics") diff --git a/packages/api/internal/handlers/sandboxes_list_metrics.go b/packages/api/internal/handlers/sandboxes_list_metrics.go index 4c2d5fb28c..53e879f3b4 100644 --- a/packages/api/internal/handlers/sandboxes_list_metrics.go +++ b/packages/api/internal/handlers/sandboxes_list_metrics.go @@ -49,7 +49,7 @@ func (a *APIStore) getSandboxesMetrics( return make(map[string]api.SandboxMetric), nil } - cluster, found := a.clustersPool.GetClusterById(clusterID) + cluster, found := a.clusters.GetClusterById(clusterID) if !found { return nil, &api.APIError{ Code: http.StatusInternalServerError, diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 5a53d4ef3c..08a728d06d 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -58,7 +58,7 @@ type APIStore struct { clickhouseStore clickhouse.Clickhouse accessTokenGenerator *sandbox.AccessTokenGenerator featureFlags *featureflags.Client - clustersPool *clusters.Pool + clusters *clusters.Pool } func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) *APIStore { @@ -110,7 +110,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) } } - clustersPool, err := clusters.NewPool(ctx, tel, sqlcDB, config) + clusters, err := clusters.NewPool(ctx, tel, sqlcDB, nomadClient, config) if err != nil { logger.L().Fatal(ctx, "initializing edge clusters pool failed", zap.Error(err)) } @@ -125,7 +125,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) logger.L().Fatal(ctx, "Initializing access token generator failed", zap.Error(err)) } - orch, err := orchestrator.New(ctx, config, tel, nomadClient, posthogClient, redisClient, sqlcDB, clustersPool, featureFlags, accessTokenGenerator) + orch, err := orchestrator.New(ctx, config, tel, nomadClient, posthogClient, redisClient, sqlcDB, clusters, featureFlags, accessTokenGenerator) if err != nil { logger.L().Fatal(ctx, "Initializing Orchestrator client", zap.Error(err)) } @@ -135,7 +135,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) templateSpawnCounter := utils.NewTemplateSpawnCounter(ctx, time.Minute, sqlcDB) templateBuildsCache := templatecache.NewTemplateBuildCache(sqlcDB) - templateManager, err := template_manager.New(sqlcDB, clustersPool, templateBuildsCache, templateCache, featureFlags) + templateManager, err := template_manager.New(sqlcDB, clusters, templateBuildsCache, templateCache, featureFlags) if err != nil { logger.L().Fatal(ctx, "Initializing Template manager client", zap.Error(err)) } @@ -157,7 +157,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) templateSpawnCounter: templateSpawnCounter, clickhouseStore: clickhouseStore, accessTokenGenerator: accessTokenGenerator, - clustersPool: clustersPool, + clusters: clusters, featureFlags: featureFlags, redisClient: redisClient, } @@ -207,6 +207,8 @@ func (a *APIStore) Close(ctx context.Context) error { } } + a.clusters.Close(ctx) + if err := a.sqlcDB.Close(); err != nil { errs = append(errs, fmt.Errorf("closing sqlc database client: %w", err)) } diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 40bb37dc96..b42f368b0e 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -68,7 +68,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDLogs(c *gin.Context, templ return } - cluster, ok := a.clustersPool.GetClusterById(utils.WithClusterFallback(team.ClusterID)) + cluster, ok := a.clusters.GetClusterById(utils.WithClusterFallback(team.ClusterID)) if !ok { telemetry.ReportError(ctx, "error when getting cluster", fmt.Errorf("cluster with ID '%s' not found", team.ClusterID)) a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when getting cluster") diff --git a/packages/api/internal/handlers/template_build_status.go b/packages/api/internal/handlers/template_build_status.go index 2587b78588..70609807d8 100644 --- a/packages/api/internal/handlers/template_build_status.go +++ b/packages/api/internal/handlers/template_build_status.go @@ -106,7 +106,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } - cluster, ok := a.clustersPool.GetClusterById(utils.WithClusterFallback(team.ClusterID)) + cluster, ok := a.clusters.GetClusterById(utils.WithClusterFallback(team.ClusterID)) if !ok { telemetry.ReportError(ctx, "error when getting cluster", fmt.Errorf("cluster with ID '%s' not found", team.ClusterID)) a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when getting cluster") diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index ad340af9f0..c18b5e4ad9 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -79,7 +79,7 @@ func (o *Orchestrator) listNomadNodes(ctx context.Context) ([]nodemanager.NomadS for _, n := range nomadNodes { result = append(result, nodemanager.NomadServiceDiscovery{ NomadNodeShortID: n.ID[:consts.NodeIDLength], - OrchestratorAddress: fmt.Sprintf("%s:%s", n.Address, consts.OrchestratorPort), + OrchestratorAddress: fmt.Sprintf("%s:%d", n.Address, consts.OrchestratorApiPort), IPAddress: n.Address, }) } diff --git a/packages/shared/pkg/consts/sandboxes.go b/packages/shared/pkg/consts/sandboxes.go index 9377ea6bdd..bab61351cb 100644 --- a/packages/shared/pkg/consts/sandboxes.go +++ b/packages/shared/pkg/consts/sandboxes.go @@ -1,6 +1,11 @@ package consts -import "os" +import ( + "strconv" + + "github.com/e2b-dev/infra/packages/shared/pkg/env" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" +) const NodeIDLength = 8 @@ -9,4 +14,4 @@ const NodeIDLength = 8 // We don't want to use some obviously dummy value such as empty zeros, because for users it will look like something is wrong with the sandbox id const ClientID = "6532622b" -var OrchestratorPort = os.Getenv("ORCHESTRATOR_PORT") +var OrchestratorApiPort = uint16(utils.Must(strconv.ParseUint(env.GetEnv("ORCHESTRATOR_PORT", "5008"), 16, 16))) From aa207bc58685d48fd051a335f3148029667e7e6c Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 15:53:58 +0100 Subject: [PATCH 19/58] Speedup clusters sync --- packages/api/internal/clusters/clusters_sync.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/clusters/clusters_sync.go b/packages/api/internal/clusters/clusters_sync.go index 0f171f4829..8f9991d95b 100644 --- a/packages/api/internal/clusters/clusters_sync.go +++ b/packages/api/internal/clusters/clusters_sync.go @@ -21,8 +21,8 @@ import ( ) const ( - poolSyncInterval = 60 * time.Second - poolSyncTimeout = 15 * time.Second + clustersSyncInterval = 15 * time.Second + clusterSyncTimeout = 5 * time.Second ) type Pool struct { @@ -70,7 +70,7 @@ func NewPool(ctx context.Context, tel *telemetry.Client, db *client.Client, noma } // Periodically sync clusters with the database - go p.synchronization.Start(ctx, poolSyncInterval, poolSyncTimeout, true) + go p.synchronization.Start(ctx, clustersSyncInterval, clusterSyncTimeout, true) return p, nil } From 5c48b0a67763b072487b4210b45d7fbe66e4890d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 15:54:04 +0100 Subject: [PATCH 20/58] Fixes --- packages/api/internal/clusters/instance_client.go | 5 ++++- packages/shared/pkg/consts/sandboxes.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/clusters/instance_client.go b/packages/api/internal/clusters/instance_client.go index d454426559..ac7a6f5de4 100644 --- a/packages/api/internal/clusters/instance_client.go +++ b/packages/api/internal/clusters/instance_client.go @@ -35,7 +35,6 @@ func (a instanceAuthorization) RequireTransportSecurity() bool { func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*GRPCClient, error) { grpcOptions := []grpc.DialOption{ - grpc.WithPerRPCCredentials(auth), grpc.WithStatsHandler( otelgrpc.NewClientHandler( otelgrpc.WithTracerProvider(tel.TracerProvider), @@ -51,6 +50,10 @@ func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoi ), } + if auth != nil { + grpcOptions = append(grpcOptions, grpc.WithPerRPCCredentials(auth)) + } + if endpointTLS { // (2025-06) AWS ALB with TLS termination is using TLS 1.2 as default so this is why we are not using TLS 1.3+ here cred := credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) diff --git a/packages/shared/pkg/consts/sandboxes.go b/packages/shared/pkg/consts/sandboxes.go index bab61351cb..a484c18082 100644 --- a/packages/shared/pkg/consts/sandboxes.go +++ b/packages/shared/pkg/consts/sandboxes.go @@ -14,4 +14,4 @@ const NodeIDLength = 8 // We don't want to use some obviously dummy value such as empty zeros, because for users it will look like something is wrong with the sandbox id const ClientID = "6532622b" -var OrchestratorApiPort = uint16(utils.Must(strconv.ParseUint(env.GetEnv("ORCHESTRATOR_PORT", "5008"), 16, 16))) +var OrchestratorApiPort = uint16(utils.Must(strconv.ParseUint(env.GetEnv("ORCHESTRATOR_PORT", "5008"), 10, 16))) From bdf3ab552225b8f7f9c5f4510b11c4a43267e180 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 16:16:02 +0100 Subject: [PATCH 21/58] Implemented local resource provider Now we have two different providers for sandbox logs/metrics and build logs. This way we are no longer using edge api for local cluster for nor sbx/build data query or service discovery. --- iac/provider-gcp/nomad/jobs/api.hcl | 4 +- iac/provider-gcp/nomad/main.tf | 7 +- packages/api/go.mod | 91 ++- packages/api/go.sum | 749 ++++++++++++++++++ packages/api/internal/cfg/model.go | 5 +- packages/api/internal/cfg/model_test.go | 1 + packages/api/internal/clusters/cluster.go | 27 +- .../api/internal/clusters/clusters_sync.go | 50 +- .../api/internal/clusters/resources_local.go | 225 ++++++ ...{resources_impl.go => resources_remote.go} | 0 packages/api/internal/handlers/store.go | 8 +- packages/clickhouse/pkg/utils/sandbox.go | 47 ++ .../internal/edge/handlers/sandbox-metrics.go | 45 +- .../internal/edge/handlers/store.go | 6 +- .../internal/edge/logger-provider/provider.go | 20 - .../pkg/logs/{logsloki => loki}/loki.go | 2 +- .../pkg/logs/loki/provider.go} | 16 +- 17 files changed, 1175 insertions(+), 128 deletions(-) create mode 100644 packages/api/internal/clusters/resources_local.go rename packages/api/internal/clusters/{resources_impl.go => resources_remote.go} (100%) create mode 100644 packages/clickhouse/pkg/utils/sandbox.go delete mode 100644 packages/client-proxy/internal/edge/logger-provider/provider.go rename packages/shared/pkg/logs/{logsloki => loki}/loki.go (99%) rename packages/{client-proxy/internal/edge/logger-provider/provider_loki.go => shared/pkg/logs/loki/provider.go} (87%) diff --git a/iac/provider-gcp/nomad/jobs/api.hcl b/iac/provider-gcp/nomad/jobs/api.hcl index fb3a613859..671c3a67f7 100644 --- a/iac/provider-gcp/nomad/jobs/api.hcl +++ b/iac/provider-gcp/nomad/jobs/api.hcl @@ -81,6 +81,7 @@ job "api" { ORCHESTRATOR_PORT = "${orchestrator_port}" POSTGRES_CONNECTION_STRING = "${postgres_connection_string}" SUPABASE_JWT_SECRETS = "${supabase_jwt_secrets}" + LOKI_URL = "${loki_url}" CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}" ENVIRONMENT = "${environment}" POSTHOG_API_KEY = "${posthog_api_key}" @@ -96,9 +97,6 @@ job "api" { REDIS_TLS_CA_BASE64 = "${redis_tls_ca_base64}" SANDBOX_ACCESS_TOKEN_HASH_SEED = "${sandbox_access_token_hash_seed}" - LOCAL_CLUSTER_ENDPOINT = "${local_cluster_endpoint}" - LOCAL_CLUSTER_TOKEN = "${local_cluster_token}" - %{ if launch_darkly_api_key != "" } LAUNCH_DARKLY_API_KEY = "${launch_darkly_api_key}" %{ endif } diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index df8740a42c..6b6e621d19 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -2,6 +2,7 @@ locals { clickhouse_connection_string = var.clickhouse_server_count > 0 ? "clickhouse://${var.clickhouse_username}:${random_password.clickhouse_password.result}@clickhouse.service.consul:${var.clickhouse_server_port.port}/${var.clickhouse_database}" : "" redis_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) == "" ? "redis.service.consul:${var.redis_port.port}" : "" redis_cluster_url = trimspace(data.google_secret_manager_secret_version.redis_cluster_url.secret_data) + loki_url = "http://loki.service.consul:${var.loki_service_port.port}" } # API @@ -98,12 +99,10 @@ resource "nomad_job" "api" { redis_cluster_url = local.redis_cluster_url redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) clickhouse_connection_string = local.clickhouse_connection_string + loki_url = local.loki_url sandbox_access_token_hash_seed = var.sandbox_access_token_hash_seed db_migrator_docker_image = data.google_artifact_registry_docker_image.db_migrator_image.self_link launch_darkly_api_key = trimspace(data.google_secret_manager_secret_version.launch_darkly_api_key.secret_data) - - local_cluster_endpoint = "edge-api.service.consul:${var.edge_api_port.port}" - local_cluster_token = var.edge_api_secret }) } @@ -157,7 +156,7 @@ resource "nomad_job" "client_proxy" { redis_cluster_url = local.redis_cluster_url redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) - loki_url = "http://loki.service.consul:${var.loki_service_port.port}" + loki_url = local.loki_url clickhouse_connection_string = local.clickhouse_connection_string proxy_port_name = var.edge_proxy_port.name diff --git a/packages/api/go.mod b/packages/api/go.mod index 36e26d9a54..0bd42f711a 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -45,6 +45,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.2 github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 + github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843 github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c github.com/jackc/pgx/v5 v5.7.4 github.com/jellydator/ttlcache/v3 v3.4.0 @@ -73,20 +74,35 @@ require ( require ( dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/ClickHouse/ch-go v0.66.1 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.37.2 // indirect github.com/DataDog/datadog-go/v5 v5.2.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Workiva/go-datastructures v1.1.0 // indirect github.com/air-verse/air v1.61.7 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.55.7 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bep/godartsass/v2 v2.3.2 // indirect github.com/bep/golibsass v1.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cli/safeexec v1.0.1 // indirect github.com/cloudwego/base64x v0.1.5 // indirect @@ -94,18 +110,24 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/creack/pty v1.1.23 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/uniuri v1.2.0 // indirect + github.com/dennwc/varint v1.0.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v28.3.3+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.4 // indirect + github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/exaring/otelpgx v0.9.3 // indirect + github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect @@ -114,38 +136,68 @@ require ( github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gohugoio/hugo v0.139.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/nftables v0.3.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grafana/dskit v0.0.0-20231120170505-765e343eda4f // indirect + github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 // indirect + github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/hashicorp/consul/api v1.30.0 // indirect github.com/hashicorp/cronexpr v1.1.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/memberlist v0.5.0 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/huandu/xstrings v1.4.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/launchdarkly/ccache v1.1.0 // indirect github.com/launchdarkly/eventsource v1.10.0 // indirect github.com/launchdarkly/go-jsonstream/v3 v3.1.0 // indirect @@ -163,8 +215,11 @@ require ( github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect github.com/mdlayher/socket v0.5.1 // indirect github.com/mfridman/interpolate v0.0.2 // indirect + github.com/miekg/dns v1.1.63 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect github.com/moby/patternmatcher v0.6.0 // indirect @@ -176,27 +231,46 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/ngrok/firewall_toolkit v0.0.18 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect + github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/alertmanager v0.26.0 // indirect + github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common/sigv4 v0.1.0 // indirect + github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/prometheus/prometheus v1.8.2-0.20200727090838-6f296594a852 // indirect github.com/rs/zerolog v1.34.0 // indirect + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/sony/gobreaker v0.5.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/speakeasy-api/openapi-overlay v0.9.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.8.0 // indirect @@ -207,10 +281,20 @@ require ( github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.9.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect + github.com/willf/bitset v1.1.11 // indirect + github.com/willf/bloom v2.0.3+incompatible // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/v3 v3.5.10 // indirect + go.mongodb.org/mongo-driver v1.17.1 // indirect go.opentelemetry.io/auto/sdk v1.2.0 // indirect + go.opentelemetry.io/collector/pdata v1.0.0-rcv0015 // indirect + go.opentelemetry.io/collector/semconv v0.81.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.13.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect @@ -220,14 +304,19 @@ require ( go.opentelemetry.io/otel/sdk v1.38.0 // indirect go.opentelemetry.io/otel/sdk/log v0.14.0 // indirect go.opentelemetry.io/proto/otlp v1.8.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect golang.org/x/arch v0.16.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect golang.org/x/image v0.25.0 // indirect golang.org/x/mod v0.29.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250908214217-97024824d090 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index af98605c93..822fd39c42 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -1,34 +1,134 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69 h1:+tu3HOoMXB7RXEINRVIpxJCT+KdYiI7LAEAUrOw3dIU= github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69/go.mod h1:L1AbZdiDllfyYH5l5OkAaZtk7VkWe89bPJFmnDBNHxg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.66.1 h1:LQHFslfVYZsISOY0dnOYOXGkOUvpv376CCm8g7W74A4= github.com/ClickHouse/ch-go v0.66.1/go.mod h1:NEYcg3aOFv2EmTJfo4m2WF7sHB/YFbLUuIWv9iq76xY= github.com/ClickHouse/clickhouse-go/v2 v2.37.2 h1:wRLNKoynvHQEN4znnVHNLaYnrqVc9sGJmGYg+GGCfto= github.com/ClickHouse/clickhouse-go/v2 v2.37.2/go.mod h1:pH2zrBGp5Y438DMwAxXMm1neSXPPjSI7tD4MURVULw8= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go/v5 v5.2.0 h1:kSptqUGSNK67DgA+By3rwtFnAh6pTBxJ7Hn8JCLZcKY= github.com/DataDog/datadog-go/v5 v5.2.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.6 h1:U68crOE3y3MPttCMQGywZOLrTeF5HHJ3/vDBCJn9/bA= +github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/Workiva/go-datastructures v1.1.0 h1:hu20UpgZneBhQ3ZvwiOGlqJSKIosin2Rd5wAKUHEO/k= +github.com/Workiva/go-datastructures v1.1.0/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/air-verse/air v1.61.7 h1:MtOZs6wYoYYXm+S4e+ORjkq9BjvyEamKJsHcvko8LrQ= github.com/air-verse/air v1.61.7/go.mod h1:QW4HkIASdtSnwaYof1zgJCSxd41ebvix10t5ubtm9cg= github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= +github.com/alicebob/miniredis/v2 v2.30.4 h1:8S4/o1/KoUArAGbGwPxcwf0krlzceva2XVOSchFS7Eo= +github.com/alicebob/miniredis/v2 v2.30.4/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M= github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= +github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/clocks v0.5.0 h1:hhvKVGLPQWRVsBP/UB7ErrHYIO42gINVbvqxvYTPVps= github.com/bep/clocks v0.5.0/go.mod h1:SUq3q+OOq41y2lRQqH5fsOoxN8GbxSiT6jvoVVLCVhU= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -53,6 +153,7 @@ github.com/bep/overlayfs v0.9.2 h1:qJEmFInsW12L7WW7dOTUhnMfyk/fN9OCDEO5Gr8HSDs= github.com/bep/overlayfs v0.9.2/go.mod h1:aYY9W7aXQsGcA7V9x/pzeR8LjEgIxbtisZm8Q7zPz40= github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= @@ -60,29 +161,43 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw= github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/cli/safeexec v1.0.1 h1:e/C79PbXF4yYTN/wauC4tviMxEV13BwljGj0N9j+N00= github.com/cli/safeexec v1.0.1/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -91,7 +206,11 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -103,8 +222,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v1.2.0 h1:koIcOUdrTIivZgSLhHQvKgqdWZq5d7KdMEWF1Ud6+5g= github.com/dchest/uniuri v1.2.0/go.mod h1:fSzm4SLHzNZvWLvWJew423PhAzkpNQYq+uNLq4kxhkY= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/digitalocean/godo v1.99.0 h1:gUHO7n9bDaZFWvbzOum4bXE0/09ZuYA9yA8idQHX57E= +github.com/digitalocean/godo v1.99.0/go.mod h1:SsS2oXo2rznfM/nORlZ/6JaUJZFhmKTib1YhopUc8NA= github.com/disintegration/gift v1.2.1 h1:Y005a1X4Z7Uc+0gLpSAsKhWi4qLtsdEcMIbbdvdZ6pc= github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dUg3/RlKGr4HI= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -124,10 +247,28 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/evanw/esbuild v0.24.0 h1:GZ78naTLp7FKr+K7eNuM/SLs5maeiHYRPsTg6kmdsSE= github.com/evanw/esbuild v0.24.0/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= github.com/exaring/otelpgx v0.9.3 h1:4yO02tXC7ZJZ+hcqcUkfxblYNCIFGVhpUWI0iw1TzPU= github.com/exaring/otelpgx v0.9.3/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= +github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -161,6 +302,19 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -169,10 +323,24 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -181,9 +349,16 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= +github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -194,6 +369,7 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -215,67 +391,178 @@ github.com/gohugoio/locales v0.14.0 h1:Q0gpsZwfv7ATHMbcTNepFd59H7GoykzWJIxi113XG github.com/gohugoio/locales v0.14.0/go.mod h1:ip8cCAv/cnmVLzzXtiTpPwgJ4xhKZranqNqtoIu0b/4= github.com/gohugoio/localescompressed v1.0.1 h1:KTYMi8fCWYLswFyJAeOtuk/EkXR/KPTHHNN9OS+RTxo= github.com/gohugoio/localescompressed v1.0.1/go.mod h1:jBF6q8D7a0vaEmcWPNcAjUZLJaIVNiwvM3WlmTvooB0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= +github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/dskit v0.0.0-20231120170505-765e343eda4f h1:gyojr97YeWZ70pKNakWv5/tKwBHuLy3icnIeCo9gQr4= +github.com/grafana/dskit v0.0.0-20231120170505-765e343eda4f/go.mod h1:8dsy5tQOkeNQyjXpm5mQsbCu3H5uzeBD35MzRQFznKU= +github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 h1:/of8Z8taCPftShATouOrBVy6GaTTjgQd/VfNiZp/VXQ= +github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= +github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843 h1:vKYSCRnLxZWH67ymOvoqliU0nxR1T2CHaqrkaj9P0jc= +github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843/go.mod h1:wWT/NUzFaxBzS1R8pnvUvCwUf9n+InyBkV9s24Nk7BU= +github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 h1:ZYk42718kSXOiIKdjZKljWLgBpzL5z1yutKABksQCMg= +github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608/go.mod h1:f3JSoxBTPXX5ec4FxxeC19nTBSxoTz+cBgS3cYLMcr0= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hairyhenderson/go-codeowners v0.6.1 h1:2OLPpLWFMxkCf9hkYzOexnCGD+kj853OqeoKq7S+9us= github.com/hairyhenderson/go-codeowners v0.6.1/go.mod h1:RFWbGcjlXhRKNezt7AQHmJucY0alk4osN0+RKOsIAa8= +github.com/hashicorp/consul/api v1.30.0 h1:ArHVMMILb1nQv8vZSGIwwQd2gtc+oSQZ6CalyiyH2XQ= +github.com/hashicorp/consul/api v1.30.0/go.mod h1:B2uGchvaXVW2JhFoS8nqTxMD5PBykr4ebY4JWHTTeLM= +github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg= +github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s= github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c h1:7QPAGE7GJfSqaSvr7/mKOGYwohz3/dfs5nt/cu0sdyI= github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c/go.mod h1:ijDwa6o1uG1jFSq6kERiX2PamKGpZzTmo0XOFNeFZgw= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hetznercloud/hcloud-go/v2 v2.0.0 h1:Sg1DJ+MAKvbYAqaBaq9tPbwXBS2ckPIaMtVdUjKu+4g= +github.com/hetznercloud/hcloud-go/v2 v2.0.0/go.mod h1:4iUG2NG8b61IAwNx6UsMWQ6IfIf/i1RsG0BbsKAyR5Q= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/ionos-cloud/sdk-go/v6 v6.1.8 h1:493wE/BkZxJf7x79UCE0cYGPZoqQcPiEBALvt7uVGY0= +github.com/ionos-cloud/sdk-go/v6 v6.1.8/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -288,11 +575,26 @@ github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -305,6 +607,11 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -312,6 +619,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U= github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/launchdarkly/ccache v1.1.0 h1:voD1M+ZJXR3MREOKtBwgTF9hYHl1jg+vFKS/+VAkR2k= @@ -336,6 +645,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linode/linodego v1.19.0 h1:n4WJrcr9+30e9JGZ6DI0nZbm5SdAj1kSwvvt/998YUw= +github.com/linode/linodego v1.19.0/go.mod h1:XZFR+yJ9mm2kwf6itZ6SCpu+6w3KnIevV0Uu5HNWJgQ= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= @@ -346,15 +657,26 @@ github.com/makeworld-the-better-one/dither/v2 v2.4.0 h1:Az/dYXiTcwcRSe59Hzw4RI1r github.com/makeworld-the-better-one/dither/v2 v2.4.0/go.mod h1:VBtN8DXO7SNtyGmLiGA7IsFeKrBkQPze1/iAeM95arc= github.com/marekm4/color-extractor v1.2.1 h1:3Zb2tQsn6bITZ8MBVhc33Qn1k5/SEuZ18mrXGUqIwn0= github.com/marekm4/color-extractor v1.2.1/go.mod h1:90VjmiHI6M8ez9eYUaXLdcKnS+BAOp7w+NpwBdkJmpA= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= @@ -363,12 +685,26 @@ github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= @@ -388,6 +724,8 @@ github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFL github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= @@ -397,6 +735,11 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/muesli/smartcrop v0.3.0 h1:JTlSkmxWg/oQ1TcLDoypuirdE8Y/jzNirQeLkxpA6Oc= github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ngrok/firewall_toolkit v0.0.18 h1:/+Rx/5qXXO8FpOoKpPnyR2nw8Y3KumuulSNZa3XGZE8= @@ -416,6 +759,8 @@ github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//J github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -436,8 +781,20 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg= +github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= +github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= +github.com/ovh/go-ovh v1.4.1 h1:VBGa5wMyQtTP7Zb+w97zRCh9sLtM/2YKRyy+MEJmWaM= +github.com/ovh/go-ovh v1.4.1/go.mod h1:6bL6pPyUT7tBfI0pqOegJgRjgjuO+mOo+MyXd1EEC0M= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= @@ -451,33 +808,88 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/posthog/posthog-go v0.0.0-20230801140217-d607812dee69 h1:01dHVodha5BzrMtVmcpPeA4VYbZEsTXQ6m4123zQXJk= github.com/posthog/posthog-go v0.0.0-20230801140217-d607812dee69/go.mod h1:migYMxlAqcnQy+3eN8mcL0b2tpKy6R+8Zc0lxwk4dKM= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pressly/goose/v3 v3.24.2 h1:c/ie0Gm8rnIVKvnDQ/scHErv46jrDv9b4I0WRcFJzYU= github.com/pressly/goose/v3 v3.24.2/go.mod h1:kjefwFB0eR4w30Td2Gj2Mznyw94vSP+2jJYkOVNbD1k= +github.com/prometheus/alertmanager v0.26.0 h1:uOMJWfIwJguc3NaM3appWNbbrh6G/OjvaHMk22aBBYc= +github.com/prometheus/alertmanager v0.26.0/go.mod h1:rVcnARltVjavgVaNnmevxK7kOn7IZavyf0KNgHkbEpU= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= +github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= +github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 h1:oHcfzdJnM/SFppy2aUlvomk37GI33x9vgJULihE5Dt8= +github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97/go.mod h1:LoBCZeRh+5hX+fSULNyFnagYlQG/gBsyA/deNzROkq8= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 h1:6ksZ7t1hNOzGPPs8DK7SvXQf6UfWzi+W5Z7PCBl8gx4= +github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510/go.mod h1:UC0TwJiF90m2T3iYPQBKnGu8gv3s55dF/EgpTq8gyvo= github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 h1:a9hSJdJcd16e0HoMsnFvaHvxB3pxSD+SC7+CISp7xY0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= +github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= @@ -486,20 +898,35 @@ github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dI github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg= github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -511,6 +938,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -531,12 +959,19 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4 github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo= github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI= +github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -544,6 +979,12 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= +github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -552,18 +993,42 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yuin/goldmark-emoji v1.0.4 h1:vCwMkPZSNefSUnOW2ZKRUjBSD5Ok3W78IXhGxxAEF90= github.com/yuin/goldmark-emoji v1.0.4/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= +github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/auto/sdk v1.2.0 h1:YpRtUFjvhSymycLS2T81lT6IGhcUP+LUPtv0iv1N8bM= go.opentelemetry.io/auto/sdk v1.2.0/go.mod h1:1deq2zL7rwjwC8mR7XgY2N+tlIl6pjmEUoLDENMEzwk= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0015 h1:8PzrQFk3oKiT1Sd5EmNEcagdMyt1KcBy5/OyF5He5gY= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0015/go.mod h1:I1PqyHJlsXjANC73tp43nDId7/jiv82NoZZ6uS0xdwM= +go.opentelemetry.io/collector/semconv v0.81.0 h1:lCYNNo3powDvFIaTPP2jDKIrBiV1T92NK4QgL/aHYXw= +go.opentelemetry.io/collector/semconv v0.81.0/go.mod h1:TlYPtzvsXyHOgr5eATi43qEMqwSmIziivJB2uctKswo= go.opentelemetry.io/contrib/bridges/otelzap v0.13.0 h1:aBKdhLVieqvwWe9A79UHI/0vgp2t/s2euY8X59pGRlw= go.opentelemetry.io/contrib/bridges/otelzap v0.13.0/go.mod h1:SYqtxLQE7iINgh6WFuVi2AI70148B8EI35DSk0Wr8m4= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.57.0 h1:1wEousrQOXTAhk16quIMIo1gSaUp1J3PEVlsiEAtmeU= @@ -604,6 +1069,8 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE= go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -612,89 +1079,266 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go4.org/netipx v0.0.0-20230125063823-8449b0a6169f h1:ketMxHg+vWm3yccyYiq+uK8D3fRmna2Fcj+awpQp84s= +go4.org/netipx v0.0.0-20230125063823-8449b0a6169f/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U= golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b h1:QoALfVG9rhQ/M7vYDScfPdWjGL9dlsVVM5VGh7aKoAA= golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -703,12 +1347,76 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto/googleapis/api v0.0.0-20250908214217-97024824d090 h1:d8Nakh1G+ur7+P3GcMjpRDEkoLUcLW2iU92XVqR+XMQ= google.golang.org/genproto/googleapis/api v0.0.0-20250908214217-97024824d090/go.mod h1:U8EXRNSd8sUYyDfs/It7KVWodQr+Hf9xtxyxWudSwEw= google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 h1:/OQuEa4YWtDt7uQWHd3q3sUMb+QOLQUg1xa8CEsRv5w= google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -716,23 +1424,35 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -742,6 +1462,26 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apimachinery v0.29.0-alpha.3 h1:Y/VavRd57V5fliXV8M2Zr1Xyzi+raIhkDemWdGuuw6w= +k8s.io/apimachinery v0.29.0-alpha.3/go.mod h1:yFk3nwBh/jXlkMvRKH7BKtX7saT1lRmmGV6Ru0cTSUA= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s= modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= @@ -751,3 +1491,12 @@ modernc.org/memory v1.9.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/packages/api/internal/cfg/model.go b/packages/api/internal/cfg/model.go index 7f07a25d0e..a52e0bab85 100644 --- a/packages/api/internal/cfg/model.go +++ b/packages/api/internal/cfg/model.go @@ -14,8 +14,9 @@ type Config struct { ClickhouseConnectionString string `env:"CLICKHOUSE_CONNECTION_STRING"` - LocalClusterEndpoint string `env:"LOCAL_CLUSTER_ENDPOINT"` - LocalClusterToken string `env:"LOCAL_CLUSTER_TOKEN"` + LokiPassword string `env:"LOKI_PASSWORD"` + LokiURL string `env:"LOKI_URL,required"` + LokiUser string `env:"LOKI_USER"` NomadAddress string `env:"NOMAD_ADDRESS" envDefault:"http://localhost:4646"` NomadToken string `env:"NOMAD_TOKEN"` diff --git a/packages/api/internal/cfg/model_test.go b/packages/api/internal/cfg/model_test.go index c1637912f4..1d9f32c945 100644 --- a/packages/api/internal/cfg/model_test.go +++ b/packages/api/internal/cfg/model_test.go @@ -11,6 +11,7 @@ import ( func TestParse(t *testing.T) { // set base required values t.Setenv("POSTGRES_CONNECTION_STRING", "postgres-connection-string") + t.Setenv("LOKI_URL", "http://loki:3100") t.Run("postgres connection string is required", func(t *testing.T) { removeEnv(t, "POSTGRES_CONNECTION_STRING") diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index cb190ed70c..ce96710123 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -13,9 +13,11 @@ import ( "go.opentelemetry.io/otel" "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" + clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -46,30 +48,11 @@ func newLocalCluster( ctx context.Context, tel *telemetry.Client, nomad *nomadapi.Client, - endpoint string, - secret string, + clickhouse clickhouse.Clickhouse, + queryLogsProvider *loki.LokiQueryProvider, ) (*Cluster, error) { clusterID := consts.LocalClusterID - httpClient, err := api.NewClientWithResponses( - fmt.Sprintf("http://%s", endpoint), - func(c *api.Client) error { - c.RequestEditors = append( - c.RequestEditors, - func(_ context.Context, req *http.Request) error { - req.Header.Set(consts.EdgeApiAuthHeader, secret) - - return nil - }, - ) - - return nil - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to create http client: %w", err) - } - instances := smap.New[*Instance]() instanceCreation := func(ctx context.Context, item discovery.Item) (*Instance, error) { // For local cluster we are doing direct connection to instance IP and API port and without additional cluster auth. @@ -84,7 +67,7 @@ func newLocalCluster( SandboxDomain: nil, instances: instances, - resources: newRemoteClusterResourceProvider(instances, httpClient), + resources: newLocalClusterResourceProvider(clickhouse, queryLogsProvider, instances), synchronization: synchronization.NewSynchronize("cluster-instances", "Cluster instances", store), } diff --git a/packages/api/internal/clusters/clusters_sync.go b/packages/api/internal/clusters/clusters_sync.go index 8f9991d95b..a295409dc3 100644 --- a/packages/api/internal/clusters/clusters_sync.go +++ b/packages/api/internal/clusters/clusters_sync.go @@ -2,7 +2,6 @@ package clusters import ( "context" - "errors" "sync" "time" @@ -10,11 +9,12 @@ import ( nomadapi "github.com/hashicorp/nomad/api" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/api/internal/cfg" + clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/db/client" "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/consts" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -33,27 +33,25 @@ type Pool struct { synchronization *synchronization.Synchronize[queries.Cluster, *Cluster] } -func localClusterConfig(clusterEndpoint, clusterToken string) (*queries.Cluster, error) { - if clusterEndpoint == "" { - return nil, nil - } - if clusterToken == "" { - return nil, errors.New("no local cluster token provided") - } - +func localClusterConfig() (*queries.Cluster, error) { return &queries.Cluster{ ID: consts.LocalClusterID, EndpointTls: false, - Endpoint: clusterEndpoint, - Token: clusterToken, SandboxProxyDomain: nil, }, nil } -func NewPool(ctx context.Context, tel *telemetry.Client, db *client.Client, nomad *nomadapi.Client, config cfg.Config) (*Pool, error) { +func NewPool( + ctx context.Context, + tel *telemetry.Client, + db *client.Client, + nomad *nomadapi.Client, + queryMetricsProvider clickhouse.Clickhouse, + queryLogsProvider *loki.LokiQueryProvider, +) (*Pool, error) { clusters := smap.New[*Cluster]() - localCluster, err := localClusterConfig(config.LocalClusterEndpoint, config.LocalClusterToken) + localCluster, err := localClusterConfig() if err != nil { return nil, err } @@ -65,7 +63,15 @@ func NewPool(ctx context.Context, tel *telemetry.Client, db *client.Client, noma synchronization: synchronization.NewSynchronize( "clusters-pool", "Clusters pool", - clustersSyncStore{db: db, tel: tel, clusters: clusters, local: localCluster, nomad: nomad}, + clustersSyncStore{ + db: db, + tel: tel, + clusters: clusters, + local: localCluster, + nomad: nomad, + queryLogsProvider: queryLogsProvider, + queryMetricsProvider: queryMetricsProvider, + }, ), } @@ -103,11 +109,13 @@ func (p *Pool) Close(ctx context.Context) { // SynchronizationStore is an interface that defines methods for synchronizing the clusters pool with the database type clustersSyncStore struct { - db *client.Client - tel *telemetry.Client - clusters *smap.Map[*Cluster] - local *queries.Cluster - nomad *nomadapi.Client + db *client.Client + tel *telemetry.Client + clusters *smap.Map[*Cluster] + local *queries.Cluster + nomad *nomadapi.Client + queryMetricsProvider clickhouse.Clickhouse + queryLogsProvider *loki.LokiQueryProvider } func (d clustersSyncStore) SourceList(ctx context.Context) ([]queries.Cluster, error) { @@ -164,7 +172,7 @@ func (d clustersSyncStore) PoolInsert(ctx context.Context, cluster queries.Clust // Local cluster if cluster.ID == consts.LocalClusterID { - c, err = newLocalCluster(context.WithoutCancel(ctx), d.tel, d.nomad, d.local.Endpoint, d.local.Token) + c, err = newLocalCluster(context.WithoutCancel(ctx), d.tel, d.nomad, d.queryMetricsProvider, d.queryLogsProvider) if err != nil { logger.L().Error(ctx, "Initializing local cluster failed", zap.Error(err), logger.WithClusterID(cluster.ID)) diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go new file mode 100644 index 0000000000..5c4c000b2c --- /dev/null +++ b/packages/api/internal/clusters/resources_local.go @@ -0,0 +1,225 @@ +package clusters + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/loki/pkg/logproto" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/e2b-dev/infra/packages/api/internal/api" + clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" + clickhouseutils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" + templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/logs" + "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" + "github.com/e2b-dev/infra/packages/shared/pkg/smap" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" +) + +type LocalClusterResourceProvider struct { + querySandboxMetricsProvider clickhouse.SandboxQueriesProvider + queryLogsProvider *loki.LokiQueryProvider + instances *smap.Map[*Instance] +} + +const ( + sandboxLogsOldestLimit = 168 * time.Hour // 7 days + defaultLogsLimit = 1000 + defaultDirection = logproto.FORWARD +) + +func newLocalClusterResourceProvider( + querySandboxMetricsProvider clickhouse.SandboxQueriesProvider, + queryLogsProvider *loki.LokiQueryProvider, + instances *smap.Map[*Instance], +) ClusterResource { + return &LocalClusterResourceProvider{ + querySandboxMetricsProvider: querySandboxMetricsProvider, + queryLogsProvider: queryLogsProvider, + instances: instances, + } +} + +func (l *LocalClusterResourceProvider) GetSandboxMetrics(ctx context.Context, teamID string, sandboxID string, qStart *int64, qEnd *int64) ([]api.SandboxMetric, error) { + start, end, err := clickhouseutils.GetSandboxStartEndTime(ctx, l.querySandboxMetricsProvider, teamID, sandboxID, qStart, qEnd) + if err != nil { + return nil, fmt.Errorf(`error when getting metrics time range: %w`, err) + } + + start, end, err = clickhouseutils.ValidateRange(start, end) + if err != nil { + return nil, fmt.Errorf(`error when validating range of metrics: %w`, err) + } + + // Calculate the step size + step := clickhouseutils.CalculateStep(start, end) + + rawMetrics, err := l.querySandboxMetricsProvider.QuerySandboxMetrics(ctx, sandboxID, teamID, start, end, step) + if err != nil { + return nil, fmt.Errorf(`error when querying sandbox metrics: %w`, err) + } + + metrics := make([]api.SandboxMetric, len(rawMetrics)) + for i, m := range rawMetrics { + metrics[i] = api.SandboxMetric{ + Timestamp: m.Timestamp, + TimestampUnix: m.Timestamp.Unix(), + CpuUsedPct: float32(m.CPUUsedPercent), + CpuCount: int32(m.CPUCount), + MemTotal: int64(m.MemTotal), + MemUsed: int64(m.MemUsed), + DiskTotal: int64(m.DiskTotal), + DiskUsed: int64(m.DiskUsed), + } + } + + return metrics, nil +} + +func (l *LocalClusterResourceProvider) GetSandboxesMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, error) { + rawMetrics, err := l.querySandboxMetricsProvider.QueryLatestMetrics(ctx, sandboxIDs, teamID) + if err != nil { + logger.L().Error(ctx, "Error fetching sandbox metrics from ClickHouse", logger.WithTeamID(teamID), zap.Error(err)) + + return nil, err + } + + metrics := make(map[string]api.SandboxMetric) + for _, m := range rawMetrics { + metrics[m.SandboxID] = api.SandboxMetric{ + Timestamp: m.Timestamp, + TimestampUnix: m.Timestamp.Unix(), + CpuUsedPct: float32(m.CPUUsedPercent), + CpuCount: int32(m.CPUCount), + MemTotal: int64(m.MemTotal), + MemUsed: int64(m.MemUsed), + DiskTotal: int64(m.DiskTotal), + DiskUsed: int64(m.DiskUsed), + } + } + + return metrics, nil +} + +func (l *LocalClusterResourceProvider) GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, qStart *int64, qLimit *int32) (api.SandboxLogs, error) { + end := time.Now() + var start time.Time + + if qStart != nil { + start = time.UnixMilli(*qStart) + } else { + start = end.Add(-sandboxLogsOldestLimit) + } + + limit := defaultLogsLimit + if qLimit != nil { + limit = int(*qLimit) + } + + raw, err := l.queryLogsProvider.QuerySandboxLogs(ctx, teamID, sandboxID, start, end, limit) + if err != nil { + return api.SandboxLogs{}, fmt.Errorf("error when fetching sandbox logs: %w", err) + } + + ll := make([]api.SandboxLog, len(raw)) + for i, row := range raw { + ll[i] = api.SandboxLog{Line: row.Raw, Timestamp: row.Timestamp} + } + + le := make([]api.SandboxLogEntry, len(raw)) + for i, row := range raw { + le[i] = api.SandboxLogEntry{ + Timestamp: row.Timestamp, + Level: api.LogLevel(row.Level), + Message: row.Message, + Fields: row.Fields, + } + } + + return api.SandboxLogs{Logs: ll, LogEntries: le}, nil +} + +func (l *LocalClusterResourceProvider) GetBuildLogs(ctx context.Context, nodeID *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) { + start, end := logQueryWindow(cursor, direction) + + lokiDirection := defaultDirection + if direction == api.LogsDirectionBackward { + lokiDirection = logproto.BACKWARD + } + + // todo + if source == nil { + // try node and then default to Loki + } else if *source == api.LogsSourcePersistent { + // force to node + } else if *source == api.LogsSourceTemporary { + // force to loki + } + + if nodeID != nil { + instance, found := l.instances.Get(*nodeID) + if found { + var lvlReq *templatemanagergrpc.LogLevel + if level != nil { + lvlReq = templatemanagergrpc.LogLevel(*level).Enum() + } + + res, err := instance.grpc.Template.TemplateBuildStatus( + ctx, &templatemanagergrpc.TemplateStatusRequest{ + TemplateID: templateID, + BuildID: buildID, + Offset: &offset, + Limit: utils.ToPtr(uint32(limit)), + Level: lvlReq, + Start: timestamppb.New(start), + End: timestamppb.New(end), + Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), + }, + ) + if err != nil { + telemetry.ReportError(ctx, "error when returning logs for template build", err) + logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) + + return nil, err + } + + raw := res.GetLogEntries() + + // Add an extra newline to each log entry to ensure proper formatting in the CLI + entries := make([]logs.LogEntry, len(raw)) + for i, entry := range raw { + entries[i] = logs.LogEntry{ + Timestamp: entry.GetTimestamp().AsTime(), + Message: entry.GetMessage(), + Level: logs.LogLevel(entry.GetLevel()), + Fields: entry.GetFields(), + } + } + + return entries, nil + } + } + + logs, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) + if err != nil { + return nil, fmt.Errorf("error when fetching build logs: %w", err) + } + + return logs, nil +} + +func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection { + switch direction { + case api.LogsDirectionForward: + return templatemanagergrpc.LogsDirection_Forward + case api.LogsDirectionBackward: + return templatemanagergrpc.LogsDirection_Backward + default: + return templatemanagergrpc.LogsDirection_Forward + } +} diff --git a/packages/api/internal/clusters/resources_impl.go b/packages/api/internal/clusters/resources_remote.go similarity index 100% rename from packages/api/internal/clusters/resources_impl.go rename to packages/api/internal/clusters/resources_remote.go diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 08a728d06d..a08ad6518e 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -33,6 +33,7 @@ import ( featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" "github.com/e2b-dev/infra/packages/shared/pkg/keys" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -110,7 +111,12 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, config cfg.Config) } } - clusters, err := clusters.NewPool(ctx, tel, sqlcDB, nomadClient, config) + queryLogsProvider, err := loki.NewLokiQueryProvider(config.LokiURL, config.LokiUser, config.LokiPassword) + if err != nil { + logger.L().Fatal(ctx, "error when getting logs query provider", zap.Error(err)) + } + + clusters, err := clusters.NewPool(ctx, tel, sqlcDB, nomadClient, clickhouseStore, queryLogsProvider) if err != nil { logger.L().Fatal(ctx, "initializing edge clusters pool failed", zap.Error(err)) } diff --git a/packages/clickhouse/pkg/utils/sandbox.go b/packages/clickhouse/pkg/utils/sandbox.go new file mode 100644 index 0000000000..22b8ae779d --- /dev/null +++ b/packages/clickhouse/pkg/utils/sandbox.go @@ -0,0 +1,47 @@ +package utils + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + + clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +func GetSandboxStartEndTime(ctx context.Context, clickhouseStore clickhouse.SandboxQueriesProvider, teamID, sandboxID string, qStart *int64, qEnd *int64) (time.Time, time.Time, error) { + // Check if the sandbox exists + var start, end time.Time + if qStart != nil { + start = time.Unix(*qStart, 0) + } + + if qEnd != nil { + end = time.Unix(*qEnd, 0) + } + + if start.IsZero() || end.IsZero() { + sbxStart, sbxEnd, err := clickhouseStore.QuerySandboxTimeRange(ctx, sandboxID, teamID) + if err != nil { + logger.L().Error(ctx, "Error fetching sandbox time range from ClickHouse", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID), + zap.Error(err), + ) + + return time.Time{}, time.Time{}, fmt.Errorf("error querying sandbox time range: %w", err) + } + + if start.IsZero() { + start = sbxStart + } + + if end.IsZero() { + end = sbxEnd + } + } + + return start, end, nil +} diff --git a/packages/client-proxy/internal/edge/handlers/sandbox-metrics.go b/packages/client-proxy/internal/edge/handlers/sandbox-metrics.go index f55e22222f..007985b82e 100644 --- a/packages/client-proxy/internal/edge/handlers/sandbox-metrics.go +++ b/packages/client-proxy/internal/edge/handlers/sandbox-metrics.go @@ -1,15 +1,13 @@ package handlers import ( - "context" "fmt" "net/http" - "time" "github.com/gin-gonic/gin" "go.uber.org/zap" - clickhouseUtils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" + clickhouseutils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -20,14 +18,14 @@ func (a *APIStore) V1SandboxMetrics(c *gin.Context, sandboxID string, params api ctx, span := tracer.Start(ctx, "sandbox-metrics") defer span.End() - start, end, err := getSandboxStartEndTime(ctx, a, sandboxID, params) + start, end, err := clickhouseutils.GetSandboxStartEndTime(ctx, a.querySandboxMetricsProvider, params.TeamID, sandboxID, params.Start, params.End) if err != nil { a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("error when getting metrics time range: %s", err)) return } - start, end, err = clickhouseUtils.ValidateRange(start, end) + start, end, err = clickhouseutils.ValidateRange(start, end) if err != nil { telemetry.ReportError(ctx, "error validating dates", err, telemetry.WithTeamID(params.TeamID)) a.sendAPIStoreError(c, http.StatusBadRequest, err.Error()) @@ -36,7 +34,7 @@ func (a *APIStore) V1SandboxMetrics(c *gin.Context, sandboxID string, params api } // Calculate the step size - step := clickhouseUtils.CalculateStep(start, end) + step := clickhouseutils.CalculateStep(start, end) metrics, err := a.querySandboxMetricsProvider.QuerySandboxMetrics(ctx, sandboxID, params.TeamID, start, end, step) if err != nil { @@ -67,38 +65,3 @@ func (a *APIStore) V1SandboxMetrics(c *gin.Context, sandboxID string, params api c.JSON(http.StatusOK, apiMetrics) } - -func getSandboxStartEndTime(ctx context.Context, a *APIStore, sandboxID string, params api.V1SandboxMetricsParams) (time.Time, time.Time, error) { - // Check if the sandbox exists - var start, end time.Time - if params.Start != nil { - start = time.Unix(*params.Start, 0) - } - - if params.End != nil { - end = time.Unix(*params.End, 0) - } - - if start.IsZero() || end.IsZero() { - sbxStart, sbxEnd, err := a.querySandboxMetricsProvider.QuerySandboxTimeRange(ctx, sandboxID, params.TeamID) - if err != nil { - logger.L().Error(ctx, "Error fetching sandbox time range from ClickHouse", - logger.WithSandboxID(sandboxID), - logger.WithTeamID(params.TeamID), - zap.Error(err), - ) - - return time.Time{}, time.Time{}, fmt.Errorf("error querying sandbox time range: %w", err) - } - - if start.IsZero() { - start = sbxStart - } - - if end.IsZero() { - end = sbxEnd - } - } - - return start, end, nil -} diff --git a/packages/client-proxy/internal/edge/handlers/store.go b/packages/client-proxy/internal/edge/handlers/store.go index 773f94442a..4d84f61d14 100644 --- a/packages/client-proxy/internal/edge/handlers/store.go +++ b/packages/client-proxy/internal/edge/handlers/store.go @@ -12,12 +12,12 @@ import ( clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/proxy/internal/cfg" "github.com/e2b-dev/infra/packages/proxy/internal/edge/info" - loggerprovider "github.com/e2b-dev/infra/packages/proxy/internal/edge/logger-provider" metricsprovider "github.com/e2b-dev/infra/packages/proxy/internal/edge/metrics-provider" e2binstances "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool" "github.com/e2b-dev/infra/packages/shared/pkg/env" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" catalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -29,7 +29,7 @@ type APIStore struct { info *info.ServiceInfo instancesPool *e2binstances.InstancesPool sandboxes catalog.SandboxesCatalog - queryLogsProvider loggerprovider.LogsQueryProvider + queryLogsProvider *loki.LokiQueryProvider querySandboxMetricsProvider clickhouse.SandboxQueriesProvider } @@ -45,7 +45,7 @@ func NewStore( catalog catalog.SandboxesCatalog, config cfg.Config, ) (*APIStore, error) { - queryLogsProvider, err := loggerprovider.GetLogsQueryProvider(config) + queryLogsProvider, err := loki.NewLokiQueryProvider(config.LokiURL, config.LokiUser, config.LokiPassword) if err != nil { return nil, fmt.Errorf("error when getting logs query provider: %w", err) } diff --git a/packages/client-proxy/internal/edge/logger-provider/provider.go b/packages/client-proxy/internal/edge/logger-provider/provider.go deleted file mode 100644 index 85dfc5bf88..0000000000 --- a/packages/client-proxy/internal/edge/logger-provider/provider.go +++ /dev/null @@ -1,20 +0,0 @@ -package logger_provider - -import ( - "context" - "time" - - "github.com/grafana/loki/pkg/logproto" - - "github.com/e2b-dev/infra/packages/proxy/internal/cfg" - "github.com/e2b-dev/infra/packages/shared/pkg/logs" -) - -type LogsQueryProvider interface { - QueryBuildLogs(ctx context.Context, templateID string, buildID string, start time.Time, end time.Time, limit int, offset int32, level *logs.LogLevel, direction logproto.Direction) ([]logs.LogEntry, error) - QuerySandboxLogs(ctx context.Context, teamID string, sandboxID string, start time.Time, end time.Time, limit int) ([]logs.LogEntry, error) -} - -func GetLogsQueryProvider(config cfg.Config) (LogsQueryProvider, error) { - return NewLokiQueryProvider(config) -} diff --git a/packages/shared/pkg/logs/logsloki/loki.go b/packages/shared/pkg/logs/loki/loki.go similarity index 99% rename from packages/shared/pkg/logs/logsloki/loki.go rename to packages/shared/pkg/logs/loki/loki.go index 94f8e90366..5b15496a20 100644 --- a/packages/shared/pkg/logs/logsloki/loki.go +++ b/packages/shared/pkg/logs/loki/loki.go @@ -1,4 +1,4 @@ -package logsloki +package loki import ( "context" diff --git a/packages/client-proxy/internal/edge/logger-provider/provider_loki.go b/packages/shared/pkg/logs/loki/provider.go similarity index 87% rename from packages/client-proxy/internal/edge/logger-provider/provider_loki.go rename to packages/shared/pkg/logs/loki/provider.go index 77186be7c0..26b2c739bd 100644 --- a/packages/client-proxy/internal/edge/logger-provider/provider_loki.go +++ b/packages/shared/pkg/logs/loki/provider.go @@ -1,4 +1,4 @@ -package logger_provider +package loki import ( "context" @@ -10,10 +10,8 @@ import ( "github.com/grafana/loki/pkg/logproto" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/proxy/internal/cfg" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" - "github.com/e2b-dev/infra/packages/shared/pkg/logs/logsloki" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -21,11 +19,11 @@ type LokiQueryProvider struct { client *loki.DefaultClient } -func NewLokiQueryProvider(config cfg.Config) (*LokiQueryProvider, error) { +func NewLokiQueryProvider(lokiURL string, lokiUser string, lokiPassword string) (*LokiQueryProvider, error) { lokiClient := &loki.DefaultClient{ - Address: config.LokiURL, - Username: config.LokiUser, - Password: config.LokiPassword, + Address: lokiURL, + Username: lokiUser, + Password: lokiPassword, } return &LokiQueryProvider{client: lokiClient}, nil @@ -47,7 +45,7 @@ func (l *LokiQueryProvider) QueryBuildLogs(ctx context.Context, templateID strin return make([]logs.LogEntry, 0), nil } - lm, err := logsloki.ResponseMapper(ctx, res, offset, level) + lm, err := ResponseMapper(ctx, res, offset, level) if err != nil { telemetry.ReportError(ctx, "error when mapping build logs", err) logger.L().Error(ctx, "error when mapping logs for template build", zap.Error(err), logger.WithBuildID(buildID)) @@ -73,7 +71,7 @@ func (l *LokiQueryProvider) QuerySandboxLogs(ctx context.Context, teamID string, return make([]logs.LogEntry, 0), nil } - lm, err := logsloki.ResponseMapper(ctx, res, 0, nil) + lm, err := ResponseMapper(ctx, res, 0, nil) if err != nil { telemetry.ReportError(ctx, "error when mapping sandbox logs", err) logger.L().Error(ctx, "error when mapping logs for sandbox", zap.Error(err), logger.WithSandboxID(sandboxID)) From a9319838ae66b44bfbd0aeb7d2ae788db7891d3e Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 16:26:47 +0100 Subject: [PATCH 22/58] Client proxy (edge api) is no longer needed for integration tests pipeline --- .github/actions/start-services/action.yml | 4 ---- packages/api/Makefile | 3 +-- packages/client-proxy/Makefile | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index e2c01cd3b6..089d2419f5 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -103,7 +103,6 @@ runs: OTEL_COLLECTOR_GRPC_ENDPOINT: "localhost:4317" MAX_PARALLEL_MEMFILE_SNAPSHOTTING: "2" SHARED_CHUNK_CACHE_PATH: "./.e2b-chunk-cache" - EDGE_TOKEN: "abdcdefghijklmnop" run: | mkdir -p $SHARED_CHUNK_CACHE_PATH mkdir -p ~/logs @@ -114,9 +113,6 @@ runs: # Start orchestrator bash ./scripts/start-service.sh "Orchestrator" packages/orchestrator run-debug ~/logs/orchestrator.log http://localhost:5008/health - # Start Edge API - bash ./scripts/start-service.sh "Edge" packages/client-proxy run ~/logs/edge.log http://localhost:3001/health - # Start API bash ./scripts/start-service.sh "API" packages/api run ~/logs/api.log http://localhost:3000/health shell: bash diff --git a/packages/api/Makefile b/packages/api/Makefile index c23753cf8f..a8310c60a8 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -42,8 +42,7 @@ run: ENVIRONMENT=$(ENVIRONMENT) \ ORCHESTRATOR_PORT=5008 \ NODE_ID=integration-tests \ - LOCAL_CLUSTER_ENDPOINT=localhost:3001 \ - LOCAL_CLUSTER_TOKEN=$(EDGE_TOKEN) \ + LOKI_URL=unset \ ./bin/api --port 3000 define setup_local_env diff --git a/packages/client-proxy/Makefile b/packages/client-proxy/Makefile index 1370a481dc..b77e7dcc59 100644 --- a/packages/client-proxy/Makefile +++ b/packages/client-proxy/Makefile @@ -34,8 +34,6 @@ run: EDGE_SECRET=$(EDGE_TOKEN) \ NODE_ID=integration-tests \ NODE_IP=127.0.0.1 \ - SD_EDGE_PROVIDER=STATIC \ - SD_EDGE_STATIC=127.0.0.1 \ SD_ORCHESTRATOR_PROVIDER=STATIC \ SD_ORCHESTRATOR_STATIC=127.0.0.1 \ SKIP_ORCHESTRATOR_READINESS_CHECK=true \ From 6eee05897bac96d8daa998f09d078a0d5bce93d7 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 16:38:13 +0100 Subject: [PATCH 23/58] Fixed missing connection for gRPC mocks --- .../api/internal/orchestrator/nodemanager/mock.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/orchestrator/nodemanager/mock.go b/packages/api/internal/orchestrator/nodemanager/mock.go index 4fa8c6632e..52df8d7f2f 100644 --- a/packages/api/internal/orchestrator/nodemanager/mock.go +++ b/packages/api/internal/orchestrator/nodemanager/mock.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/clusters" @@ -58,10 +59,14 @@ func (n *mockSandboxClientWithSleep) Create(_ context.Context, _ *orchestrator.S // newMockGRPCClient creates a new mock gRPC connection for testing func newMockGRPCClient() *clusters.GRPCClient { + // Create a dummy connection that will never be used + conn, _ := grpc.NewClient("localhost:0", grpc.WithTransportCredentials(insecure.NewCredentials())) + return &clusters.GRPCClient{ - Info: &mockInfoClient{}, - Sandbox: &mockSandboxClient{}, - Template: &mockTemplateClient{}, + Info: &mockInfoClient{}, + Sandbox: &mockSandboxClient{}, + Template: &mockTemplateClient{}, + Connection: conn, } } From e66fbc6ee9d958c06a696206840f3742e787a443 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 16:43:56 +0100 Subject: [PATCH 24/58] Fixed not checked err for build logs --- packages/api/internal/handlers/template_build_status.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/api/internal/handlers/template_build_status.go b/packages/api/internal/handlers/template_build_status.go index 70609807d8..06938feb4d 100644 --- a/packages/api/internal/handlers/template_build_status.go +++ b/packages/api/internal/handlers/template_build_status.go @@ -120,6 +120,13 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem } logs, err := cluster.GetResources().GetBuildLogs(ctx, buildInfo.NodeID, templateID, buildID, offset, limit, apiToLogLevel(params.Level), nil, api.LogsDirectionForward, nil) + if err != nil { + telemetry.ReportError(ctx, "error when getting build logs", err, telemetry.WithTemplateID(templateID), telemetry.WithBuildID(buildID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when getting build logs") + + return + } + for _, entry := range logs { if legacyLogs { lgs = append(lgs, fmt.Sprintf("[%s] %s\n", entry.Timestamp.Format(time.RFC3339), entry.Message)) From f692fb80eb24a371ecc2b9f9d8cfc673110bbe01 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 17:26:34 +0100 Subject: [PATCH 25/58] For local and integration tests inject static orch instance --- .github/actions/start-services/action.yml | 1 - .../api/internal/clusters/discovery/local.go | 23 +++++++++++++++++++ .../api/internal/clusters/instances_sync.go | 8 ------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 089d2419f5..d87c349e59 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -96,7 +96,6 @@ runs: ENVD_TIMEOUT: "60s" ORCHESTRATOR_SERVICES: "orchestrator,template-manager" SANDBOX_ACCESS_TOKEN_HASH_SEED: "abcdefghijklmnopqrstuvwxyz" - TEMPLATE_MANAGER_HOST: "localhost:5008" ARTIFACTS_REGISTRY_PROVIDER: "Local" STORAGE_PROVIDER: "Local" ENVIRONMENT: "local" diff --git a/packages/api/internal/clusters/discovery/local.go b/packages/api/internal/clusters/discovery/local.go index 7b285442a5..54a1431fcd 100644 --- a/packages/api/internal/clusters/discovery/local.go +++ b/packages/api/internal/clusters/discovery/local.go @@ -10,9 +10,13 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/clusters/discovery" "github.com/e2b-dev/infra/packages/shared/pkg/consts" + "github.com/e2b-dev/infra/packages/shared/pkg/env" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) +var testsInstanceHost = env.GetEnv("TESTS_ORCH_INSTANCE_HOST", "localhost") + type LocalServiceDiscovery struct { nomad *nomadapi.Client clusterID uuid.UUID @@ -29,6 +33,25 @@ func (sd *LocalServiceDiscovery) Query(ctx context.Context) ([]Item, error) { _, span := tracer.Start(ctx, "query-local-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) defer span.End() + // Static discovery for local environment + if env.IsLocal() { + if testsInstanceHost == "" { + logger.L().Debug(ctx, "Service discovery is disabled in local environment") + + return []Item{}, nil + } + + return []Item{ + { + UniqueIdentifier: "local", + NodeID: "local", + InstanceID: "unknown", + LocalIPAddress: testsInstanceHost, + LocalInstanceApiPort: consts.OrchestratorApiPort, + }, + }, nil + } + // For now, we want to search only for template builders as local orchestrators are still discovered // old way via Nomad discovery directly inside node manager flow. To minimize changes, we keep it this way for now. alloc, err := discovery.ListOrchestratorAndTemplateBuilderAllocations(ctx, sd.nomad, discovery.FilterTemplateBuilders) diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 5cfcf2f970..1d90d6e30a 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -8,7 +8,6 @@ import ( "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" - "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/smap" ) @@ -23,13 +22,6 @@ type instancesSyncStore struct { } func (d instancesSyncStore) SourceList(ctx context.Context) ([]discovery.Item, error) { - // Disable discovery for local environments - if env.IsLocal() { - logger.L().Debug(ctx, "Service discovery is disabled in local environment") - - return []discovery.Item{}, nil - } - items, err := d.discovery.Query(ctx) if err != nil { return nil, fmt.Errorf("failed to get cluster instances from service discovery: %w", err) From 2dd8221bd6b1b630a4cbe4286e25315f6581d7fb Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 17:56:21 +0100 Subject: [PATCH 26/58] Implemented builds logs for both local and remote resource providers --- packages/api/internal/clusters/resources.go | 72 ++++++++++++ .../api/internal/clusters/resources_local.go | 109 ++++++------------ .../api/internal/clusters/resources_remote.go | 57 +++++++-- 3 files changed, 154 insertions(+), 84 deletions(-) diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go index f305d58574..aa51308bdd 100644 --- a/packages/api/internal/clusters/resources.go +++ b/packages/api/internal/clusters/resources.go @@ -4,8 +4,16 @@ import ( "context" "time" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/e2b-dev/infra/packages/api/internal/api" + templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) type ClusterResource interface { @@ -35,3 +43,67 @@ func logQueryWindow(cursor *time.Time, direction api.LogsDirection) (time.Time, return start, end } + +func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection { + switch direction { + case api.LogsDirectionForward: + return templatemanagergrpc.LogsDirection_Forward + case api.LogsDirectionBackward: + return templatemanagergrpc.LogsDirection_Backward + default: + return templatemanagergrpc.LogsDirection_Forward + } +} + +func logCheckSourceType(source *api.LogsSource, sourceType api.LogsSource) bool { + return source == nil || *source == sourceType +} + +func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { + if level == nil { + return nil + } + + value := edgeapi.LogLevel(logs.LevelToString(*level)) + + return &value +} + +func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { + var lvlReq *templatemanagergrpc.LogLevel + if level != nil { + lvlReq = templatemanagergrpc.LogLevel(*level).Enum() + } + + res, err := instance.GetConnection().Template.TemplateBuildStatus( + ctx, &templatemanagergrpc.TemplateStatusRequest{ + TemplateID: templateID, + BuildID: buildID, + Offset: &offset, + Limit: utils.ToPtr(uint32(limit)), + Level: lvlReq, + Start: timestamppb.New(start), + End: timestamppb.New(end), + Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), + }, + ) + if err != nil { + telemetry.ReportError(ctx, "error when returning logs for template build", err) + logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) + + return nil, err + } + + raw := res.GetLogEntries() + entries := make([]logs.LogEntry, len(raw)) + for i, entry := range res.GetLogEntries() { + entries[i] = logs.LogEntry{ + Timestamp: entry.GetTimestamp().AsTime(), + Message: entry.GetMessage(), + Level: logs.LogLevel(entry.GetLevel()), + Fields: entry.GetFields(), + } + } + + return entries, nil +} diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go index 5c4c000b2c..326edbb0d2 100644 --- a/packages/api/internal/clusters/resources_local.go +++ b/packages/api/internal/clusters/resources_local.go @@ -7,18 +7,14 @@ import ( "github.com/grafana/loki/pkg/logproto" "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/api/internal/api" clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" clickhouseutils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils" - templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" "github.com/e2b-dev/infra/packages/shared/pkg/smap" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) type LocalClusterResourceProvider struct { @@ -144,82 +140,49 @@ func (l *LocalClusterResourceProvider) GetSandboxLogs(ctx context.Context, teamI return api.SandboxLogs{Logs: ll, LogEntries: le}, nil } -func (l *LocalClusterResourceProvider) GetBuildLogs(ctx context.Context, nodeID *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) { +func (l *LocalClusterResourceProvider) GetBuildLogs( + ctx context.Context, + nodeID *string, + templateID string, + buildID string, + offset int32, + limit int32, + level *logs.LogLevel, + cursor *time.Time, + direction api.LogsDirection, + source *api.LogsSource, +) ([]logs.LogEntry, error) { start, end := logQueryWindow(cursor, direction) - lokiDirection := defaultDirection - if direction == api.LogsDirectionBackward { - lokiDirection = logproto.BACKWARD - } - - // todo - if source == nil { - // try node and then default to Loki - } else if *source == api.LogsSourcePersistent { - // force to node - } else if *source == api.LogsSourceTemporary { - // force to loki - } - - if nodeID != nil { + // Fetch logs directly from template builder instance + if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { instance, found := l.instances.Get(*nodeID) - if found { - var lvlReq *templatemanagergrpc.LogLevel - if level != nil { - lvlReq = templatemanagergrpc.LogLevel(*level).Enum() - } - - res, err := instance.grpc.Template.TemplateBuildStatus( - ctx, &templatemanagergrpc.TemplateStatusRequest{ - TemplateID: templateID, - BuildID: buildID, - Offset: &offset, - Limit: utils.ToPtr(uint32(limit)), - Level: lvlReq, - Start: timestamppb.New(start), - End: timestamppb.New(end), - Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), - }, - ) - if err != nil { - telemetry.ReportError(ctx, "error when returning logs for template build", err) - logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) - - return nil, err - } - - raw := res.GetLogEntries() - - // Add an extra newline to each log entry to ensure proper formatting in the CLI - entries := make([]logs.LogEntry, len(raw)) - for i, entry := range raw { - entries[i] = logs.LogEntry{ - Timestamp: entry.GetTimestamp().AsTime(), - Message: entry.GetMessage(), - Level: logs.LogLevel(entry.GetLevel()), - Fields: entry.GetFields(), - } - } - - return entries, nil + if !found { + return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) } - } - logs, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) - if err != nil { - return nil, fmt.Errorf("error when fetching build logs: %w", err) + entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + if err != nil { + return nil, fmt.Errorf("error getting build logs from node: %w", err) + } + + return entries, nil } - return logs, nil -} + // Fetch logs from Loki backend + if logCheckSourceType(source, api.LogsSourcePersistent) { + lokiDirection := defaultDirection + if direction == api.LogsDirectionBackward { + lokiDirection = logproto.BACKWARD + } -func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection { - switch direction { - case api.LogsDirectionForward: - return templatemanagergrpc.LogsDirection_Forward - case api.LogsDirectionBackward: - return templatemanagergrpc.LogsDirection_Backward - default: - return templatemanagergrpc.LogsDirection_Forward + entries, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) + if err != nil { + return nil, fmt.Errorf("error when fetching build logs: %w", err) + } + + return entries, nil } + + return nil, nil } diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index b5370702eb..a97fd9e0ca 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -123,8 +123,53 @@ func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID return api.SandboxLogs{Logs: l, LogEntries: le}, nil } -func (r *ClusterResourceProviderImpl) GetBuildLogs(ctx context.Context, _ *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) { +func (r *ClusterResourceProviderImpl) GetBuildLogs( + ctx context.Context, + nodeID *string, + templateID string, + buildID string, + offset int32, + limit int32, + level *logs.LogLevel, + cursor *time.Time, + direction api.LogsDirection, + source *api.LogsSource, +) ([]logs.LogEntry, error) { + ctx, span := tracer.Start(ctx, "get build-logs") + defer span.End() + start, end := logQueryWindow(cursor, direction) + + // Fetch logs directly from template builder instance + if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { + instance, found := r.instances.Get(*nodeID) + if !found { + return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) + } + + entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + if err != nil { + return nil, fmt.Errorf("error getting build logs from node: %w", err) + } + + return entries, nil + } + + // Fetch logs from edge API + if logCheckSourceType(source, api.LogsSourcePersistent) { + // Fetch logs from edge API + entries, err := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction) + if err != nil { + return nil, fmt.Errorf("error getting build logs from edge API: %w", err) + } + + return entries, nil + } + + return nil, nil +} + +func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { res, err := r.client.V1TemplateBuildLogsWithResponse( ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{ TemplateID: templateID, @@ -161,13 +206,3 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs(ctx context.Context, _ *strin return l, nil } - -func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { - if level == nil { - return nil - } - - value := edgeapi.LogLevel(logs.LevelToString(*level)) - - return &value -} From 41045e8bcad9bd6ee4b36a21f505a021e6625ba3 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 22 Dec 2025 18:31:39 +0100 Subject: [PATCH 27/58] Code review fixes --- packages/api/internal/clusters/instances_sync.go | 10 ++++++++++ packages/api/internal/clusters/resources_remote.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 1d90d6e30a..b7827ec599 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -119,5 +119,15 @@ func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) logger.WithServiceInstanceID(instance.InstanceID), ) + closeErr := instance.Close() + if closeErr != nil { + logger.L().Error(ctx, "Failed to close cluster instance after sync failure", + zap.Error(closeErr), + logger.WithClusterID(d.clusterID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) + } + d.instances.Remove(instance.NodeID) } diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index a97fd9e0ca..af449e908e 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -88,7 +88,7 @@ func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, t } } - return map[string]api.SandboxMetric{}, nil + return items, nil } func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, limit *int32) (api.SandboxLogs, error) { From 60532992648042c0e468a53215cf4ed4a8f0a324 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Sun, 28 Dec 2025 19:23:15 +0100 Subject: [PATCH 28/58] Fixed place where local ctx was not used --- packages/api/internal/clusters/discovery/local.go | 2 +- packages/api/internal/clusters/discovery/remote.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/clusters/discovery/local.go b/packages/api/internal/clusters/discovery/local.go index 54a1431fcd..55c42d87c3 100644 --- a/packages/api/internal/clusters/discovery/local.go +++ b/packages/api/internal/clusters/discovery/local.go @@ -30,7 +30,7 @@ func NewLocalDiscovery(clusterID uuid.UUID, nomad *nomadapi.Client) Discovery { } func (sd *LocalServiceDiscovery) Query(ctx context.Context) ([]Item, error) { - _, span := tracer.Start(ctx, "query-local-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) + ctx, span := tracer.Start(ctx, "query-local-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) defer span.End() // Static discovery for local environment diff --git a/packages/api/internal/clusters/discovery/remote.go b/packages/api/internal/clusters/discovery/remote.go index 95fc22ab52..9470fa4b34 100644 --- a/packages/api/internal/clusters/discovery/remote.go +++ b/packages/api/internal/clusters/discovery/remote.go @@ -26,7 +26,7 @@ func NewRemoteServiceDiscovery(clusterID uuid.UUID, client *api.ClientWithRespon } func (sd *RemoteServiceDiscovery) Query(ctx context.Context) ([]Item, error) { - _, span := tracer.Start(ctx, "query-remote-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) + ctx, span := tracer.Start(ctx, "query-remote-cluster-nodes", trace.WithAttributes(telemetry.WithClusterID(sd.clusterID))) defer span.End() res, err := sd.client.V1ServiceDiscoveryGetOrchestratorsWithResponse(ctx) From 423332f109a5f53cf084027ff634d9018ba68551 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Sun, 28 Dec 2025 19:30:06 +0100 Subject: [PATCH 29/58] Updated text for instance sync store struct --- packages/api/internal/clusters/instances_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index b7827ec599..6d892488a6 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -12,7 +12,7 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/smap" ) -// SynchronizationStore defines methods for synchronizing cluster instances +// Instance sync store handles synchronization of instances in each cluster and checking its state type instancesSyncStore struct { clusterID uuid.UUID From d03d51d592bedaa989c9ea971c758b645ebd0cab Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Sun, 28 Dec 2025 19:34:48 +0100 Subject: [PATCH 30/58] Unify duplicated logic for both instance create and instance sync --- .../api/internal/clusters/instances_sync.go | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 6d892488a6..21c0d0126a 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -75,24 +75,10 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item discovery.Item) return } - err = instance.Sync(ctx) - if err != nil { - closeErr := instance.Close() - if closeErr != nil { - logger.L().Error(ctx, "Failed to close cluster instance after sync failure", - zap.Error(closeErr), - logger.WithClusterID(d.clusterID), - logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), - ) - } - - logger.L().Error(ctx, "Failed to sync cluster instance during pool insert", - zap.Error(err), - logger.WithClusterID(d.clusterID), - logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), - ) + ok := d.tryToSyncInstance(ctx, instance) + if !ok { + // Try to gracefully close the instance + d.tryToCloseInstance(ctx, instance) return } @@ -101,15 +87,7 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item discovery.Item) } func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) { - err := instance.Sync(ctx) - if err != nil { - logger.L().Error(ctx, "Failed to sync cluster instance during pool update", - zap.Error(err), - logger.WithClusterID(d.clusterID), - logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), - ) - } + _ = d.tryToSyncInstance(ctx, instance) } func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) { @@ -119,6 +97,13 @@ func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) logger.WithServiceInstanceID(instance.InstanceID), ) + // Try to gracefully close the instance + d.tryToCloseInstance(ctx, instance) + + d.instances.Remove(instance.NodeID) +} + +func (d instancesSyncStore) tryToCloseInstance(ctx context.Context, instance *Instance) { closeErr := instance.Close() if closeErr != nil { logger.L().Error(ctx, "Failed to close cluster instance after sync failure", @@ -128,6 +113,20 @@ func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) logger.WithServiceInstanceID(instance.InstanceID), ) } +} - d.instances.Remove(instance.NodeID) +func (d instancesSyncStore) tryToSyncInstance(ctx context.Context, instance *Instance) bool { + err := instance.Sync(ctx) + if err != nil { + logger.L().Error(ctx, "Failed to sync cluster instance", + zap.Error(err), + logger.WithClusterID(d.clusterID), + logger.WithNodeID(instance.NodeID), + logger.WithServiceInstanceID(instance.InstanceID), + ) + + return false + } + + return true } From 54b0cd698fae53d3e0d7d6c87834f7849e305260 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Sun, 28 Dec 2025 19:37:08 +0100 Subject: [PATCH 31/58] Removed sandbox prefix from snadbox info struct Removed sandbox prefix on struct that is alreadt obviously related to sandbox informations. --- packages/api/internal/orchestrator/lifecycle.go | 8 ++++---- .../client-proxy/internal/edge-pass-through/events.go | 6 +++--- packages/shared/pkg/sandbox-catalog/catalog.go | 6 +++--- packages/shared/pkg/sandbox-catalog/catalog_memory.go | 2 +- packages/shared/pkg/sandbox-catalog/catalog_redis.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/api/internal/orchestrator/lifecycle.go b/packages/api/internal/orchestrator/lifecycle.go index 51689f0804..57ea556938 100644 --- a/packages/api/internal/orchestrator/lifecycle.go +++ b/packages/api/internal/orchestrator/lifecycle.go @@ -20,12 +20,12 @@ func (o *Orchestrator) addSandboxToRoutingTable(ctx context.Context, sandbox san OrchestratorID: node.Metadata().ServiceInstanceID, OrchestratorIP: node.IPAddress, - SandboxExecutionID: sandbox.ExecutionID, - SandboxStartedAt: sandbox.StartTime, - SandboxMaxLengthInHours: int64(sandbox.MaxInstanceLength / time.Hour), + ExecutionID: sandbox.ExecutionID, + StartedAt: sandbox.StartTime, + MaxLengthInHours: int64(sandbox.MaxInstanceLength / time.Hour), } - lifetime := time.Duration(info.SandboxMaxLengthInHours) * time.Hour + lifetime := time.Duration(info.MaxLengthInHours) * time.Hour err := o.routingCatalog.StoreSandbox(ctx, sandbox.SandboxID, &info, lifetime) if err != nil { logger.L().Error(ctx, "error adding routing record to catalog", zap.Error(err), logger.WithSandboxID(sandbox.SandboxID)) diff --git a/packages/client-proxy/internal/edge-pass-through/events.go b/packages/client-proxy/internal/edge-pass-through/events.go index 2617fdf745..6829d13167 100644 --- a/packages/client-proxy/internal/edge-pass-through/events.go +++ b/packages/client-proxy/internal/edge-pass-through/events.go @@ -54,9 +54,9 @@ func (s *NodePassThroughServer) catalogCreateEventHandler(ctx context.Context, m OrchestratorID: c.OrchestratorID, OrchestratorIP: o.GetInfo().IP, - SandboxExecutionID: c.ExecutionID, - SandboxStartedAt: c.SandboxStartTime, - SandboxMaxLengthInHours: c.SandboxMaxLengthInHours, + ExecutionID: c.ExecutionID, + StartedAt: c.SandboxStartTime, + MaxLengthInHours: c.SandboxMaxLengthInHours, }, time.Duration(c.SandboxMaxLengthInHours)*time.Hour, ) diff --git a/packages/shared/pkg/sandbox-catalog/catalog.go b/packages/shared/pkg/sandbox-catalog/catalog.go index 3ae5e7f761..7b82497f4a 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog.go +++ b/packages/shared/pkg/sandbox-catalog/catalog.go @@ -12,9 +12,9 @@ type SandboxInfo struct { OrchestratorID string `json:"orchestrator_id"` OrchestratorIP string `json:"orchestrator_ip"` // used only for cases where orchestrator is not registered in edge pool - SandboxExecutionID string `json:"execution_id"` - SandboxStartedAt time.Time `json:"sandbox_started_at"` // when sandbox was started - SandboxMaxLengthInHours int64 `json:"sandbox_max_length_in_hours"` // how long can sandbox can possibly run (in hours) + ExecutionID string `json:"execution_id"` + StartedAt time.Time `json:"sandbox_started_at"` // when sandbox was started + MaxLengthInHours int64 `json:"sandbox_max_length_in_hours"` // how long can sandbox can possibly run (in hours) } type SandboxesCatalog interface { diff --git a/packages/shared/pkg/sandbox-catalog/catalog_memory.go b/packages/shared/pkg/sandbox-catalog/catalog_memory.go index 8300eae6d2..b922117bc4 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_memory.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_memory.go @@ -71,7 +71,7 @@ func (c *MemorySandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID stri } // Different execution is stored in the cache, we don't want to remove it - if item.Value().SandboxExecutionID != executionID { + if item.Value().ExecutionID != executionID { return nil } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_redis.go b/packages/shared/pkg/sandbox-catalog/catalog_redis.go index 2e7bd5af41..7fc75986bc 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_redis.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_redis.go @@ -118,7 +118,7 @@ func (c *RedisSandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID strin } // Different execution is stored in the cache, we don't want to remove it - if info.SandboxExecutionID != executionID { + if info.ExecutionID != executionID { return nil } From 454d1b063a95e8627aef575984728bc9a7cbbd52 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Sun, 28 Dec 2025 19:40:07 +0100 Subject: [PATCH 32/58] Field rename --- packages/api/internal/clusters/cluster.go | 2 +- packages/api/internal/clusters/instance.go | 18 +++++++++--------- .../api/internal/clusters/instances_sync.go | 6 +++--- .../internal/orchestrator/nodemanager/node.go | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index ce96710123..a9293f1552 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -157,7 +157,7 @@ func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*Instance, bool) { for _, instance := range c.instances.Items() { - if instance.InstanceID == serviceInstanceID { + if instance.ServiceInstanceID == serviceInstanceID { return instance, true } } diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 0691541487..7cf4e5c5b6 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -23,10 +23,10 @@ type Instance struct { // Depending on service discovery used, it can be combination of different parameters, what service discovery gives us. UniqueIdentifier string - ClusterID uuid.UUID - NodeID string - InstanceID string + ClusterID uuid.UUID + NodeID string + ServiceInstanceID string ServiceVersion string ServiceVersionCommit string @@ -58,10 +58,10 @@ func newInstance( // For case with local cluster we will not receive instance ID from service discovery, but its not needed for proxy routing, // so it can be empty and will be filled after first sync. i := &Instance{ - UniqueIdentifier: sd.UniqueIdentifier, - NodeID: sd.NodeID, - InstanceID: sd.InstanceID, - ClusterID: clusterID, + UniqueIdentifier: sd.UniqueIdentifier, + NodeID: sd.NodeID, + ServiceInstanceID: sd.InstanceID, + ClusterID: clusterID, grpc: conn, mutex: sync.RWMutex{}, @@ -76,7 +76,7 @@ func newInstance( zap.Error(err), logger.WithNodeID(i.NodeID), logger.WithClusterID(i.ClusterID), - logger.WithServiceInstanceID(i.InstanceID), + logger.WithServiceInstanceID(i.ServiceInstanceID), ) } @@ -102,7 +102,7 @@ func (i *Instance) Sync(ctx context.Context) error { i.roles = info.GetServiceRoles() i.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) - i.InstanceID = info.GetServiceId() + i.ServiceInstanceID = info.GetServiceId() i.ServiceVersion = info.GetServiceVersion() i.ServiceVersionCommit = info.GetServiceCommit() diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 21c0d0126a..49fb1b0ab3 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -94,7 +94,7 @@ func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) logger.L().Info(ctx, "Removing instance from cluster pool", logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), + logger.WithServiceInstanceID(instance.ServiceInstanceID), ) // Try to gracefully close the instance @@ -110,7 +110,7 @@ func (d instancesSyncStore) tryToCloseInstance(ctx context.Context, instance *In zap.Error(closeErr), logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), + logger.WithServiceInstanceID(instance.ServiceInstanceID), ) } } @@ -122,7 +122,7 @@ func (d instancesSyncStore) tryToSyncInstance(ctx context.Context, instance *Ins zap.Error(err), logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.InstanceID), + logger.WithServiceInstanceID(instance.ServiceInstanceID), ) return false diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index b4b8af7184..b2f6200c6d 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -123,7 +123,7 @@ func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID go buildCache.Start() nodeMetadata := NodeMetadata{ - ServiceInstanceID: i.InstanceID, + ServiceInstanceID: i.ServiceInstanceID, Commit: i.ServiceVersionCommit, Version: i.ServiceVersion, } From 54f59a7cbe75f3ac79f57f93bba71e4b8776cd13 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 29 Dec 2025 17:19:11 +0100 Subject: [PATCH 33/58] Fix issue with concurrent access to instance fields Before it was possible that at same time instances service id/version/commit was read in one thread and updated in another thread. This can cause issue with string headers. This commit fixes by making fields on instance struct private and use getter that copy data to new variables every time. --- packages/api/internal/clusters/cluster.go | 9 ++-- packages/api/internal/clusters/instance.go | 43 +++++++++++++------ .../api/internal/clusters/instances_sync.go | 11 +++-- .../internal/orchestrator/nodemanager/node.go | 20 ++++----- 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index a9293f1552..1a23758147 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -148,7 +148,8 @@ func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { return nil, ErrTemplateBuilderNotFound } - if instance.GetStatus() == infogrpc.ServiceInfoStatus_Unhealthy || !instance.IsBuilder() { + info := instance.GetInfo() + if info.Status == infogrpc.ServiceInfoStatus_Unhealthy || !instance.IsBuilder() { return nil, ErrTemplateBuilderNotFound } @@ -157,7 +158,8 @@ func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*Instance, bool) { for _, instance := range c.instances.Items() { - if instance.ServiceInstanceID == serviceInstanceID { + info := instance.GetInfo() + if info.ServiceInstanceID == serviceInstanceID { return instance, true } } @@ -179,7 +181,8 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e rand.Shuffle(len(instances), func(i, j int) { instances[i], instances[j] = instances[j], instances[i] }) for _, instance := range instances { - if instance.GetStatus() != infogrpc.ServiceInfoStatus_Healthy { + info := instance.GetInfo() + if info.Status != infogrpc.ServiceInfoStatus_Healthy { continue } diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 7cf4e5c5b6..ca06b73e9a 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -21,14 +21,14 @@ import ( type Instance struct { // Identifier that uniquely identifies the instance so it will not be registered multiple times. // Depending on service discovery used, it can be combination of different parameters, what service discovery gives us. - UniqueIdentifier string + uniqueIdentifier string ClusterID uuid.UUID NodeID string - ServiceInstanceID string - ServiceVersion string - ServiceVersionCommit string + serviceInstanceID string + serviceVersion string + serviceVersionCommit string roles []infogrpc.ServiceInfoRole machineInfo machineinfo.MachineInfo @@ -38,6 +38,15 @@ type Instance struct { mutex sync.RWMutex } +// InstanceInfo contains synchronized instance information +type InstanceInfo struct { + ServiceInstanceID string + ServiceVersion string + ServiceVersionCommit string + Status infogrpc.ServiceInfoStatus + Roles []infogrpc.ServiceInfoRole +} + func newInstance( ctx context.Context, tel *telemetry.Client, @@ -58,10 +67,10 @@ func newInstance( // For case with local cluster we will not receive instance ID from service discovery, but its not needed for proxy routing, // so it can be empty and will be filled after first sync. i := &Instance{ - UniqueIdentifier: sd.UniqueIdentifier, + uniqueIdentifier: sd.UniqueIdentifier, NodeID: sd.NodeID, - ServiceInstanceID: sd.InstanceID, ClusterID: clusterID, + serviceInstanceID: sd.InstanceID, grpc: conn, mutex: sync.RWMutex{}, @@ -76,7 +85,7 @@ func newInstance( zap.Error(err), logger.WithNodeID(i.NodeID), logger.WithClusterID(i.ClusterID), - logger.WithServiceInstanceID(i.ServiceInstanceID), + logger.WithServiceInstanceID(i.serviceInstanceID), ) } @@ -102,25 +111,31 @@ func (i *Instance) Sync(ctx context.Context) error { i.roles = info.GetServiceRoles() i.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) - i.ServiceInstanceID = info.GetServiceId() - i.ServiceVersion = info.GetServiceVersion() - i.ServiceVersionCommit = info.GetServiceCommit() + i.serviceInstanceID = info.GetServiceId() + i.serviceVersion = info.GetServiceVersion() + i.serviceVersionCommit = info.GetServiceCommit() return nil } -func (i *Instance) GetStatus() infogrpc.ServiceInfoStatus { +func (i *Instance) GetMachineInfo() machineinfo.MachineInfo { i.mutex.RLock() defer i.mutex.RUnlock() - return i.status + return i.machineInfo } -func (i *Instance) GetMachineInfo() machineinfo.MachineInfo { +func (i *Instance) GetInfo() InstanceInfo { i.mutex.RLock() defer i.mutex.RUnlock() - return i.machineInfo + return InstanceInfo{ + ServiceInstanceID: i.serviceInstanceID, + ServiceVersion: i.serviceVersion, + ServiceVersionCommit: i.serviceVersionCommit, + Status: i.status, + Roles: slices.Clone(i.roles), + } } func (i *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 49fb1b0ab3..1b9c5ff686 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -33,7 +33,7 @@ func (d instancesSyncStore) SourceList(ctx context.Context) ([]discovery.Item, e func (d instancesSyncStore) SourceExists(_ context.Context, s []discovery.Item, p *Instance) bool { for _, item := range s { // With comparing unique identifier that should ensure we are not re-adding same instance again - if item.UniqueIdentifier == p.UniqueIdentifier { + if item.UniqueIdentifier == p.uniqueIdentifier { return true } } @@ -91,10 +91,11 @@ func (d instancesSyncStore) PoolUpdate(ctx context.Context, instance *Instance) } func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) { + info := instance.GetInfo() logger.L().Info(ctx, "Removing instance from cluster pool", logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.ServiceInstanceID), + logger.WithServiceInstanceID(info.ServiceInstanceID), ) // Try to gracefully close the instance @@ -106,11 +107,12 @@ func (d instancesSyncStore) PoolRemove(ctx context.Context, instance *Instance) func (d instancesSyncStore) tryToCloseInstance(ctx context.Context, instance *Instance) { closeErr := instance.Close() if closeErr != nil { + info := instance.GetInfo() logger.L().Error(ctx, "Failed to close cluster instance after sync failure", zap.Error(closeErr), logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.ServiceInstanceID), + logger.WithServiceInstanceID(info.ServiceInstanceID), ) } } @@ -118,11 +120,12 @@ func (d instancesSyncStore) tryToCloseInstance(ctx context.Context, instance *In func (d instancesSyncStore) tryToSyncInstance(ctx context.Context, instance *Instance) bool { err := instance.Sync(ctx) if err != nil { + info := instance.GetInfo() logger.L().Error(ctx, "Failed to sync cluster instance", zap.Error(err), logger.WithClusterID(d.clusterID), logger.WithNodeID(instance.NodeID), - logger.WithServiceInstanceID(instance.ServiceInstanceID), + logger.WithServiceInstanceID(info.ServiceInstanceID), ) return false diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index b2f6200c6d..2f163ec672 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -113,19 +113,20 @@ func New( } func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID uuid.UUID, sandboxDomain *string, i *clusters.Instance) (*Node, error) { - nodeStatus, ok := OrchestratorToApiNodeStateMapper[i.GetStatus()] + info := i.GetInfo() + status, ok := OrchestratorToApiNodeStateMapper[info.Status] if !ok { - logger.L().Error(ctx, "Unknown service info status", zap.String("status", i.GetStatus().String()), logger.WithNodeID(i.NodeID)) - nodeStatus = api.NodeStatusUnhealthy + logger.L().Error(ctx, "Unknown service info status", zap.String("status", info.Status.String()), logger.WithNodeID(i.NodeID)) + status = api.NodeStatusUnhealthy } buildCache := ttlcache.New[string, any]() go buildCache.Start() nodeMetadata := NodeMetadata{ - ServiceInstanceID: i.ServiceInstanceID, - Commit: i.ServiceVersionCommit, - Version: i.ServiceVersion, + ServiceInstanceID: info.ServiceInstanceID, + Commit: info.ServiceVersionCommit, + Version: info.ServiceVersion, } n := &Node{ @@ -142,7 +143,7 @@ func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID }, connection: client, - status: nodeStatus, + status: status, meta: nodeMetadata, buildCache: buildCache, @@ -165,13 +166,12 @@ func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID func (n *Node) Close(ctx context.Context) error { if n.IsNomadManaged() { logger.L().Info(ctx, "Closing local node", logger.WithNodeID(n.ID)) - err := n.connection.Close() - if err != nil { + if err := n.connection.Close(); err != nil { logger.L().Error(ctx, "Error closing connection to node", zap.Error(err), logger.WithNodeID(n.ID)) } } else { logger.L().Info(ctx, "Closing cluster node", logger.WithNodeID(n.ID), logger.WithClusterID(n.ClusterID)) - // We are not closing grpc connection, because it is shared between all cluster nodes, and it's handled by the cluster + // We are not closing grpc connection, because it is managed by cluster instance } n.buildCache.Stop() From 001e4ed3d21ddb70145b2d7de8ee6f7f3816a78d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 29 Dec 2025 17:28:10 +0100 Subject: [PATCH 34/58] Removed roles getters and replaced with pre-cached values Before for checking if instance is orchestrator and/or builder array contains search was needed. Now this is pre-cached during instance sync call and each time value is just returned and used. --- packages/api/internal/clusters/cluster.go | 12 +--- packages/api/internal/clusters/instance.go | 58 +++++++++---------- .../api/internal/clusters/instance_client.go | 2 +- 3 files changed, 30 insertions(+), 42 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 1a23758147..137931c0bd 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -148,8 +148,7 @@ func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*Instance, error) { return nil, ErrTemplateBuilderNotFound } - info := instance.GetInfo() - if info.Status == infogrpc.ServiceInfoStatus_Unhealthy || !instance.IsBuilder() { + if info := instance.GetInfo(); info.Status == infogrpc.ServiceInfoStatus_Unhealthy || !info.IsBuilder { return nil, ErrTemplateBuilderNotFound } @@ -181,12 +180,7 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e rand.Shuffle(len(instances), func(i, j int) { instances[i], instances[j] = instances[j], instances[i] }) for _, instance := range instances { - info := instance.GetInfo() - if info.Status != infogrpc.ServiceInfoStatus_Healthy { - continue - } - - if !instance.IsBuilder() { + if info := instance.GetInfo(); info.Status != infogrpc.ServiceInfoStatus_Healthy || !info.IsBuilder { continue } @@ -199,7 +193,7 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e func (c *Cluster) GetOrchestrators() []*Instance { instances := make([]*Instance, 0) for _, i := range c.instances.Items() { - if i.IsOrchestrator() { + if i.GetInfo().IsOrchestrator { instances = append(instances, i) } } diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index ca06b73e9a..83404ce89f 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -30,12 +30,14 @@ type Instance struct { serviceVersion string serviceVersionCommit string - roles []infogrpc.ServiceInfoRole - machineInfo machineinfo.MachineInfo - grpc *GRPCClient - - status infogrpc.ServiceInfoStatus - mutex sync.RWMutex + client *GRPCClient + status infogrpc.ServiceInfoStatus + machine machineinfo.MachineInfo + roles []infogrpc.ServiceInfoRole + isBuilder bool + isOrchestrator bool + + mutex sync.RWMutex } // InstanceInfo contains synchronized instance information @@ -44,7 +46,8 @@ type InstanceInfo struct { ServiceVersion string ServiceVersionCommit string Status infogrpc.ServiceInfoStatus - Roles []infogrpc.ServiceInfoRole + IsOrchestrator bool + IsBuilder bool } func newInstance( @@ -58,7 +61,7 @@ func newInstance( ) (*Instance, error) { conn, err := createConnection(tel, clusterAuth, connAddr, connTls) if err != nil { - return nil, fmt.Errorf("failed to create cluster instance grpc client: %w", err) + return nil, fmt.Errorf("failed to create cluster instance client client: %w", err) } // Create with default values that will be updated on sync before returning the instance, @@ -68,12 +71,12 @@ func newInstance( // so it can be empty and will be filled after first sync. i := &Instance{ uniqueIdentifier: sd.UniqueIdentifier, + serviceInstanceID: sd.InstanceID, NodeID: sd.NodeID, ClusterID: clusterID, - serviceInstanceID: sd.InstanceID, - grpc: conn, - mutex: sync.RWMutex{}, + client: conn, + mutex: sync.RWMutex{}, } err = i.Sync(ctx) @@ -98,7 +101,7 @@ func newInstance( // Sync function can be called on freshly initialized instance to populate its data // In initial case its possible that service instance id needed for proper remote cluster routing is not yet set. func (i *Instance) Sync(ctx context.Context) error { - info, err := i.grpc.Info.ServiceInfo(ctx, &emptypb.Empty{}) + info, err := i.client.Info.ServiceInfo(ctx, &emptypb.Empty{}) err = utils.UnwrapGRPCError(err) if err != nil { return err @@ -109,12 +112,17 @@ func (i *Instance) Sync(ctx context.Context) error { i.status = info.GetServiceStatus() i.roles = info.GetServiceRoles() - i.machineInfo = machineinfo.FromGRPCInfo(info.GetMachineInfo()) + i.machine = machineinfo.FromGRPCInfo(info.GetMachineInfo()) i.serviceInstanceID = info.GetServiceId() i.serviceVersion = info.GetServiceVersion() i.serviceVersionCommit = info.GetServiceCommit() + // We don't want to check array every time, + // this is why we are caching value during instance sync. + i.isBuilder = slices.Contains(i.roles, infogrpc.ServiceInfoRole_TemplateBuilder) + i.isOrchestrator = slices.Contains(i.roles, infogrpc.ServiceInfoRole_Orchestrator) + return nil } @@ -122,7 +130,7 @@ func (i *Instance) GetMachineInfo() machineinfo.MachineInfo { i.mutex.RLock() defer i.mutex.RUnlock() - return i.machineInfo + return i.machine } func (i *Instance) GetInfo() InstanceInfo { @@ -134,29 +142,15 @@ func (i *Instance) GetInfo() InstanceInfo { ServiceVersion: i.serviceVersion, ServiceVersionCommit: i.serviceVersionCommit, Status: i.status, - Roles: slices.Clone(i.roles), + IsOrchestrator: i.isOrchestrator, + IsBuilder: i.isBuilder, } } -func (i *Instance) hasRole(r infogrpc.ServiceInfoRole) bool { - i.mutex.RLock() - defer i.mutex.RUnlock() - - return slices.Contains(i.roles, r) -} - -func (i *Instance) IsBuilder() bool { - return i.hasRole(infogrpc.ServiceInfoRole_TemplateBuilder) -} - -func (i *Instance) IsOrchestrator() bool { - return i.hasRole(infogrpc.ServiceInfoRole_Orchestrator) -} - func (i *Instance) GetConnection() *GRPCClient { - return i.grpc + return i.client } func (i *Instance) Close() error { - return i.grpc.Close() + return i.client.Close() } diff --git a/packages/api/internal/clusters/instance_client.go b/packages/api/internal/clusters/instance_client.go index ac7a6f5de4..9a26ef14ae 100644 --- a/packages/api/internal/clusters/instance_client.go +++ b/packages/api/internal/clusters/instance_client.go @@ -64,7 +64,7 @@ func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoi conn, err := grpc.NewClient(endpoint, grpcOptions...) if err != nil { - return nil, fmt.Errorf("failed to create grpc client: %w", err) + return nil, fmt.Errorf("failed to create client client: %w", err) } return &GRPCClient{ From 5cfd8482c4d29324600059ad8787b890c9143b0c Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 30 Dec 2025 10:55:39 +0000 Subject: [PATCH 35/58] Log close err --- packages/api/internal/clusters/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 83404ce89f..789b4a24e6 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -85,7 +85,7 @@ func newInstance( if closeErr != nil { logger.L().Error( ctx, "Failed to close gRPC Connection after instance sync failure", - zap.Error(err), + zap.Error(closeErr), logger.WithNodeID(i.NodeID), logger.WithClusterID(i.ClusterID), logger.WithServiceInstanceID(i.serviceInstanceID), From b35d2ff78fb98cdf6cb06055f12639ce145aad0b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 30 Dec 2025 11:10:53 +0000 Subject: [PATCH 36/58] Explicitly check if res body parsed struct is nil --- .../api/internal/clusters/resources_remote.go | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index af449e908e..9e430e9720 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -7,11 +7,8 @@ import ( "net/http" "time" - "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/api/internal/api" edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/utils" @@ -45,6 +42,10 @@ func (r *ClusterResourceProviderImpl) GetSandboxMetrics(ctx context.Context, tea return nil, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) } + if res.JSON200 == nil { + return nil, errors.New("request returned nil response") + } + raw := *res.JSON200 items := make([]api.SandboxMetric, len(raw)) for i, m := range raw { @@ -73,6 +74,10 @@ func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, t return nil, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) } + if res.JSON200 == nil { + return nil, errors.New("request returned nil response") + } + raw := *res.JSON200 items := make(map[string]api.SandboxMetric, len(raw.Sandboxes)) for sbxID, v := range raw.Sandboxes { @@ -101,6 +106,10 @@ func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID return api.SandboxLogs{}, fmt.Errorf("unexpected response with HTTP status '%d'", res.StatusCode()) } + if res.JSON200 == nil { + return api.SandboxLogs{}, errors.New("request returned nil response") + } + raw := *res.JSON200 l := make([]api.SandboxLog, len(raw.Logs)) for i, row := range raw.Logs { @@ -188,11 +197,13 @@ func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, } if res.StatusCode() != 200 || res.JSON200 == nil { - logger.L().Error(ctx, "failed to get build logs in template manager", zap.String("body", string(res.Body))) - return nil, errors.New("failed to get build logs in template manager") } + if res.JSON200 == nil { + return nil, errors.New("request returned nil response") + } + raw := *res.JSON200 l := make([]logs.LogEntry, len(raw.LogEntries)) for i, entry := range raw.LogEntries { From e87b9a95d1e5bbe50931edee170bb506ebf05d48 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 30 Dec 2025 11:31:34 +0000 Subject: [PATCH 37/58] Close instances during cluster close --- packages/api/internal/clusters/cluster.go | 23 ++++++++++++++++++- .../api/internal/clusters/clusters_sync.go | 14 +++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 137931c0bd..30d3dc9b30 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -6,17 +6,20 @@ import ( "fmt" "math/rand/v2" "net/http" + "sync" "time" "github.com/google/uuid" nomadapi "github.com/hashicorp/nomad/api" "go.opentelemetry.io/otel" + "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/clusters/discovery" clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/shared/pkg/consts" infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" @@ -136,9 +139,27 @@ func newRemoteCluster( return c, nil } -func (c *Cluster) Close() error { +func (c *Cluster) Close(ctx context.Context) error { c.synchronization.Close() + instances := c.instances.Items() + wg := sync.WaitGroup{} + + for _, instance := range instances { + wg.Go(func() { + if closeErr := instance.Close(); closeErr != nil { + logger.L().Error(ctx, "Failed to close cluster instance during cluster closing", + zap.Error(closeErr), + logger.WithClusterID(c.ID), + logger.WithNodeID(instance.NodeID), + ) + } + }) + } + + // Wait for all instances to be closed + wg.Wait() + return nil } diff --git a/packages/api/internal/clusters/clusters_sync.go b/packages/api/internal/clusters/clusters_sync.go index a295409dc3..7ec2ce4e05 100644 --- a/packages/api/internal/clusters/clusters_sync.go +++ b/packages/api/internal/clusters/clusters_sync.go @@ -94,15 +94,13 @@ func (p *Pool) Close(ctx context.Context) { wg := &sync.WaitGroup{} for _, cluster := range p.clusters.Items() { - wg.Add(1) - go func(c *Cluster) { - defer wg.Done() - logger.L().Info(ctx, "Closing cluster", logger.WithClusterID(c.ID)) - err := c.Close() + wg.Go(func() { + logger.L().Info(ctx, "Closing cluster", logger.WithClusterID(cluster.ID)) + err := cluster.Close(ctx) if err != nil { - logger.L().Error(ctx, "Error closing cluster", zap.Error(err), logger.WithClusterID(c.ID)) + logger.L().Error(ctx, "Error closing cluster", zap.Error(err), logger.WithClusterID(cluster.ID)) } - }(cluster) + }) } wg.Wait() } @@ -204,7 +202,7 @@ func (d clustersSyncStore) PoolUpdate(_ context.Context, _ *Cluster) { func (d clustersSyncStore) PoolRemove(ctx context.Context, cluster *Cluster) { logger.L().Info(ctx, "Removing cluster from pool", logger.WithClusterID(cluster.ID)) - err := cluster.Close() + err := cluster.Close(ctx) if err != nil { logger.L().Error(ctx, "Error during removing cluster from pool", zap.Error(err), logger.WithClusterID(cluster.ID)) } From b49d6f406ee0ba7f051a01890143a3afefc3e123 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 30 Dec 2025 11:31:48 +0000 Subject: [PATCH 38/58] Removed double check of res body struct is nil --- packages/api/internal/clusters/resources_remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index 9e430e9720..aea204d298 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -196,7 +196,7 @@ func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, return nil, fmt.Errorf("failed to get build logs in template manager: %w", err) } - if res.StatusCode() != 200 || res.JSON200 == nil { + if res.StatusCode() != 200 { return nil, errors.New("failed to get build logs in template manager") } From eeeb5b6c1deba6985a363b7fb6833d531744df0d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 10:38:07 +0100 Subject: [PATCH 39/58] Remove file after conflict resolved --- packages/api/internal/edge/cluster.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/api/internal/edge/cluster.go diff --git a/packages/api/internal/edge/cluster.go b/packages/api/internal/edge/cluster.go deleted file mode 100644 index e69de29bb2..0000000000 From fa2b8fb78a43a9134a56bde2da191fa30b54b22f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 11:39:42 +0100 Subject: [PATCH 40/58] Use callback system for build logs query To replicate previous logic of fetching build logs with different source settings but still support fallback i re-do how logs are returned via callback where you can have multipel callback for each call. --- packages/api/internal/clusters/resources.go | 78 ++++++++------- .../api/internal/clusters/resources_local.go | 40 ++++++-- .../api/internal/clusters/resources_remote.go | 97 ++++++++++--------- 3 files changed, 124 insertions(+), 91 deletions(-) diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go index aa51308bdd..fc1c83437a 100644 --- a/packages/api/internal/clusters/resources.go +++ b/packages/api/internal/clusters/resources.go @@ -55,10 +55,6 @@ func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templat } } -func logCheckSourceType(source *api.LogsSource, sourceType api.LogsSource) bool { - return source == nil || *source == sourceType -} - func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { if level == nil { return nil @@ -69,41 +65,49 @@ func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel { return &value } -func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { - var lvlReq *templatemanagergrpc.LogLevel - if level != nil { - lvlReq = templatemanagergrpc.LogLevel(*level).Enum() - } +func logCheckSourceType(source *api.LogsSource, sourceType api.LogsSource) bool { + return source == nil || *source == sourceType +} - res, err := instance.GetConnection().Template.TemplateBuildStatus( - ctx, &templatemanagergrpc.TemplateStatusRequest{ - TemplateID: templateID, - BuildID: buildID, - Offset: &offset, - Limit: utils.ToPtr(uint32(limit)), - Level: lvlReq, - Start: timestamppb.New(start), - End: timestamppb.New(end), - Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), - }, - ) - if err != nil { - telemetry.ReportError(ctx, "error when returning logs for template build", err) - logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) - - return nil, err - } +type logSourceFunc func() ([]logs.LogEntry, error) - raw := res.GetLogEntries() - entries := make([]logs.LogEntry, len(raw)) - for i, entry := range res.GetLogEntries() { - entries[i] = logs.LogEntry{ - Timestamp: entry.GetTimestamp().AsTime(), - Message: entry.GetMessage(), - Level: logs.LogLevel(entry.GetLevel()), - Fields: entry.GetFields(), +func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) logSourceFunc { + return func() ([]logs.LogEntry, error) { + var lvlReq *templatemanagergrpc.LogLevel + if level != nil { + lvlReq = templatemanagergrpc.LogLevel(*level).Enum() + } + + res, err := instance.GetConnection().Template.TemplateBuildStatus( + ctx, &templatemanagergrpc.TemplateStatusRequest{ + TemplateID: templateID, + BuildID: buildID, + Offset: &offset, + Limit: utils.ToPtr(uint32(limit)), + Level: lvlReq, + Start: timestamppb.New(start), + End: timestamppb.New(end), + Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)), + }, + ) + if err != nil { + telemetry.ReportError(ctx, "error when returning logs for template build", err) + logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID)) + + return nil, err } - } - return entries, nil + raw := res.GetLogEntries() + entries := make([]logs.LogEntry, len(raw)) + for i, entry := range res.GetLogEntries() { + entries[i] = logs.LogEntry{ + Timestamp: entry.GetTimestamp().AsTime(), + Message: entry.GetMessage(), + Level: logs.LogLevel(entry.GetLevel()), + Fields: entry.GetFields(), + } + } + + return entries, nil + } } diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go index 326edbb0d2..363813364f 100644 --- a/packages/api/internal/clusters/resources_local.go +++ b/packages/api/internal/clusters/resources_local.go @@ -152,33 +152,42 @@ func (l *LocalClusterResourceProvider) GetBuildLogs( direction api.LogsDirection, source *api.LogsSource, ) ([]logs.LogEntry, error) { + ctx, span := tracer.Start(ctx, "get build-logs") + defer span.End() + start, end := logQueryWindow(cursor, direction) - // Fetch logs directly from template builder instance + var sources []logSourceFunc + if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { instance, found := l.instances.Get(*nodeID) if !found { return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) } - entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - if err != nil { - return nil, fmt.Errorf("error getting build logs from node: %w", err) - } - - return entries, nil + sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) } - // Fetch logs from Loki backend + // Fetch logs from Loki backend (explicit persistent source or default) if logCheckSourceType(source, api.LogsSourcePersistent) { lokiDirection := defaultDirection if direction == api.LogsDirectionBackward { lokiDirection = logproto.BACKWARD } - entries, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) + sourceCallback := l.logsFromLocalLoki(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) + sources = append(sources, sourceCallback) + } + + // Iterate through sources and return the first successful fetch, + // it depends on nodeID and source what sources are available here. + for _, sourceFetch := range sources { + entries, err := sourceFetch() if err != nil { - return nil, fmt.Errorf("error when fetching build logs: %w", err) + logger.L().Warn(ctx, "Error fetching build logs", logger.WithTemplateID(templateID), logger.WithBuildID(buildID), zap.Error(err)) + + continue } return entries, nil @@ -186,3 +195,14 @@ func (l *LocalClusterResourceProvider) GetBuildLogs( return nil, nil } + +func (l *LocalClusterResourceProvider) logsFromLocalLoki(ctx context.Context, templateID string, buildID string, start time.Time, end time.Time, limit int, offset int32, level *logs.LogLevel, direction logproto.Direction) logSourceFunc { + return func() ([]logs.LogEntry, error) { + entries, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, limit, offset, level, direction) + if err != nil { + return nil, fmt.Errorf("error when fetching build logs from Loki: %w", err) + } + + return entries, nil + } +} diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index aea204d298..04789ed8db 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -7,8 +7,11 @@ import ( "net/http" "time" + "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/api/internal/api" edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/utils" @@ -149,27 +152,31 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs( start, end := logQueryWindow(cursor, direction) - // Fetch logs directly from template builder instance + var sources []logSourceFunc + if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { instance, found := r.instances.Get(*nodeID) if !found { return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) } - entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - if err != nil { - return nil, fmt.Errorf("error getting build logs from node: %w", err) - } - - return entries, nil + sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) } - // Fetch logs from edge API if logCheckSourceType(source, api.LogsSourcePersistent) { - // Fetch logs from edge API - entries, err := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction) + sourceCallback := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) + } + + // Iterate through sources and return the first successful fetch, + // it depends on nodeID and source what sources are available here. + for _, sourceFetch := range sources { + entries, err := sourceFetch() if err != nil { - return nil, fmt.Errorf("error getting build logs from edge API: %w", err) + logger.L().Warn(ctx, "Error fetching build logs", logger.WithTemplateID(templateID), logger.WithBuildID(buildID), zap.Error(err)) + + continue } return entries, nil @@ -178,42 +185,44 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs( return nil, nil } -func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) { - res, err := r.client.V1TemplateBuildLogsWithResponse( - ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{ - TemplateID: templateID, - Offset: &offset, - Limit: &limit, - Level: logToEdgeLevel(level), - // TODO: remove this once the API spec is not required to have orchestratorID (https://linear.app/e2b/issue/ENG-3352) - OrchestratorID: utils.ToPtr("unused"), - Start: utils.ToPtr(start.UnixMilli()), - End: utils.ToPtr(end.UnixMilli()), - Direction: utils.ToPtr(edgeapi.V1TemplateBuildLogsParamsDirection(direction)), - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to get build logs in template manager: %w", err) - } +func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) logSourceFunc { + return func() ([]logs.LogEntry, error) { + res, err := r.client.V1TemplateBuildLogsWithResponse( + ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{ + TemplateID: templateID, + Offset: &offset, + Limit: &limit, + Level: logToEdgeLevel(level), + // TODO: remove this once the API spec is not required to have orchestratorID (https://linear.app/e2b/issue/ENG-3352) + OrchestratorID: utils.ToPtr("unused"), + Start: utils.ToPtr(start.UnixMilli()), + End: utils.ToPtr(end.UnixMilli()), + Direction: utils.ToPtr(edgeapi.V1TemplateBuildLogsParamsDirection(direction)), + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to get build logs in template manager: %w", err) + } - if res.StatusCode() != 200 { - return nil, errors.New("failed to get build logs in template manager") - } + if res.StatusCode() != 200 { + return nil, errors.New("failed to get build logs in template manager") + } - if res.JSON200 == nil { - return nil, errors.New("request returned nil response") - } + if res.JSON200 == nil { + return nil, errors.New("request returned nil response") + } - raw := *res.JSON200 - l := make([]logs.LogEntry, len(raw.LogEntries)) - for i, entry := range raw.LogEntries { - l[i] = logs.LogEntry{ - Timestamp: entry.Timestamp, - Message: entry.Message, - Level: logs.StringToLevel(string(entry.Level)), - Fields: entry.Fields, + raw := *res.JSON200 + l := make([]logs.LogEntry, len(raw.LogEntries)) + for i, entry := range raw.LogEntries { + l[i] = logs.LogEntry{ + Timestamp: entry.Timestamp, + Message: entry.Message, + Level: logs.StringToLevel(string(entry.Level)), + Fields: entry.Fields, + } } - } - return l, nil + return l, nil + } } From 045ef035795fd8a506c089b1d698fb59e5ae84bf Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 12:06:30 +0100 Subject: [PATCH 41/58] When instance was not found, just log err do not return --- packages/api/internal/clusters/resources_local.go | 15 ++++++++++----- .../api/internal/clusters/resources_remote.go | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go index 363813364f..5ca5c35800 100644 --- a/packages/api/internal/clusters/resources_local.go +++ b/packages/api/internal/clusters/resources_local.go @@ -161,12 +161,17 @@ func (l *LocalClusterResourceProvider) GetBuildLogs( if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { instance, found := l.instances.Get(*nodeID) - if !found { - return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) + if found { + sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) + } else { + logger.L().Warn( + ctx, "Node instance not found for build logs, falling back to other sources", + logger.WithNodeID(*nodeID), + logger.WithTemplateID(templateID), + logger.WithBuildID(buildID), + ) } - - sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - sources = append(sources, sourceCallback) } // Fetch logs from Loki backend (explicit persistent source or default) diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index 04789ed8db..ec535f7a45 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -156,12 +156,17 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs( if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { instance, found := r.instances.Get(*nodeID) - if !found { - return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID) + if found { + sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) + } else { + logger.L().Warn( + ctx, "Node instance not found for build logs, falling back to other sources", + logger.WithNodeID(*nodeID), + logger.WithTemplateID(templateID), + logger.WithBuildID(buildID), + ) } - - sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - sources = append(sources, sourceCallback) } if logCheckSourceType(source, api.LogsSourcePersistent) { From a2c97795725627e6778a143eb552045a9e59c70b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 13:30:01 +0100 Subject: [PATCH 42/58] After-merge changes --- packages/api/internal/edge/cluster.go | 189 ------------------ .../template-manager/template_manager.go | 2 +- 2 files changed, 1 insertion(+), 190 deletions(-) delete mode 100644 packages/api/internal/edge/cluster.go diff --git a/packages/api/internal/edge/cluster.go b/packages/api/internal/edge/cluster.go deleted file mode 100644 index d40a62f2ab..0000000000 --- a/packages/api/internal/edge/cluster.go +++ /dev/null @@ -1,189 +0,0 @@ -package edge - -import ( - "context" - "errors" - "fmt" - "math/rand/v2" - "net/http" - - "github.com/google/uuid" - "go.opentelemetry.io/otel" - "google.golang.org/grpc/metadata" - - grpclient "github.com/e2b-dev/infra/packages/api/internal/grpc" - "github.com/e2b-dev/infra/packages/shared/pkg/consts" - infogrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator-info" - api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" - "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" - "github.com/e2b-dev/infra/packages/shared/pkg/smap" - "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" -) - -var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/api/internal/edge") - -type Cluster struct { - ID uuid.UUID - - httpClient *api.ClientWithResponses - grpcClient *grpclient.GRPCClient - - instances *smap.Map[*ClusterInstance] - synchronization *synchronization.Synchronize[api.ClusterOrchestratorNode, *ClusterInstance] - SandboxDomain *string -} - -type ClusterGRPC struct { - Client *grpclient.GRPCClient - Metadata metadata.MD -} - -type ClusterHTTP struct { - Client *api.ClientWithResponses -} - -var ( - ErrTemplateBuilderNotFound = errors.New("template builder not found") - ErrAvailableTemplateBuilderNotFound = errors.New("available template builder not found") -) - -func NewCluster(ctx context.Context, tel *telemetry.Client, endpoint string, endpointTLS bool, secret string, clusterID uuid.UUID, sandboxDomain *string) (*Cluster, error) { - clientAuthMiddleware := func(c *api.Client) error { - c.RequestEditors = append( - c.RequestEditors, - func(_ context.Context, req *http.Request) error { - req.Header.Set(consts.EdgeApiAuthHeader, secret) - - return nil - }, - ) - - return nil - } - - // generate the full endpoint URL - scheme := "http" - if endpointTLS { - scheme = "https" - } - endpointBaseUrl := fmt.Sprintf("%s://%s", scheme, endpoint) - - httpClient, err := api.NewClientWithResponses(endpointBaseUrl, clientAuthMiddleware) - if err != nil { - return nil, fmt.Errorf("failed to create http client: %w", err) - } - - grpcAuthorization := clientAuthorization{secret: secret, tls: endpointTLS} - grpcClient, err := createClusterClient(tel, grpcAuthorization, endpoint, endpointTLS) - if err != nil { - return nil, fmt.Errorf("failed to create grpc client: %w", err) - } - - c := &Cluster{ - ID: clusterID, - SandboxDomain: sandboxDomain, - - instances: smap.New[*ClusterInstance](), - httpClient: httpClient, - grpcClient: grpcClient, - } - - store := clusterSynchronizationStore{cluster: c} - c.synchronization = synchronization.NewSynchronize("cluster-instances", "Cluster instances", store) - - // periodically sync cluster instances - go c.startSync(ctx) - - return c, nil -} - -func (c *Cluster) Close() error { - c.synchronization.Close() - err := c.grpcClient.Close() - - return err -} - -func (c *Cluster) GetTemplateBuilderByNodeID(nodeID string) (*ClusterInstance, error) { - instance, found := c.instances.Get(nodeID) - if !found { - return nil, ErrTemplateBuilderNotFound - } - - if instance.GetStatus() == infogrpc.ServiceInfoStatus_Unhealthy || !instance.IsBuilder() { - return nil, ErrTemplateBuilderNotFound - } - - return instance, nil -} - -func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*ClusterInstance, bool) { - for _, instance := range c.instances.Items() { - if instance.ServiceInstanceID == serviceInstanceID { - return instance, true - } - } - - return nil, false -} - -func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, info machineinfo.MachineInfo) (*ClusterInstance, error) { - _, span := tracer.Start(ctx, "template-builder-get-available-instance") - span.SetAttributes(telemetry.WithClusterID(c.ID)) - defer span.End() - - // convert map to slice - mapItems := c.instances.Items() - instances := make([]*ClusterInstance, 0, len(mapItems)) - for _, instance := range mapItems { - instances = append(instances, instance) - } - - // Make sure we will always iterate in different order and when there is bigger amount of builders, we will not always pick the same one - rand.Shuffle(len(instances), func(i, j int) { - instances[i], instances[j] = instances[j], instances[i] - }) - - for _, instance := range instances { - if instance.GetStatus() != infogrpc.ServiceInfoStatus_Healthy { - continue - } - - if !instance.IsBuilder() { - continue - } - - if info.CPUModel != "" && !info.IsCompatibleWith(instance.machineInfo) { - continue - } - - return instance, nil - } - - return nil, ErrAvailableTemplateBuilderNotFound -} - -func (c *Cluster) GetGRPC(serviceInstanceID string) *ClusterGRPC { - return &ClusterGRPC{c.grpcClient, metadata.New(map[string]string{consts.EdgeRpcServiceInstanceIDHeader: serviceInstanceID})} -} - -func (c *Cluster) GetHTTP() *ClusterHTTP { - return &ClusterHTTP{c.httpClient} -} - -func (c *Cluster) GetOrchestrators() []*ClusterInstance { - mapItems := c.instances.Items() - instances := make([]*ClusterInstance, 0, len(mapItems)) - for _, instance := range mapItems { - if instance.IsOrchestrator() { - instances = append(instances, instance) - } - } - - return instances -} - -func (c *Cluster) GetHttpClient() *api.ClientWithResponses { - return c.httpClient -} diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index f61a0c1370..8f6bd9a0a6 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -121,7 +121,7 @@ func (tm *TemplateManager) GetAvailableBuildClient(ctx context.Context, clusterI nodeInfo := machineinfo.FromLDValue(ctx, nodeInfoJSON) builder, err := cluster.GetAvailableTemplateBuilder(ctx, nodeInfo) if err != nil { - if errors.Is(err, edge.ErrAvailableTemplateBuilderNotFound) { + if errors.Is(err, clusters.ErrAvailableTemplateBuilderNotFound) { // Fallback to any template builder logger.L().Warn(ctx, "No available template builder found with the specified machine info, falling back to any available template builder", zap.String("clusterID", clusterID.String())) From 2ef3a74d78842413dd0731739aaf6677261afe88 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 13:30:18 +0100 Subject: [PATCH 43/58] Implemented template builder cpu check from 05ac56bf4a12cf01275c0339baefd596b6e91084 --- packages/api/internal/clusters/cluster.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 30d3dc9b30..24e5ed79a5 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -21,6 +21,7 @@ import ( api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs/loki" + "github.com/e2b-dev/infra/packages/shared/pkg/machineinfo" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/synchronization" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -187,7 +188,7 @@ func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*Instance, b return nil, false } -func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, error) { +func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, info machineinfo.MachineInfo) (*Instance, error) { _, span := tracer.Start(ctx, "template-builder-get-available-instance") span.SetAttributes(telemetry.WithClusterID(c.ID)) defer span.End() @@ -201,10 +202,16 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context) (*Instance, e rand.Shuffle(len(instances), func(i, j int) { instances[i], instances[j] = instances[j], instances[i] }) for _, instance := range instances { + // Check availability and builder role if info := instance.GetInfo(); info.Status != infogrpc.ServiceInfoStatus_Healthy || !info.IsBuilder { continue } + // Check machine compatibility + if info.CPUModel != "" && !info.IsCompatibleWith(instance.machine) { + continue + } + return instance, nil } From 8397a810410225e074ea6aebc2eed79b91228df0 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 13:58:15 +0100 Subject: [PATCH 44/58] Access machine info on copied struct --- packages/api/internal/clusters/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 24e5ed79a5..11d9fbea00 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -208,7 +208,7 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, info machinei } // Check machine compatibility - if info.CPUModel != "" && !info.IsCompatibleWith(instance.machine) { + if machineInfo := instance.GetMachineInfo(); machineInfo.CPUModel != "" && !machineInfo.IsCompatibleWith(instance.machine) { continue } From 1e629258bd9020a9b9ea9c83f3239171427956e1 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 14:03:56 +0100 Subject: [PATCH 45/58] Make parameter naming more obvious, fix use --- packages/api/internal/clusters/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 11d9fbea00..341c85080e 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -188,7 +188,7 @@ func (c *Cluster) GetByServiceInstanceID(serviceInstanceID string) (*Instance, b return nil, false } -func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, info machineinfo.MachineInfo) (*Instance, error) { +func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, expectedInfo machineinfo.MachineInfo) (*Instance, error) { _, span := tracer.Start(ctx, "template-builder-get-available-instance") span.SetAttributes(telemetry.WithClusterID(c.ID)) defer span.End() @@ -208,7 +208,7 @@ func (c *Cluster) GetAvailableTemplateBuilder(ctx context.Context, info machinei } // Check machine compatibility - if machineInfo := instance.GetMachineInfo(); machineInfo.CPUModel != "" && !machineInfo.IsCompatibleWith(instance.machine) { + if machineInfo := instance.GetMachineInfo(); expectedInfo.CPUModel != "" && !expectedInfo.IsCompatibleWith(machineInfo) { continue } From 62fc82e4c2f3c91c585506269c0fd462fb596fc5 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 14:17:07 +0100 Subject: [PATCH 46/58] Removed duplicated instance sync When new instance was created, function callback did init sync instantly when created and then we sync again before inserting into instances pool. Second sync was removed and we are only doing one that will make sure instance values are filled as expected. --- packages/api/internal/clusters/instances_sync.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/api/internal/clusters/instances_sync.go b/packages/api/internal/clusters/instances_sync.go index 1b9c5ff686..786b7c4d55 100644 --- a/packages/api/internal/clusters/instances_sync.go +++ b/packages/api/internal/clusters/instances_sync.go @@ -63,6 +63,7 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item discovery.Item) logger.WithServiceInstanceID(item.InstanceID), ) + // Instant is synced immediately after creation to ensure it's working before adding to the pool. instance, err := d.instanceCreation(ctx, item) if err != nil { logger.L().Error(ctx, "Failed to create cluster instance during pool insert", @@ -75,14 +76,6 @@ func (d instancesSyncStore) PoolInsert(ctx context.Context, item discovery.Item) return } - ok := d.tryToSyncInstance(ctx, instance) - if !ok { - // Try to gracefully close the instance - d.tryToCloseInstance(ctx, instance) - - return - } - d.instances.Insert(item.NodeID, instance) } From d2fc2af777a8b42846806313d75db3de4aecedb2 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 5 Jan 2026 14:22:14 +0100 Subject: [PATCH 47/58] Fixed double fetch --- packages/api/internal/clusters/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go index fc1c83437a..5700597c32 100644 --- a/packages/api/internal/clusters/resources.go +++ b/packages/api/internal/clusters/resources.go @@ -99,7 +99,7 @@ func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID raw := res.GetLogEntries() entries := make([]logs.LogEntry, len(raw)) - for i, entry := range res.GetLogEntries() { + for i, entry := range raw { entries[i] = logs.LogEntry{ Timestamp: entry.GetTimestamp().AsTime(), Message: entry.GetMessage(), From 857b75bc4a8c330ab0bbfd391bbcfb633578f96f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 6 Jan 2026 12:10:06 +0100 Subject: [PATCH 48/58] Fail instance health on too many failed synces This fixes issue where service discovery still returns instance so its still present in pool but for some reason its not possible to connect to it. When instance fails 3 times in row it will be marked as unhealthy until next sucess sync is executed, then counter is reset. --- packages/api/internal/clusters/instance.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 789b4a24e6..c375a71240 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -18,6 +18,12 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) +const ( + // maxSyncFailuresBeforeUnhealthy defines the number of consecutive sync failures + // before an instance is marked as unhealthy. + maxSyncFailuresBeforeUnhealthy = 3 +) + type Instance struct { // Identifier that uniquely identifies the instance so it will not be registered multiple times. // Depending on service discovery used, it can be combination of different parameters, what service discovery gives us. @@ -37,6 +43,8 @@ type Instance struct { isBuilder bool isOrchestrator bool + syncFailCount int + mutex sync.RWMutex } @@ -104,12 +112,24 @@ func (i *Instance) Sync(ctx context.Context) error { info, err := i.client.Info.ServiceInfo(ctx, &emptypb.Empty{}) err = utils.UnwrapGRPCError(err) if err != nil { + i.mutex.Lock() + defer i.mutex.Unlock() + + // Increase fail count and set unhealthy status if needed + i.syncFailCount++ + if i.syncFailCount >= maxSyncFailuresBeforeUnhealthy { + i.status = infogrpc.ServiceInfoStatus_Unhealthy + } + return err } i.mutex.Lock() defer i.mutex.Unlock() + // Reset fail count on successful sync + i.syncFailCount = 0 + i.status = info.GetServiceStatus() i.roles = info.GetServiceRoles() i.machine = machineinfo.FromGRPCInfo(info.GetMachineInfo()) From 7e39e0bc7ad21c6acfb4e4113e02eda07a30f450 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 6 Jan 2026 12:12:50 +0100 Subject: [PATCH 49/58] Added log line during too many failed sync --- packages/api/internal/clusters/instance.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index c375a71240..2fdcc64fb6 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -118,6 +118,14 @@ func (i *Instance) Sync(ctx context.Context) error { // Increase fail count and set unhealthy status if needed i.syncFailCount++ if i.syncFailCount >= maxSyncFailuresBeforeUnhealthy { + logger.L().Warn(ctx, "Instance sync failed multiple times, marking instance as unhealthy", + logger.WithNodeID(i.NodeID), + logger.WithClusterID(i.ClusterID), + logger.WithServiceInstanceID(i.serviceInstanceID), + zap.Int("counter", i.syncFailCount), + zap.Error(err), + ) + i.status = infogrpc.ServiceInfoStatus_Unhealthy } From e6890ef1fff7af23d5d593dc90bbe577f8324d1a Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 6 Jan 2026 12:22:16 +0100 Subject: [PATCH 50/58] Make instance sync call lower timeouted Before there was shared sync timeout for whole of 5 seconds for whole call managed by sync managed. Now gRPC call is limited to 1 seconds. --- packages/api/internal/clusters/instance.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 2fdcc64fb6..86373f321e 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -5,6 +5,7 @@ import ( "fmt" "slices" "sync" + "time" "github.com/google/uuid" "go.uber.org/zap" @@ -22,6 +23,8 @@ const ( // maxSyncFailuresBeforeUnhealthy defines the number of consecutive sync failures // before an instance is marked as unhealthy. maxSyncFailuresBeforeUnhealthy = 3 + + maxInstanceSyncCallTimeout = 1 * time.Second ) type Instance struct { @@ -109,6 +112,9 @@ func newInstance( // Sync function can be called on freshly initialized instance to populate its data // In initial case its possible that service instance id needed for proper remote cluster routing is not yet set. func (i *Instance) Sync(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, maxInstanceSyncCallTimeout) + defer cancel() + info, err := i.client.Info.ServiceInfo(ctx, &emptypb.Empty{}) err = utils.UnwrapGRPCError(err) if err != nil { From fe889cd2a6d650c708eb5205e28330550eda1802 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 7 Jan 2026 10:52:11 +0100 Subject: [PATCH 51/58] Fix after merge conflict --- packages/api/internal/handlers/sandbox_metrics.go | 2 +- packages/api/internal/handlers/sandboxes_list_metrics.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/handlers/sandbox_metrics.go b/packages/api/internal/handlers/sandbox_metrics.go index 72e4fc8169..5b94cb6c55 100644 --- a/packages/api/internal/handlers/sandbox_metrics.go +++ b/packages/api/internal/handlers/sandbox_metrics.go @@ -34,7 +34,7 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(c *gin.Context, sandboxID string Build(), ) - metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlagName) + metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlag) if !metricsReadFlag { logger.L().Debug(ctx, "sandbox metrics read feature flag is disabled") // If we are not reading from ClickHouse, we can return an empty map diff --git a/packages/api/internal/handlers/sandboxes_list_metrics.go b/packages/api/internal/handlers/sandboxes_list_metrics.go index 9cab674ac4..cf1024129e 100644 --- a/packages/api/internal/handlers/sandboxes_list_metrics.go +++ b/packages/api/internal/handlers/sandboxes_list_metrics.go @@ -41,7 +41,7 @@ func (a *APIStore) getSandboxesMetrics( ) // Get metrics for all sandboxes - metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlagName) + metricsReadFlag := a.featureFlags.BoolFlag(ctx, featureflags.MetricsReadFlag) if !metricsReadFlag { logger.L().Debug(ctx, "sandbox metrics read feature flag is disabled") // If we are not reading from ClickHouse, we can return an empty map From a2fbac6d1692ec35849c8f347b027ee347508fe4 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 7 Jan 2026 15:55:48 +0100 Subject: [PATCH 52/58] After conflict deps tidy --- packages/api/go.mod | 2 -- packages/api/go.sum | 4 ++-- packages/client-proxy/go.mod | 2 -- packages/shared/go.mod | 5 +++-- packages/shared/go.sum | 8 ++++---- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index f8d69be34d..4f0487094e 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -218,7 +218,6 @@ require ( github.com/miekg/dns v1.1.63 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -267,7 +266,6 @@ require ( github.com/sercand/kuberesolver/v5 v5.1.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil/v4 v4.25.6 // indirect - github.com/shoenig/test v1.8.2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index a0be49f075..ae4f9aaa1c 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -35,8 +35,8 @@ dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index bed086d5cd..186e95e227 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -167,7 +167,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -204,7 +203,6 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sercand/kuberesolver/v5 v5.1.1 // indirect - github.com/shoenig/test v1.8.2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 631e930ec8..6ad82e5745 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -45,7 +45,7 @@ require ( github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 github.com/hashicorp/go-retryablehttp v0.7.7 - github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c + github.com/hashicorp/nomad/api v0.0.0-20240813123601-b34a6fe10b82 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/launchdarkly/go-sdk-common/v3 v3.3.0 github.com/launchdarkly/go-server-sdk/v7 v7.13.0 @@ -207,7 +207,6 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/memberlist v0.5.0 // indirect - github.com/hashicorp/nomad/api v0.0.0-20240813123601-b34a6fe10b82 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -243,6 +242,7 @@ require ( github.com/miekg/dns v1.1.63 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -284,6 +284,7 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sercand/kuberesolver/v5 v5.1.1 // indirect + github.com/shoenig/test v1.8.2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect diff --git a/packages/shared/go.sum b/packages/shared/go.sum index f0d48dad02..d8e1ecdf66 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -680,8 +680,8 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 h1:drhDO54gdT/a15GBcMRmunZiNcLgPiFIJa23KzmcvcU= +github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770/go.mod h1:SO/iHr6q2EzbqRApt+8/E9wqebTwQn5y+UlB04bxzo0= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= @@ -829,8 +829,8 @@ github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtr github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= -github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= -github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/shoenig/test v1.8.2 h1:WDlty8UBqJRdmgdJX8lMwvCq97tiN7Um/GZD2vBDuug= +github.com/shoenig/test v1.8.2/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= From 2daf8502e8759a02e94f64606ff448878f0b23c7 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 11:12:23 +0100 Subject: [PATCH 53/58] Renamed const for orchestrator API port --- packages/api/internal/clusters/discovery/local.go | 4 ++-- packages/shared/pkg/consts/sandboxes.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/clusters/discovery/local.go b/packages/api/internal/clusters/discovery/local.go index 55c42d87c3..f6740c0aa6 100644 --- a/packages/api/internal/clusters/discovery/local.go +++ b/packages/api/internal/clusters/discovery/local.go @@ -47,7 +47,7 @@ func (sd *LocalServiceDiscovery) Query(ctx context.Context) ([]Item, error) { NodeID: "local", InstanceID: "unknown", LocalIPAddress: testsInstanceHost, - LocalInstanceApiPort: consts.OrchestratorApiPort, + LocalInstanceApiPort: consts.OrchestratorAPIPort, }, }, nil } @@ -73,7 +73,7 @@ func (sd *LocalServiceDiscovery) Query(ctx context.Context) ([]Item, error) { // For now, we assume ports that are used for gRPC api and proxy are static, // in future we should be able to take port numbers from Nomad API and map them accordingly here. LocalIPAddress: v.AllocationIP, - LocalInstanceApiPort: consts.OrchestratorApiPort, + LocalInstanceApiPort: consts.OrchestratorAPIPort, } result[i] = item diff --git a/packages/shared/pkg/consts/sandboxes.go b/packages/shared/pkg/consts/sandboxes.go index a484c18082..fa6a006fb2 100644 --- a/packages/shared/pkg/consts/sandboxes.go +++ b/packages/shared/pkg/consts/sandboxes.go @@ -14,4 +14,4 @@ const NodeIDLength = 8 // We don't want to use some obviously dummy value such as empty zeros, because for users it will look like something is wrong with the sandbox id const ClientID = "6532622b" -var OrchestratorApiPort = uint16(utils.Must(strconv.ParseUint(env.GetEnv("ORCHESTRATOR_PORT", "5008"), 10, 16))) +var OrchestratorAPIPort = uint16(utils.Must(strconv.ParseUint(env.GetEnv("ORCHESTRATOR_PORT", "5008"), 10, 16))) From ce46916b5f6940ef9f69a4c90546fd96a2c276b9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 11:12:41 +0100 Subject: [PATCH 54/58] Build cluster api endpoint --- packages/api/internal/clusters/cluster.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/clusters/cluster.go b/packages/api/internal/clusters/cluster.go index 341c85080e..7df4d2767f 100644 --- a/packages/api/internal/clusters/cluster.go +++ b/packages/api/internal/clusters/cluster.go @@ -90,11 +90,13 @@ func newRemoteCluster( clusterID uuid.UUID, sandboxDomain *string, ) (*Cluster, error) { - endpointBaseUrl := fmt.Sprintf("http://%s", endpoint) + scheme := "http" if endpointTLS { - endpointBaseUrl = fmt.Sprintf("https://%s", endpoint) + scheme = "https" } + endpointBaseUrl := fmt.Sprintf("%s://%s", scheme, endpoint) + httpClient, err := api.NewClientWithResponses( endpointBaseUrl, func(c *api.Client) error { From 3226d6c7c21e48530c7a9ec0dc591ad4c5ece06d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 11:14:56 +0100 Subject: [PATCH 55/58] Use client instead of connection --- packages/api/internal/clusters/instance.go | 8 +++--- .../api/internal/clusters/instance_client.go | 2 +- packages/api/internal/clusters/resources.go | 2 +- packages/api/internal/orchestrator/client.go | 2 +- .../orchestrator/nodemanager/builds.go | 2 +- .../orchestrator/nodemanager/metadata.go | 8 +++--- .../internal/orchestrator/nodemanager/mock.go | 16 ++++++------ .../internal/orchestrator/nodemanager/node.go | 26 +++++++++---------- .../orchestrator/nodemanager/status.go | 2 +- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/api/internal/clusters/instance.go b/packages/api/internal/clusters/instance.go index 86373f321e..5a58f14dd3 100644 --- a/packages/api/internal/clusters/instance.go +++ b/packages/api/internal/clusters/instance.go @@ -70,7 +70,7 @@ func newInstance( connAddr string, connTls bool, ) (*Instance, error) { - conn, err := createConnection(tel, clusterAuth, connAddr, connTls) + client, err := createClient(tel, clusterAuth, connAddr, connTls) if err != nil { return nil, fmt.Errorf("failed to create cluster instance client client: %w", err) } @@ -86,13 +86,13 @@ func newInstance( NodeID: sd.NodeID, ClusterID: clusterID, - client: conn, + client: client, mutex: sync.RWMutex{}, } err = i.Sync(ctx) if err != nil { - closeErr := conn.Close() + closeErr := client.Close() if closeErr != nil { logger.L().Error( ctx, "Failed to close gRPC Connection after instance sync failure", @@ -181,7 +181,7 @@ func (i *Instance) GetInfo() InstanceInfo { } } -func (i *Instance) GetConnection() *GRPCClient { +func (i *Instance) GetClient() *GRPCClient { return i.client } diff --git a/packages/api/internal/clusters/instance_client.go b/packages/api/internal/clusters/instance_client.go index 9a26ef14ae..027313261f 100644 --- a/packages/api/internal/clusters/instance_client.go +++ b/packages/api/internal/clusters/instance_client.go @@ -33,7 +33,7 @@ func (a instanceAuthorization) RequireTransportSecurity() bool { return a.tls } -func createConnection(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*GRPCClient, error) { +func createClient(tel *telemetry.Client, auth *instanceAuthorization, endpoint string, endpointTLS bool) (*GRPCClient, error) { grpcOptions := []grpc.DialOption{ grpc.WithStatsHandler( otelgrpc.NewClientHandler( diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go index 5700597c32..41da6867e7 100644 --- a/packages/api/internal/clusters/resources.go +++ b/packages/api/internal/clusters/resources.go @@ -78,7 +78,7 @@ func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID lvlReq = templatemanagergrpc.LogLevel(*level).Enum() } - res, err := instance.GetConnection().Template.TemplateBuildStatus( + res, err := instance.GetClient().Template.TemplateBuildStatus( ctx, &templatemanagergrpc.TemplateStatusRequest{ TemplateID: templateID, BuildID: buildID, diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index c18b5e4ad9..faa7791280 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -33,7 +33,7 @@ func (o *Orchestrator) connectToNode(ctx context.Context, discovered nodemanager } func (o *Orchestrator) connectToClusterNode(ctx context.Context, cluster *clusters.Cluster, i *clusters.Instance) { - orchestratorNode, err := nodemanager.NewClusterNode(ctx, i.GetConnection(), cluster.ID, cluster.SandboxDomain, i) + orchestratorNode, err := nodemanager.NewClusterNode(ctx, i.GetClient(), cluster.ID, cluster.SandboxDomain, i) if err != nil { logger.L().Error(ctx, "Failed to create node", zap.Error(err)) diff --git a/packages/api/internal/orchestrator/nodemanager/builds.go b/packages/api/internal/orchestrator/nodemanager/builds.go index 88470d8b26..6c5ff447a4 100644 --- a/packages/api/internal/orchestrator/nodemanager/builds.go +++ b/packages/api/internal/orchestrator/nodemanager/builds.go @@ -34,7 +34,7 @@ func (n *Node) listCachedBuilds(ctx context.Context) ([]*orchestrator.CachedBuil childCtx, childSpan := tracer.Start(ctx, "list-cached-builds") defer childSpan.End() - res, err := n.connection.Sandbox.ListCachedBuilds(childCtx, &empty.Empty{}) + res, err := n.client.Sandbox.ListCachedBuilds(childCtx, &empty.Empty{}) err = utils.UnwrapGRPCError(err) if err != nil { return nil, fmt.Errorf("failed to list sandboxes: %w", err) diff --git a/packages/api/internal/orchestrator/nodemanager/metadata.go b/packages/api/internal/orchestrator/nodemanager/metadata.go index a4a0bfdd2c..04afa44d12 100644 --- a/packages/api/internal/orchestrator/nodemanager/metadata.go +++ b/packages/api/internal/orchestrator/nodemanager/metadata.go @@ -49,8 +49,8 @@ func (n *Node) GetSandboxCreateCtx(ctx context.Context, req *orchestrator.Sandbo ) } - // Merge medata from connection (auth, routing with service instance id) and event metadata. - return n.connection, appendMetadataCtx(ctx, md) + // Merge medata from client (auth, routing with service instance id) and event metadata. + return n.client, appendMetadataCtx(ctx, md) } func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, executionID string) (*clusters.GRPCClient, context.Context) { @@ -65,8 +65,8 @@ func (n *Node) GetSandboxDeleteCtx(ctx context.Context, sandboxID string, execut ) } - // Merge medata from connection (auth, routing with service instance id) and event metadata. - return n.connection, appendMetadataCtx(ctx, md) + // Merge medata from client (auth, routing with service instance id) and event metadata. + return n.client, appendMetadataCtx(ctx, md) } func appendMetadataCtx(ctx context.Context, md metadata.MD) context.Context { diff --git a/packages/api/internal/orchestrator/nodemanager/mock.go b/packages/api/internal/orchestrator/nodemanager/mock.go index 52df8d7f2f..22272c23df 100644 --- a/packages/api/internal/orchestrator/nodemanager/mock.go +++ b/packages/api/internal/orchestrator/nodemanager/mock.go @@ -57,7 +57,7 @@ func (n *mockSandboxClientWithSleep) Create(_ context.Context, _ *orchestrator.S return &orchestrator.SandboxCreateResponse{}, nil } -// newMockGRPCClient creates a new mock gRPC connection for testing +// newMockGRPCClient creates a new mock gRPC client for testing func newMockGRPCClient() *clusters.GRPCClient { // Create a dummy connection that will never be used conn, _ := grpc.NewClient("localhost:0", grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -74,7 +74,7 @@ type TestOptions func(node *TestNode) func WithSandboxSleepingClient(baseSandboxCreateTime time.Duration) TestOptions { return func(node *TestNode) { - node.connection.Sandbox = &mockSandboxClientWithSleep{ + node.client.Sandbox = &mockSandboxClientWithSleep{ baseSandboxCreateTime: baseSandboxCreateTime, } } @@ -102,7 +102,7 @@ func (n *mockSandboxClientWithError) Create(_ context.Context, _ *orchestrator.S func WithSandboxCreateError(err error) TestOptions { return func(node *TestNode) { - node.connection.Sandbox = &mockSandboxClientWithError{ + node.client.Sandbox = &mockSandboxClientWithError{ err: err, } } @@ -126,13 +126,13 @@ func (n *MockSandboxClientCustom) Create(_ context.Context, _ *orchestrator.Sand return &orchestrator.SandboxCreateResponse{}, nil } -// SetSandboxClient allows setting a custom sandbox connection on a test node +// SetSandboxClient allows setting a custom sandbox client on a test node func (n *TestNode) SetSandboxClient(client orchestrator.SandboxServiceClient) { - n.connection.Sandbox = client + n.client.Sandbox = client } // NewTestNode creates a properly initialized Node for testing purposes -// It uses a mock gRPC connection and has simplified Status() method behavior +// It uses a mock gRPC client and has simplified Status() method behavior func NewTestNode(id string, status api.NodeStatus, cpuAllocated int64, cpuCount uint32, options ...TestOptions) *TestNode { node := &Node{ ID: id, @@ -145,8 +145,8 @@ func NewTestNode(id string, status api.NodeStatus, cpuAllocated int64, cpuCount createFails: atomic.Uint64{}, }, - connection: newMockGRPCClient(), - status: status, + client: newMockGRPCClient(), + status: status, metrics: Metrics{ CpuAllocated: uint32(cpuAllocated), CpuCount: cpuCount, diff --git a/packages/api/internal/orchestrator/nodemanager/node.go b/packages/api/internal/orchestrator/nodemanager/node.go index 2f163ec672..568cbe5427 100644 --- a/packages/api/internal/orchestrator/nodemanager/node.go +++ b/packages/api/internal/orchestrator/nodemanager/node.go @@ -39,8 +39,8 @@ type Node struct { IPAddress string SandboxDomain *string - connection *clusters.GRPCClient - status api.NodeStatus + client *clusters.GRPCClient + status api.NodeStatus metrics Metrics metricsMu sync.RWMutex @@ -95,9 +95,9 @@ func New( IPAddress: discoveredNode.IPAddress, SandboxDomain: nil, - connection: client, - status: nodeStatus, - meta: nodeMetadata, + client: client, + status: nodeStatus, + meta: nodeMetadata, buildCache: buildCache, PlacementMetrics: PlacementMetrics{ @@ -142,9 +142,9 @@ func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID createFails: atomic.Uint64{}, }, - connection: client, - status: status, - meta: nodeMetadata, + client: client, + status: status, + meta: nodeMetadata, buildCache: buildCache, } @@ -166,21 +166,21 @@ func NewClusterNode(ctx context.Context, client *clusters.GRPCClient, clusterID func (n *Node) Close(ctx context.Context) error { if n.IsNomadManaged() { logger.L().Info(ctx, "Closing local node", logger.WithNodeID(n.ID)) - if err := n.connection.Close(); err != nil { - logger.L().Error(ctx, "Error closing connection to node", zap.Error(err), logger.WithNodeID(n.ID)) + if err := n.client.Close(); err != nil { + logger.L().Error(ctx, "Error closing client to node", zap.Error(err), logger.WithNodeID(n.ID)) } } else { logger.L().Info(ctx, "Closing cluster node", logger.WithNodeID(n.ID), logger.WithClusterID(n.ClusterID)) - // We are not closing grpc connection, because it is managed by cluster instance + // We are not closing grpc client, because it is managed by cluster instance } n.buildCache.Stop() return nil } -// Ensures that GRPC connection request context always has the latest service instance ID +// Ensures that GRPC client request context always has the latest service instance ID func (n *Node) GetClient(ctx context.Context) (*clusters.GRPCClient, context.Context) { - return n.connection, ctx + return n.client, ctx } func (n *Node) IsNomadManaged() bool { diff --git a/packages/api/internal/orchestrator/nodemanager/status.go b/packages/api/internal/orchestrator/nodemanager/status.go index 0b5e9db6b6..8c45c0f66a 100644 --- a/packages/api/internal/orchestrator/nodemanager/status.go +++ b/packages/api/internal/orchestrator/nodemanager/status.go @@ -26,7 +26,7 @@ func (n *Node) Status() api.NodeStatus { return n.status } - switch n.connection.Connection.GetState() { + switch n.client.Connection.GetState() { case connectivity.Shutdown: return api.NodeStatusUnhealthy case connectivity.TransientFailure: From d510feacc0ef226505560aac67d9238cd76faeca Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 11:15:03 +0100 Subject: [PATCH 56/58] Nits --- packages/api/internal/orchestrator/client.go | 2 +- .../api/internal/template-manager/template_manager.go | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/api/internal/orchestrator/client.go b/packages/api/internal/orchestrator/client.go index faa7791280..62b5c6f669 100644 --- a/packages/api/internal/orchestrator/client.go +++ b/packages/api/internal/orchestrator/client.go @@ -79,7 +79,7 @@ func (o *Orchestrator) listNomadNodes(ctx context.Context) ([]nodemanager.NomadS for _, n := range nomadNodes { result = append(result, nodemanager.NomadServiceDiscovery{ NomadNodeShortID: n.ID[:consts.NodeIDLength], - OrchestratorAddress: fmt.Sprintf("%s:%d", n.Address, consts.OrchestratorApiPort), + OrchestratorAddress: fmt.Sprintf("%s:%d", n.Address, consts.OrchestratorAPIPort), IPAddress: n.Address, }) } diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 8f6bd9a0a6..92a56931c3 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -33,8 +33,7 @@ type processingBuilds struct { } type TemplateManager struct { - clusters *clusters.Pool - + clusters *clusters.Pool lock sync.Mutex processing map[uuid.UUID]processingBuilds buildCache *templatecache.TemplatesBuildCache @@ -58,7 +57,7 @@ const ( func New( sqlcDB *sqlcdb.Client, - edgePool *clusters.Pool, + clusters *clusters.Pool, buildCache *templatecache.TemplatesBuildCache, templateCache *templatecache.TemplateCache, featureFlags *featureflags.Client, @@ -67,7 +66,7 @@ func New( sqlcDB: sqlcDB, buildCache: buildCache, templateCache: templateCache, - clusters: edgePool, + clusters: clusters, featureFlags: featureFlags, lock: sync.Mutex{}, @@ -159,7 +158,7 @@ func (tm *TemplateManager) GetClusterBuildClient(clusterID uuid.UUID, nodeID str return nil, fmt.Errorf("failed to get builder by id '%s': %w", nodeID, err) } - return instance.GetConnection(), nil + return instance.GetClient(), nil } func (tm *TemplateManager) DeleteBuild(ctx context.Context, buildID uuid.UUID, templateID string, clusterID uuid.UUID, nodeID string) error { From 6422db3c6bff436992a248581459c11da6a3f05f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 18:43:11 +0100 Subject: [PATCH 57/58] For statis service discovery use list with predefined size --- .../service-discovery/static-service-discovery.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/client-proxy/internal/service-discovery/static-service-discovery.go b/packages/client-proxy/internal/service-discovery/static-service-discovery.go index 315a7ec311..678e573b29 100644 --- a/packages/client-proxy/internal/service-discovery/static-service-discovery.go +++ b/packages/client-proxy/internal/service-discovery/static-service-discovery.go @@ -7,12 +7,9 @@ type StaticServiceDiscovery struct { } func NewStaticServiceDiscovery(results []string, port uint16) *StaticServiceDiscovery { - items := make([]DiscoveredInstance, 0) - - for _, result := range results { - items = append( - items, DiscoveredInstance{InstanceIPAddress: result, InstancePort: port}, - ) + items := make([]DiscoveredInstance, len(results)) + for i, result := range results { + items[i] = DiscoveredInstance{InstanceIPAddress: result, InstancePort: port} } return &StaticServiceDiscovery{items: items} From 914d08b5e2da036f7d6c7809b5756ad0aa70c14e Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 12 Jan 2026 18:52:09 +0100 Subject: [PATCH 58/58] Deduplicate function for fetching build logs Previously we used almost identical implementation for fetching build logs in both local and remote resources providers. Now we have one unifier implementation that only differes with persistent provider that si different for local vs remote - its providede as function callback. --- packages/api/internal/clusters/resources.go | 69 +++++++++++++++++++ .../api/internal/clusters/resources_local.go | 48 ++----------- .../api/internal/clusters/resources_remote.go | 45 +----------- 3 files changed, 78 insertions(+), 84 deletions(-) diff --git a/packages/api/internal/clusters/resources.go b/packages/api/internal/clusters/resources.go index 41da6867e7..453f997654 100644 --- a/packages/api/internal/clusters/resources.go +++ b/packages/api/internal/clusters/resources.go @@ -12,6 +12,7 @@ import ( edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" + "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) @@ -111,3 +112,71 @@ func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID return entries, nil } } + +// getBuildLogsWithSources implements the shared logic for fetching build logs from multiple sources. +// This function extracts the common pattern used by both local and remote cluster resource providers, +// avoiding code duplication between the two implementations. +// +// The function tries log sources in order based on availability and configuration: +// 1. Temporary logs from the builder instance (if nodeID is provided and source allows) +// 2. Persistent logs from backend storage (strategy provided by caller) +// +// It returns the first successful result, logging warnings for any failures encountered. +// This unified approach ensures consistent behavior and makes maintenance easier by centralizing +// the source selection and fallback logic. +func getBuildLogsWithSources( + ctx context.Context, + instances *smap.Map[*Instance], + nodeID *string, + templateID string, + buildID string, + offset int32, + limit int32, + level *logs.LogLevel, + cursor *time.Time, + direction api.LogsDirection, + source *api.LogsSource, + persistentLogFetcher logSourceFunc, // Backend-specific strategy for persistent logs (Loki for local, Edge API for remote) +) ([]logs.LogEntry, error) { + ctx, span := tracer.Start(ctx, "get build-logs") + defer span.End() + + start, end := logQueryWindow(cursor, direction) + + var sources []logSourceFunc + + // Handle temporary logs from builder instance + if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { + instance, found := instances.Get(*nodeID) + if found { + sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) + sources = append(sources, sourceCallback) + } else { + logger.L().Warn( + ctx, "Node instance not found for build logs, falling back to other sources", + logger.WithNodeID(*nodeID), + logger.WithTemplateID(templateID), + logger.WithBuildID(buildID), + ) + } + } + + // Handle persistent logs (backend-specific implementation provided by caller) + if logCheckSourceType(source, api.LogsSourcePersistent) { + sources = append(sources, persistentLogFetcher) + } + + // Iterate through sources and return the first successful fetch + for _, sourceFetch := range sources { + entries, err := sourceFetch() + if err != nil { + logger.L().Warn(ctx, "Error fetching build logs", logger.WithTemplateID(templateID), logger.WithBuildID(buildID), zap.Error(err)) + + continue + } + + return entries, nil + } + + return nil, nil +} diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go index 5ca5c35800..36ac46033c 100644 --- a/packages/api/internal/clusters/resources_local.go +++ b/packages/api/internal/clusters/resources_local.go @@ -152,53 +152,17 @@ func (l *LocalClusterResourceProvider) GetBuildLogs( direction api.LogsDirection, source *api.LogsSource, ) ([]logs.LogEntry, error) { - ctx, span := tracer.Start(ctx, "get build-logs") - defer span.End() - + // Use shared implementation with Loki as the persistent log backend start, end := logQueryWindow(cursor, direction) - var sources []logSourceFunc - - if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { - instance, found := l.instances.Get(*nodeID) - if found { - sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - sources = append(sources, sourceCallback) - } else { - logger.L().Warn( - ctx, "Node instance not found for build logs, falling back to other sources", - logger.WithNodeID(*nodeID), - logger.WithTemplateID(templateID), - logger.WithBuildID(buildID), - ) - } - } - - // Fetch logs from Loki backend (explicit persistent source or default) - if logCheckSourceType(source, api.LogsSourcePersistent) { - lokiDirection := defaultDirection - if direction == api.LogsDirectionBackward { - lokiDirection = logproto.BACKWARD - } - - sourceCallback := l.logsFromLocalLoki(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) - sources = append(sources, sourceCallback) + lokiDirection := defaultDirection + if direction == api.LogsDirectionBackward { + lokiDirection = logproto.BACKWARD } - // Iterate through sources and return the first successful fetch, - // it depends on nodeID and source what sources are available here. - for _, sourceFetch := range sources { - entries, err := sourceFetch() - if err != nil { - logger.L().Warn(ctx, "Error fetching build logs", logger.WithTemplateID(templateID), logger.WithBuildID(buildID), zap.Error(err)) - - continue - } - - return entries, nil - } + persistentFetcher := l.logsFromLocalLoki(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection) - return nil, nil + return getBuildLogsWithSources(ctx, l.instances, nodeID, templateID, buildID, offset, limit, level, cursor, direction, source, persistentFetcher) } func (l *LocalClusterResourceProvider) logsFromLocalLoki(ctx context.Context, templateID string, buildID string, start time.Time, end time.Time, limit int, offset int32, level *logs.LogLevel, direction logproto.Direction) logSourceFunc { diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go index ec535f7a45..6398db8c9d 100644 --- a/packages/api/internal/clusters/resources_remote.go +++ b/packages/api/internal/clusters/resources_remote.go @@ -7,11 +7,8 @@ import ( "net/http" "time" - "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/api/internal/api" edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge" - "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/logs" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/utils" @@ -147,47 +144,11 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs( direction api.LogsDirection, source *api.LogsSource, ) ([]logs.LogEntry, error) { - ctx, span := tracer.Start(ctx, "get build-logs") - defer span.End() - + // Use shared implementation with Edge API as the persistent log backend start, end := logQueryWindow(cursor, direction) + persistentFetcher := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction) - var sources []logSourceFunc - - if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) { - instance, found := r.instances.Get(*nodeID) - if found { - sourceCallback := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction) - sources = append(sources, sourceCallback) - } else { - logger.L().Warn( - ctx, "Node instance not found for build logs, falling back to other sources", - logger.WithNodeID(*nodeID), - logger.WithTemplateID(templateID), - logger.WithBuildID(buildID), - ) - } - } - - if logCheckSourceType(source, api.LogsSourcePersistent) { - sourceCallback := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction) - sources = append(sources, sourceCallback) - } - - // Iterate through sources and return the first successful fetch, - // it depends on nodeID and source what sources are available here. - for _, sourceFetch := range sources { - entries, err := sourceFetch() - if err != nil { - logger.L().Warn(ctx, "Error fetching build logs", logger.WithTemplateID(templateID), logger.WithBuildID(buildID), zap.Error(err)) - - continue - } - - return entries, nil - } - - return nil, nil + return getBuildLogsWithSources(ctx, r.instances, nodeID, templateID, buildID, offset, limit, level, cursor, direction, source, persistentFetcher) } func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) logSourceFunc {