Skip to content

Sandbox logs v2 endpoint #1843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sitole wants to merge 2 commits into from
Closed

Sandbox logs v2 endpoint #1843

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0999225
Update openapi spec
sitole Feb 3, 2026
72d5e0d
Implement direction for sandbox logs
sitole Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
533 changes: 399 additions & 134 deletions packages/api/internal/api/api.gen.go
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions packages/api/internal/clusters/resources.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,26 @@ import (
type ClusterResource interface {
GetSandboxMetrics(ctx context.Context, teamID string, sandboxID string, qStart *int64, qEnd *int64) ([]api.SandboxMetric, *api.APIError)
GetSandboxesMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, *api.APIError)
GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, limit *int32) (api.SandboxLogs, *api.APIError)
GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, end *int64, limit *int32, direction *api.LogsDirection) (api.SandboxLogs, *api.APIError)
GetBuildLogs(ctx context.Context, nodeID *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, *api.APIError)
}

const (
maxTimeRangeDuration = 7 * 24 * time.Hour
MaxTimeRangeDuration = 7 * 24 * time.Hour
)

func logQueryWindow(cursor *time.Time, direction api.LogsDirection) (time.Time, time.Time) {
start, end := time.Now().Add(-maxTimeRangeDuration), time.Now()
start, end := time.Now().Add(-MaxTimeRangeDuration), time.Now()
if cursor == nil {
return start, end
}

if direction == api.LogsDirectionForward {
start = *cursor
end = start.Add(maxTimeRangeDuration)
end = start.Add(MaxTimeRangeDuration)
} else {
end = *cursor
start = end.Add(-maxTimeRangeDuration)
start = end.Add(-MaxTimeRangeDuration)
}

return start, end
Expand Down
47 changes: 29 additions & 18 deletions packages/api/internal/clusters/resources_local.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,6 @@ type LocalClusterResourceProvider struct {
instances *smap.Map[*Instance]
}

const (
sandboxLogsOldestLimit = 168 * time.Hour // 7 days
defaultLogsLimit = 1000
defaultDirection = logproto.FORWARD
)

func newLocalClusterResourceProvider(
querySandboxMetricsProvider clickhouse.SandboxQueriesProvider,
queryLogsProvider *loki.LokiQueryProvider,
Expand Down Expand Up @@ -115,22 +109,39 @@ func (l *LocalClusterResourceProvider) GetSandboxesMetrics(ctx context.Context,
return metrics, nil
}

func (l *LocalClusterResourceProvider) GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, qStart *int64, qLimit *int32) (api.SandboxLogs, *api.APIError) {
end := time.Now()
var start time.Time

if qStart != nil {
start = time.UnixMilli(*qStart)
func (l *LocalClusterResourceProvider) GetSandboxLogs(
ctx context.Context,
teamID string,
sandboxID string,
start *int64,
end *int64,
limit *int32,
direction *api.LogsDirection,
) (api.SandboxLogs, *api.APIError) {
endTime := time.Now()
var startTime time.Time

if start != nil {
startTime = time.UnixMilli(*start)
} else {
start = end.Add(-sandboxLogsOldestLimit)
startTime = endTime.Add(-MaxTimeRangeDuration)
}

limit := defaultLogsLimit
if qLimit != nil {
limit = int(*qLimit)
if end != nil {
endTime = time.UnixMilli(*end)
}

lokiLimit := loki.DefaultLogsLimit
if limit != nil {
lokiLimit = int(*limit)
}

lokiDirection := loki.DefaultDirection
if direction != nil && *direction == api.LogsDirectionBackward {
lokiDirection = logproto.BACKWARD
}

raw, err := l.queryLogsProvider.QuerySandboxLogs(ctx, teamID, sandboxID, start, end, limit)
raw, err := l.queryLogsProvider.QuerySandboxLogs(ctx, teamID, sandboxID, startTime, endTime, lokiLimit, lokiDirection)
if err != nil {
return api.SandboxLogs{}, &api.APIError{
Err: fmt.Errorf("error when fetching sandbox logs: %w", err),
Expand Down Expand Up @@ -172,7 +183,7 @@ func (l *LocalClusterResourceProvider) GetBuildLogs(
// Use shared implementation with Loki as the persistent log backend
start, end := logQueryWindow(cursor, direction)

lokiDirection := defaultDirection
lokiDirection := loki.DefaultDirection
if direction == api.LogsDirectionBackward {
lokiDirection = logproto.BACKWARD
}
Expand Down
37 changes: 33 additions & 4 deletions packages/api/internal/clusters/resources_remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@ func newRemoteClusterResourceProvider(instances *smap.Map[*Instance], client *ed
}
}

func (r *ClusterResourceProviderImpl) GetSandboxMetrics(ctx context.Context, teamID string, sandboxID string, qStart *int64, qEnd *int64) ([]api.SandboxMetric, *api.APIError) {
func (r *ClusterResourceProviderImpl) GetSandboxMetrics(
ctx context.Context,
teamID string,
sandboxID string,
qStart *int64,
qEnd *int64,
) ([]api.SandboxMetric, *api.APIError) {
req := &edgeapi.V1SandboxMetricsParams{
TeamID: teamID,
Start: qStart,
Expand Down Expand Up @@ -63,7 +69,11 @@ func (r *ClusterResourceProviderImpl) GetSandboxMetrics(ctx context.Context, tea
return items, nil
}

func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, teamID string, sandboxIDs []string) (map[string]api.SandboxMetric, *api.APIError) {
func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(
ctx context.Context,
teamID string,
sandboxIDs []string,
) (map[string]api.SandboxMetric, *api.APIError) {
res, err := r.client.V1SandboxesMetricsWithResponse(ctx, &edgeapi.V1SandboxesMetricsParams{TeamID: teamID, SandboxIds: sandboxIDs})
if err != nil {
return nil, &api.APIError{
Expand Down Expand Up @@ -95,8 +105,27 @@ func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, t
return items, nil
}

func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID string, sandboxID string, start *int64, limit *int32) (api.SandboxLogs, *api.APIError) {
res, err := r.client.V1SandboxLogsWithResponse(ctx, sandboxID, &edgeapi.V1SandboxLogsParams{TeamID: teamID, Start: start, Limit: limit})
func (r *ClusterResourceProviderImpl) GetSandboxLogs(
ctx context.Context,
teamID string,
sandboxID string,
start *int64,
end *int64,
limit *int32,
direction *api.LogsDirection,
) (api.SandboxLogs, *api.APIError) {
edgeDirection := edgeapi.V1SandboxLogsParamsDirectionForward
if direction != nil && *direction == api.LogsDirectionBackward {
edgeDirection = edgeapi.V1SandboxLogsParamsDirectionBackward
}

res, err := r.client.V1SandboxLogsWithResponse(ctx, sandboxID, &edgeapi.V1SandboxLogsParams{
TeamID: teamID,
Start: start,
End: end,
Limit: limit,
Direction: utils.ToPtr(edgeDirection),
})
if err != nil {
return api.SandboxLogs{}, &api.APIError{
Err: err,
Expand Down
58 changes: 57 additions & 1 deletion packages/api/internal/handlers/sandbox_logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ package handlers
import (
"fmt"
"net/http"
"time"

"github.com/gin-gonic/gin"
"go.opentelemetry.io/otel/attribute"

"github.com/e2b-dev/infra/packages/api/internal/api"
"github.com/e2b-dev/infra/packages/api/internal/auth"
"github.com/e2b-dev/infra/packages/api/internal/clusters"
"github.com/e2b-dev/infra/packages/api/internal/db/types"
"github.com/e2b-dev/infra/packages/api/internal/utils"
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
Expand All @@ -34,7 +36,7 @@ func (a *APIStore) GetSandboxesSandboxIDLogs(c *gin.Context, sandboxID string, p
return
}

logs, apiErr := cluster.GetResources().GetSandboxLogs(ctx, team.ID.String(), sandboxID, params.Start, params.Limit)
logs, apiErr := cluster.GetResources().GetSandboxLogs(ctx, team.ID.String(), sandboxID, params.Start, nil, params.Limit, nil)
if apiErr != nil {
telemetry.ReportCriticalError(ctx, "error when returning logs for sandbox", apiErr.Err)
a.sendAPIStoreError(c, apiErr.Code, apiErr.ClientMsg)
Expand All @@ -44,3 +46,57 @@ func (a *APIStore) GetSandboxesSandboxIDLogs(c *gin.Context, sandboxID string, p

c.JSON(http.StatusOK, logs)
}

func (a *APIStore) GetV2SandboxesSandboxIDLogs(c *gin.Context, sandboxID api.SandboxID, params api.GetV2SandboxesSandboxIDLogsParams) {
ctx := c.Request.Context()
sandboxID = utils.ShortID(sandboxID)

team := c.Value(auth.TeamContextKey).(*types.Team)

telemetry.SetAttributes(ctx,
attribute.String("instance.id", sandboxID),
telemetry.WithTeamID(team.ID.String()),
)

clusterID := utils.WithClusterFallback(team.ClusterID)
cluster, ok := a.clusters.GetClusterById(clusterID)
if !ok {
telemetry.ReportCriticalError(ctx, "error getting cluster by ID", fmt.Errorf("cluster with ID '%s' not found", clusterID))
a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error getting cluster '%s'", clusterID))

return
}

// Default to forward direction if not specified
direction := api.LogsDirectionForward
if params.Direction != nil {
direction = *params.Direction
}

start, end := time.Now().Add(-clusters.MaxTimeRangeDuration), time.Now()
if params.Cursor != nil {
cursor := time.UnixMilli(*params.Cursor)
if direction == api.LogsDirectionForward {
start = cursor
end = cursor.Add(clusters.MaxTimeRangeDuration)
} else {
end = cursor
start = cursor.Add(-clusters.MaxTimeRangeDuration)
}
}

startMs := start.UnixMilli()
endMs := end.UnixMilli()

logs, apiErr := cluster.GetResources().GetSandboxLogs(ctx, team.ID.String(), sandboxID, &startMs, &endMs, params.Limit, params.Direction)
Comment thread
sitole marked this conversation as resolved.
if apiErr != nil {
telemetry.ReportCriticalError(ctx, "error when returning logs for sandbox", apiErr.Err)
a.sendAPIStoreError(c, apiErr.Code, apiErr.ClientMsg)

return
}

c.JSON(http.StatusOK, api.SandboxLogsV2Response{
Logs: logs.LogEntries,
})
}
51 changes: 49 additions & 2 deletions packages/shared/pkg/http/edge/generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 26 additions & 3 deletions packages/shared/pkg/logs/loki/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ type LokiQueryProvider struct {
client *loki.DefaultClient
}

const (
DefaultLogsLimit = 1000
DefaultDirection = logproto.FORWARD
)

func NewLokiQueryProvider(lokiURL string, lokiUser string, lokiPassword string) (*LokiQueryProvider, error) {
lokiClient := &loki.DefaultClient{
Address: lokiURL,
Expand All @@ -29,7 +34,17 @@ func NewLokiQueryProvider(lokiURL string, lokiUser string, lokiPassword string)
return &LokiQueryProvider{client: lokiClient}, nil
}

func (l *LokiQueryProvider) QueryBuildLogs(ctx context.Context, templateID string, buildID string, start time.Time, end time.Time, limit int, offset int32, level *logs.LogLevel, direction logproto.Direction) ([]logs.LogEntry, error) {
func (l *LokiQueryProvider) QueryBuildLogs(
ctx context.Context,
templateID string,
buildID string,
start time.Time,
end time.Time,
limit int,
offset int32,
level *logs.LogLevel,
direction logproto.Direction,
) ([]logs.LogEntry, error) {
// https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/
templateIDSanitized := strings.ReplaceAll(templateID, "`", "")
buildIDSanitized := strings.ReplaceAll(buildID, "`", "")
Expand All @@ -56,14 +71,22 @@ func (l *LokiQueryProvider) QueryBuildLogs(ctx context.Context, templateID strin
return lm, nil
}

func (l *LokiQueryProvider) QuerySandboxLogs(ctx context.Context, teamID string, sandboxID string, start time.Time, end time.Time, limit int) ([]logs.LogEntry, error) {
func (l *LokiQueryProvider) QuerySandboxLogs(
ctx context.Context,
teamID string,
sandboxID string,
start time.Time,
end time.Time,
limit int,
direction logproto.Direction,
) ([]logs.LogEntry, error) {
// https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/
sandboxIdSanitized := strings.ReplaceAll(sandboxID, "`", "")
teamIdSanitized := strings.ReplaceAll(teamID, "`", "")

query := fmt.Sprintf("{teamID=`%s`, sandboxID=`%s`, category!=\"metrics\"}", teamIdSanitized, sandboxIdSanitized)

res, err := l.client.QueryRange(query, limit, start, end, logproto.FORWARD, time.Duration(0), time.Duration(0), true)
res, err := l.client.QueryRange(query, limit, start, end, direction, time.Duration(0), time.Duration(0), true)
if err != nil {
telemetry.ReportError(ctx, "error when returning logs for sandbox", err)
logger.L().Error(ctx, "error when returning logs for sandbox", zap.Error(err), logger.WithSandboxID(sandboxID))
Expand Down
Loading