Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions controller/Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,18 @@ RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1
go build -a \
-ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \
-o router ./cmd/router
RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \
--mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \
CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \
go build -a \
-ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \
-o telemetry ./cmd/telemetry

FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1786321990@sha256:7e7f79ab747bf2b452e3043dd89f388e92be4c7fdcc8b815b58adf6c99c39c95
WORKDIR /
COPY --from=builder /build/manager .
COPY --from=builder /build/router .
COPY --from=builder /build/telemetry .
USER 65532:65532

ENTRYPOINT ["/manager"]
2 changes: 2 additions & 0 deletions controller/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ build-operator-ci:
build: manifests generate fmt vet ## Build manager binary.
go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go
go build -ldflags "$(LDFLAGS)" -o bin/router ./cmd/router
go build -ldflags "$(LDFLAGS)" -o bin/telemetry ./cmd/telemetry
go build -ldflags "$(LDFLAGS)" -o bin/exporter-set-controller cmd/exporter-set-controller/main.go

.PHONY: run
Expand All @@ -149,6 +150,7 @@ docker-build-ci: ## Build docker images from pre-compiled host binaries (fast CI
rm -rf bin/ci-stage && mkdir -p bin/ci-stage/controller bin/ci-stage/esc
CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/manager cmd/main.go
CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/router ./cmd/router
CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/telemetry ./cmd/telemetry
CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/esc/exporter-set-controller cmd/exporter-set-controller/main.go
$(CONTAINER_TOOL) build --build-arg BIN=manager -t $(IMG) -f Containerfile.prebuilt bin/ci-stage/controller
$(CONTAINER_TOOL) build --build-arg BIN=exporter-set-controller -t $(EXPORTER_SET_CONTROLLER_IMG) -f Containerfile.prebuilt bin/ci-stage/esc
Expand Down
49 changes: 42 additions & 7 deletions controller/cmd/telemetry/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// jumpstarter-telemetry receives structured log entries from exporters and clients
// via the PushLogs gRPC RPC and writes them to structured stdout for downstream
// log shippers (Promtail, Grafana Alloy, Vector) to forward to Loki.
// jumpstarter-telemetry reverse-scrapes exporter metrics via MetricsStream and
// receives structured log entries via PushLogs. Logs are written to structured
// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector).
// Loki push is a later Phase 3 PR.
//
// TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of
// operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a
Expand All @@ -30,16 +31,17 @@ limitations under the License.
// certificate; the controller uses it to advertise the address to exporters via
// GetServiceEndpoints. A mismatch causes TLS hostname verification failures.
//
// Future phases will add direct Loki push and MetricsStream for reverse-scrape
// of exporter prometheus_client registries.
// HTTP: GET /metrics, /healthz, and /readyz bind separately (default :8080).
package main

import (
"context"
"flag"
"os"
"os/signal"
"strings"
"syscall"
"time"

ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
Expand All @@ -55,9 +57,36 @@ var (
buildDate = "unknown"
)

func splitCSV(s string) []string {
if s == "" {
return nil
}
parts := strings.Split(s, ",")
out := make([]string, 0, len(parts))
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
out = append(out, p)
}
}
return out
}

func main() {
var bindAddr string
var metricsAddr string
var scrapeTimeout time.Duration
var driverTypeEnum string
var exemplarKeys string
flag.StringVar(&bindAddr, "grpc-bind", ":9093", "TCP address to bind the gRPC server to")
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080",
"TCP address for HTTP GET /metrics, /healthz, and /readyz. Use 0 to disable.")
flag.DurationVar(&scrapeTimeout, "scrape-timeout", 7*time.Second,
"Max wait for parallel MetricsStream scrape responses")
flag.StringVar(&driverTypeEnum, "driver-type-enum", strings.Join(service.DefaultDriverTypeEnum, ","),
"Comma-separated allowlist of driver_type values; others are remapped to other")
flag.StringVar(&exemplarKeys, "exemplar-keys", strings.Join(service.DefaultExemplarKeys, ","),
"Comma-separated allowlist of Prometheus exemplar keys")

opts := zap.Options{}
opts.BindFlags(flag.CommandLine)
Expand All @@ -71,6 +100,8 @@ func main() {
"gitCommit", gitCommit,
"buildDate", buildDate,
"bindAddr", bindAddr,
"metricsBindAddr", metricsAddr,
"scrapeTimeout", scrapeTimeout,
)

ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -87,8 +118,12 @@ func main() {
}

svc := &service.TelemetryService{
BindAddr: bindAddr,
Signer: signer,
BindAddr: bindAddr,
MetricsBindAddr: metricsAddr,
ScrapeTimeout: scrapeTimeout,
DriverTypeEnum: splitCSV(driverTypeEnum),
ExemplarKeys: splitCSV(exemplarKeys),
Signer: signer,
}

// Register signal handler before starting the service so no signal
Expand Down
20 changes: 20 additions & 0 deletions controller/deploy/operator/api/v1alpha1/jumpstarter_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,10 @@ type TelemetryConfig struct {
// gRPC configuration for the telemetry service.
// Use this to configure TLS when not using cert-manager.
GRPC TelemetryGRPCConfig `json:"grpc,omitempty"`

// Metrics configures reverse-scrape fan-out and Prometheus exposition
// (JEP-0013). Loki and ServiceMonitor fields are later phases.
Metrics TelemetryMetricsConfig `json:"metrics,omitempty"`
}

// TelemetryGRPCConfig defines gRPC configuration for the telemetry service.
Expand All @@ -325,6 +329,22 @@ type TelemetryGRPCConfig struct {
TLS TLSConfig `json:"tls,omitempty"`
}

// TelemetryMetricsConfig configures telemetry /metrics reverse-scrape behavior.
type TelemetryMetricsConfig struct {
// Allowlist of keys to include in Prometheus exemplars. Unlisted keys are omitted.
// +kubebuilder:default={"client","lease_id"}
ExemplarKeys []string `json:"exemplarKeys,omitempty"`

// Allowed driver_type label values. Unlisted types are remapped to "other".
// +kubebuilder:default={"power","storage","network","serial","console","video","composite"}
DriverTypeEnum []string `json:"driverTypeEnum,omitempty"`

Comment on lines +334 to +341

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A user with CR write access can set either list to an arbitrarily large array for ExemplarKeys []string and DriverTypeEnum []string. Adding // +kubebuilder:validation:MaxItems=<insert a good limit here> and // +kubebuilder:validation:MaxLength=<insert a good limit here> per item would enforce a bound.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could be worth limiting to 16 for example? /8 default + 8 custom, more than enough.

// Max wait for parallel exporter MetricsStream responses during a /metrics fan-out.
// Should be lower than the Prometheus scrape_timeout.
// +kubebuilder:default="7s"
ScrapeTimeout *metav1.Duration `json:"scrapeTimeout,omitempty"`
Comment on lines +342 to +345

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also set an upper limit to avoid keeping connections open for very long times.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2 minutes max? (and sounds excessive)

}

// TelemetryLoggingConfig configures the log push path to the telemetry service.
type TelemetryLoggingConfig struct {
// Filter controls which log entries are forwarded to the telemetry service.
Expand Down
31 changes: 31 additions & 0 deletions controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.18.0
controller-gen.kubebuilder.io/version: v0.21.0
name: jumpstarters.operator.jumpstarter.dev
spec:
group: operator.jumpstarter.dev
Expand Down Expand Up @@ -1087,7 +1087,8 @@ spec:
Number of controller replicas to run.
Currently only 1 replica is supported because the controller uses in-memory
state for gRPC stream coordination (Dial/Listen). Values greater than 1 will
be clamped to 1 with a warning. See issue 1013 for HA controller support.
be clamped to 1 with a warning. See https://github.com/jumpstarter-dev/jumpstarter/issues/1013
for the tracking issue on HA controller support.
format: int32
minimum: 1
type: integer
Expand Down Expand Up @@ -2163,6 +2164,41 @@ spec:
type: string
type: object
type: object
metrics:
description: |-
Metrics configures reverse-scrape fan-out and Prometheus exposition
(JEP-0013). Loki and ServiceMonitor fields are later phases.
properties:
driverTypeEnum:
default:
- power
- storage
- network
- serial
- console
- video
- composite
description: Allowed driver_type label values. Unlisted types
are remapped to "other".
items:
type: string
type: array
exemplarKeys:
default:
- client
- lease_id
description: Allowlist of keys to include in Prometheus exemplars.
Unlisted keys are omitted.
items:
type: string
type: array
scrapeTimeout:
default: 7s
description: |-
Max wait for parallel exporter MetricsStream responses during a /metrics fan-out.
Should be lower than the Prometheus scrape_timeout.
type: string
type: object
replicas:
default: 1
description: |-
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1326,7 +1326,8 @@ func (r *JumpstarterReconciler) buildConfig(ctx context.Context, jumpstarter *op
Keys: jumpstarter.Spec.DeprecatedLabels.Keys,
}

// Telemetry configuration.
// Telemetry configuration. When cert-manager is enabled, inline the CA so
// GetServiceEndpoints.certificate lets exporters verify telemetry TLS.
if jumpstarter.Spec.Telemetry != nil && jumpstarter.Spec.Telemetry.Enabled {
t := jumpstarter.Spec.Telemetry
telemetryCfg := &config.Telemetry{
Expand Down
Loading
Loading