Repository navigation
ci(release): publish Edge release metadata for the dashboard #28
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
stelioschar
merged 10 commits into
NOFireAI:main
from
stelioschar:feat/27-release-metadata
Nov 17, 2026
Merged
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
388f992
test(chart): assert the values the dashboard install command sets
41b535c
ci(release): publish Edge release metadata to releases.nofire.ai
9ca4bd2
test(render): render a dashboard-made 0.9.0 release reused onto this …
2f9163c
ci(release): read latest.json with the role and write it conditionally
a3d886c
test(render): say what the dashboard reuse case does and does not assert
e81e47b
ci(release): treat a conflicting versioned write like an existing one
e7d903a
ci(release): fail before publishing metadata with a missing field
a4cfaa9
ci(release): publish release metadata from its own job, with no downl…
2f68def
test(render): drop the dashboard reuse case
0db32f1
ci(release): port the metadata publisher from Python to Go
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,212 @@ | ||
| // publish-release-metadata publishes Edge release metadata to releases.nofire.ai | ||
| // (NOFireAI/engineering#1983). | ||
| // | ||
| // The NOFire AI dashboard reads edge/latest.json to pick the chart version its | ||
| // install commands use. For the chart version in Chart.yaml this: | ||
| // | ||
| // 1. waits until the Helm repo serves it, so latest.json never names a version | ||
| // `helm repo update` cannot find yet; | ||
| // 2. writes edge/<version>.json once (never replaced); | ||
| // 3. moves edge/latest.json to <version>, unless it already names the same or a | ||
| // newer version, or <version> is a pre-release. | ||
| // | ||
| // Safe to re-run. Standard library only; it calls the tools on the runner: the | ||
| // AWS CLI (credentials for the release metadata role), gh and helm. | ||
| // | ||
| // Run from the chart's root: go run ./.github/scripts/publish-release-metadata | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| const ( | ||
| bucket = "releases.nofire.ai" | ||
| chartRepo = "https://nofireai.github.io/edge-helm-chart" | ||
| ) | ||
|
|
||
| type result struct { | ||
| ok bool | ||
| stdout, stderr string | ||
| } | ||
|
|
||
| func run(name string, args ...string) result { | ||
| var stdout, stderr bytes.Buffer | ||
| cmd := exec.Command(name, args...) | ||
| cmd.Stdout, cmd.Stderr = &stdout, &stderr | ||
| err := cmd.Run() | ||
| return result{err == nil, stdout.String(), stderr.String()} | ||
| } | ||
|
|
||
| func fail(format string, args ...any) { | ||
| fmt.Fprintf(os.Stderr, "::error::"+format+"\n", args...) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| // chartField reads a top-level scalar from Chart.yaml, e.g. appVersion. | ||
| func chartField(key string) string { | ||
| data, err := os.ReadFile("Chart.yaml") | ||
| if err != nil { | ||
| fail("reading Chart.yaml: %v", err) | ||
| } | ||
| for _, line := range strings.Split(string(data), "\n") { | ||
| name, value, found := strings.Cut(line, ":") | ||
| if found && name == key && strings.TrimSpace(value) != "" { | ||
| return strings.Trim(strings.TrimSpace(value), `"'`) | ||
| } | ||
| } | ||
| fail("Chart.yaml has no %s", key) | ||
| return "" | ||
| } | ||
|
|
||
| // parseVersion returns major, minor and patch for a plain x.y.z, else nil. | ||
| func parseVersion(version string) []int { | ||
| parts := strings.Split(version, ".") | ||
| if len(parts) != 3 { | ||
| return nil | ||
| } | ||
| nums := make([]int, 3) | ||
| for i, part := range parts { | ||
| n, err := strconv.Atoi(part) | ||
| if err != nil || n < 0 || strconv.Itoa(n) != part { | ||
| return nil | ||
| } | ||
| nums[i] = n | ||
| } | ||
| return nums | ||
| } | ||
|
|
||
| // atLeast reports whether version a >= version b. | ||
| func atLeast(a, b []int) bool { | ||
| for i := range a { | ||
| if a[i] != b[i] { | ||
| return a[i] > b[i] | ||
| } | ||
| } | ||
| return true | ||
| } | ||
|
|
||
| func waitForHelmRepo(name, version string) { | ||
| // GitHub Pages deploys after the gh-pages push; give it 10 minutes. | ||
| run("helm", "repo", "add", "--force-update", "nofire", chartRepo) | ||
| for range 40 { | ||
| run("helm", "repo", "update", "nofire") | ||
| if run("helm", "show", "chart", "nofire/"+name, "--version", version).ok { | ||
| return | ||
| } | ||
| time.Sleep(15 * time.Second) | ||
| } | ||
| fail("%s %s is not in the Helm repo after 10 minutes", name, version) | ||
| } | ||
|
|
||
| func s3(args ...string) result { | ||
| return run("aws", append(append([]string{"s3api"}, args...), "--bucket", bucket)...) | ||
| } | ||
|
|
||
| func put(key, body, cacheControl string, condition ...string) result { | ||
| args := []string{"put-object", "--key", key, "--body", body, | ||
| "--content-type", "application/json", "--cache-control", cacheControl} | ||
| return s3(append(args, condition...)...) | ||
| } | ||
|
|
||
| // lostRace reports S3's answer when a conditional write's condition no longer holds. | ||
| func lostRace(r result) bool { | ||
| return strings.Contains(r.stderr, "PreconditionFailed") || | ||
| strings.Contains(r.stderr, "ConditionalRequestConflict") | ||
| } | ||
|
|
||
| func main() { | ||
| name, version := chartField("name"), chartField("version") | ||
| waitForHelmRepo(name, version) | ||
|
|
||
| release := run("gh", "release", "view", name+"-"+version, "--json", "publishedAt,url") | ||
| if !release.ok { | ||
| fail("no GitHub release %s-%s\n%s", name, version, release.stderr) | ||
| } | ||
| var published struct{ PublishedAt, URL string } | ||
| if err := json.Unmarshal([]byte(release.stdout), &published); err != nil { | ||
| fail("reading gh release output: %v", err) | ||
| } | ||
|
|
||
| work, err := os.MkdirTemp("", "release-metadata") | ||
| if err != nil { | ||
| fail("%v", err) | ||
| } | ||
| metadata, _ := json.MarshalIndent(map[string]any{ | ||
|
stelioschar marked this conversation as resolved.
Resolved
|
||
| "schema": 1, | ||
| "chart": map[string]string{"name": name, "version": version, "repo": chartRepo}, | ||
| "edge": map[string]string{"version": chartField("appVersion")}, | ||
| "releasedAt": published.PublishedAt, | ||
| "releaseUrl": published.URL, | ||
| }, "", " ") | ||
| body := filepath.Join(work, "metadata.json") | ||
| if err := os.WriteFile(body, append(metadata, '\n'), 0o644); err != nil { | ||
| fail("%v", err) | ||
| } | ||
| fmt.Println(string(metadata)) | ||
|
|
||
| r := put("edge/"+version+".json", body, "public, max-age=31536000, immutable", "--if-none-match", "*") | ||
| if !r.ok && !lostRace(r) { | ||
| fail("writing edge/%s.json failed\n%s", version, r.stderr) | ||
| } | ||
| fmt.Printf("edge/%s.json is published\n", version) | ||
|
|
||
| next := parseVersion(version) | ||
| if next == nil { | ||
| fmt.Printf("%s is not a plain release, leaving latest.json alone\n", version) | ||
| return | ||
| } | ||
|
|
||
| // Read with the role, from S3 rather than through Cloudflare's cache. Only | ||
| // NoSuchKey means there is no latest.json; any other error stops here. The | ||
| // write only lands if latest.json still holds what was read, so of two | ||
| // releases racing, the loser reads again and compares again. | ||
| currentFile := filepath.Join(work, "latest.json") | ||
| for range 5 { | ||
| var current string | ||
| var condition []string | ||
| r := s3("get-object", "--key", "edge/latest.json", currentFile) | ||
| switch { | ||
| case r.ok: | ||
| var doc struct{ Chart struct{ Version string } } | ||
| if data, err := os.ReadFile(currentFile); err == nil { | ||
| _ = json.Unmarshal(data, &doc) // a malformed file is replaced | ||
| } | ||
| var head struct{ ETag string } | ||
| if err := json.Unmarshal([]byte(r.stdout), &head); err != nil || head.ETag == "" { | ||
| fail("no ETag reading edge/latest.json\n%s", r.stdout) | ||
| } | ||
| current, condition = doc.Chart.Version, []string{"--if-match", head.ETag} | ||
| case strings.Contains(r.stderr, "NoSuchKey"): | ||
| condition = []string{"--if-none-match", "*"} | ||
| default: | ||
| fail("reading edge/latest.json failed\n%s", r.stderr) | ||
| } | ||
|
|
||
| if held := parseVersion(current); held != nil && atLeast(held, next) { | ||
|
stelioschar marked this conversation as resolved.
Resolved
|
||
| fmt.Printf("latest.json holds %s, leaving it\n", current) | ||
| return | ||
| } | ||
| r = put("edge/latest.json", body, "public, max-age=300", condition...) | ||
| if r.ok { | ||
| if current == "" { | ||
| current = "nothing" | ||
| } | ||
| fmt.Printf("latest.json now holds %s (was %s)\n", version, current) | ||
| return | ||
| } | ||
| if !lostRace(r) { | ||
| fail("writing edge/latest.json failed\n%s", r.stderr) | ||
| } | ||
| fmt.Println("latest.json changed while this ran, reading it again") | ||
| } | ||
| fail("latest.json kept changing, gave up after 5 attempts") | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| # The values the NOFire AI dashboard's install, sources and disconnect commands | ||
| # set (app/dashboard/activation/installCommand.ts in NOFireAI/dashboard). | ||
| # | ||
| # Once a chart release publishes edge/latest.json, the dashboard offers that | ||
| # version to customers with no review step of its own, so this suite is the | ||
| # gate. Helm ignores values keys a chart does not read, so a renamed or dropped | ||
| # key would still render cleanly: each case asserts the value comes out the | ||
| # other side, not only that the render succeeds. | ||
| # | ||
| # Change these together with the dashboard's command builders. | ||
| suite: dashboard install command | ||
| templates: | ||
| - templates/configmap.yaml | ||
| - templates/deployment.yaml | ||
| - templates/edge-proxy-configmap.yaml | ||
| - templates/edge-proxy-deployment.yaml | ||
| tests: | ||
| - it: install puts the token, cluster name and netobs switch in config.json | ||
| template: templates/configmap.yaml | ||
| set: | ||
| config.publisher.apiKey: dash-test-token | ||
| config.kube.clusterName: dash-test-cluster | ||
| config.netobs.enabled: true | ||
| asserts: | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"apiKey": "dash-test-token"' | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"clusterName": "dash-test-cluster"' | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"netobs": \{\s*"enabled": true' | ||
|
|
||
| - it: a non-canonical origin sets the publisher graph URL | ||
| template: templates/configmap.yaml | ||
| set: | ||
| config.publisher.graph.url: https://dash-test.example/api/edge | ||
| asserts: | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"url": "https://dash-test.example/api/edge"' | ||
|
|
||
| - it: the dnstap verify command can select the edge pod by name label | ||
| template: templates/deployment.yaml | ||
| asserts: | ||
| - equal: | ||
| path: spec.template.metadata.labels["app.kubernetes.io/name"] | ||
| value: nofire-edge | ||
|
|
||
| - it: sources deploy edge-proxy with each connection | ||
| template: templates/edge-proxy-configmap.yaml | ||
| set: | ||
| config.publisher.apiKey: dash-test-token | ||
| config.kube.clusterName: dash-test-cluster | ||
| edgeProxy.enabled: true | ||
| onPremConnections: | ||
| prometheus: | ||
| type: prometheus | ||
| url: http://prometheus.monitoring:9090 | ||
| enabled: true | ||
| elasticsearch: | ||
| type: elasticsearch | ||
| url: http://es.logging:9200 | ||
| index: logs-* | ||
| enabled: true | ||
| asserts: | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"clusterName": "dash-test-cluster"' | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"prometheus": \{\s*"type": "prometheus",\s*"enabled": true' | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"url": "http://prometheus.monitoring:9090"' | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"index": "logs-\*"' | ||
|
|
||
| - it: edgeProxy.enabled renders the edge-proxy deployment | ||
| template: templates/edge-proxy-deployment.yaml | ||
| set: | ||
| edgeProxy.enabled: true | ||
| asserts: | ||
| - hasDocuments: | ||
| count: 1 | ||
| - isKind: | ||
| of: Deployment | ||
|
|
||
| - it: disconnect turns one connection off and keeps it listed | ||
| template: templates/edge-proxy-configmap.yaml | ||
| set: | ||
| edgeProxy.enabled: true | ||
| onPremConnections: | ||
| prometheus: | ||
| type: prometheus | ||
| url: http://prometheus.monitoring:9090 | ||
| enabled: false | ||
| asserts: | ||
| - matchRegex: | ||
| path: data["config.json"] | ||
| pattern: '"prometheus": \{\s*"type": "prometheus",\s*"enabled": false' |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.