From 9afa79d944cdac55bd00c0b21ab0c2dc24fabaeb Mon Sep 17 00:00:00 2001 From: Manjunath Basavaraj Bhadrannavar Date: Tue, 12 Aug 2025 23:57:54 +0200 Subject: [PATCH 1/4] feat: Add Github Status Metrics --- docs/partials/metrics.md | 30 ++++ go.mod | 2 + go.sum | 7 + hack/generate-metrics-docs.go | 5 + pkg/action/server.go | 13 ++ pkg/command/command.go | 7 + pkg/config/config.go | 1 + pkg/exporter/status.go | 306 ++++++++++++++++++++++++++++++++++ pkg/exporter/status_test.go | 137 +++++++++++++++ 9 files changed, 508 insertions(+) create mode 100644 pkg/exporter/status.go create mode 100644 pkg/exporter/status_test.go diff --git a/docs/partials/metrics.md b/docs/partials/metrics.md index 0111c106..0b0e95ee 100644 --- a/docs/partials/metrics.md +++ b/docs/partials/metrics.md @@ -241,6 +241,36 @@ github_runner_repo_busy{owner, id, name, os, status} github_runner_repo_online{owner, id, name, os, status} : Static metrics of runner is online or not +github_status_actions_up{} +: Current health status of Actions on githubstatus.com + +github_status_api_requests_up{} +: Current health status of API Requests on githubstatus.com + +github_status_codespaces_up{} +: Current health status of Codespaces on githubstatus.com + +github_status_copilot_up{} +: Current health status of Copilot on githubstatus.com + +github_status_git_operations_up{} +: Current health status of Git Operations on githubstatus.com + +github_status_issues_up{} +: Current health status of Issues on githubstatus.com + +github_status_packages_up{} +: Current health status of Packages on githubstatus.com + +github_status_pages_up{} +: Current health status of Pages on githubstatus.com + +github_status_pull_requests_up{} +: Current health status of Pull Requests on githubstatus.com + +github_status_webhooks_up{} +: Current health status of Webhooks on githubstatus.com + github_workflow_job_created_timestamp{owner, repo, name, title, branch, sha, identifier, run_id, run_attempt, labels, runner_id, runner_name, runner_group_id, runner_group_name, workflow_name, conclusion} : Timestamp when the workflow job have been created diff --git a/go.mod b/go.mod index 130f2291..65e5165e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.0 require ( github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 + github.com/PuerkitoBio/goquery v1.9.2 github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 github.com/cenkalti/backoff/v5 v5.0.3 github.com/chaisql/chai v0.16.0 @@ -47,6 +48,7 @@ require ( github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/ashanbrown/forbidigo/v2 v2.1.0 // indirect github.com/ashanbrown/makezero/v2 v2.0.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index b6e88aa5..76a7f863 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7r github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.19.0 h1:Im+SLRgT8maArxv81mULDWN8oKxkzboH07CHesxElq4= @@ -50,6 +52,8 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/ashanbrown/forbidigo/v2 v2.1.0 h1:NAxZrWqNUQiDz19FKScQ/xvwzmij6BiOw3S0+QUQ+Hs= github.com/ashanbrown/forbidigo/v2 v2.1.0/go.mod h1:0zZfdNAuZIL7rSComLGthgc/9/n2FqspBOH90xlCHdA= github.com/ashanbrown/makezero/v2 v2.0.1 h1:r8GtKetWOgoJ4sLyUx97UTwyt2dO7WkGFHizn/Lo8TY= @@ -620,6 +624,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -656,6 +661,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -664,6 +670,7 @@ golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= diff --git a/hack/generate-metrics-docs.go b/hack/generate-metrics-docs.go index 168bdb06..a119efb1 100644 --- a/hack/generate-metrics-docs.go +++ b/hack/generate-metrics-docs.go @@ -65,6 +65,11 @@ func main() { exporter.NewWorkflowJobCollector(slog.Default(), nil, nil, nil, nil, cfg).Metrics()..., ) + collectors = append( + collectors, + exporter.NewStatusCollector(slog.Default(), nil, nil, nil, nil, cfg).Metrics()..., + ) + metrics := make([]metric, 0) metrics = append(metrics, metric{ diff --git a/pkg/action/server.go b/pkg/action/server.go index 63a314b6..d8ca7693 100644 --- a/pkg/action/server.go +++ b/pkg/action/server.go @@ -201,6 +201,19 @@ func handler(cfg *config.Config, db store.Store, logger *slog.Logger, client *gi )) } + if cfg.Collector.Status { + logger.Debug("Status collector registered") + + registry.MustRegister(exporter.NewStatusCollector( + logger, + client, + db, + requestFailures, + requestDuration, + cfg.Target, + )) + } + reg := promhttp.HandlerFor( registry, promhttp.HandlerOpts{ diff --git a/pkg/command/command.go b/pkg/command/command.go index d0cd021b..7bba2dd1 100644 --- a/pkg/command/command.go +++ b/pkg/command/command.go @@ -363,6 +363,13 @@ func RootFlags(cfg *config.Config) []cli.Flag { Sources: cli.EnvVars("GITHUB_EXPORTER_COLLECTOR_RUNNERS"), Destination: &cfg.Collector.Runners, }, + &cli.BoolFlag{ + Name: "collector.status", + Value: false, + Usage: "Enable collector for github.com service status", + Sources: cli.EnvVars("GITHUB_EXPORTER_COLLECTOR_STATUS"), + Destination: &cfg.Collector.Status, + }, &cli.StringSliceFlag{ Name: "collector.runners.labels", Value: config.RunnerLabels(), diff --git a/pkg/config/config.go b/pkg/config/config.go index 18f47abe..fd3826ed 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -75,6 +75,7 @@ type Collector struct { WorkflowRuns bool WorkflowJobs bool Runners bool + Status bool } // Database defines the database specific configuration. diff --git a/pkg/exporter/status.go b/pkg/exporter/status.go new file mode 100644 index 00000000..4f1c5308 --- /dev/null +++ b/pkg/exporter/status.go @@ -0,0 +1,306 @@ +package exporter + +import ( + "io" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/google/go-github/v74/github" + "github.com/prometheus/client_golang/prometheus" + "github.com/promhippie/github_exporter/pkg/config" + "github.com/promhippie/github_exporter/pkg/store" +) + +// statusComponent represents a GitHub.com service shown on githubstatus.com. +type statusComponent string + +const ( + compGitOperations statusComponent = "Git Operations" + compWebhooks statusComponent = "Webhooks" + compAPIRequests statusComponent = "API Requests" + compIssues statusComponent = "Issues" + compPullRequests statusComponent = "Pull Requests" + compActions statusComponent = "Actions" + compPackages statusComponent = "Packages" + compPages statusComponent = "Pages" + compCodespaces statusComponent = "Codespaces" + compCopilot statusComponent = "Copilot" +) + +// statusComponents defines the ordered list of services we expose as gauges. +var statusComponents = []statusComponent{ + compGitOperations, + compWebhooks, + compAPIRequests, + compIssues, + compPullRequests, + compActions, + compPackages, + compPages, + compCodespaces, + compCopilot, +} + +func isStatusComponent(name string) bool { + trimmed := strings.TrimSpace(name) + for _, c := range statusComponents { + if string(c) == trimmed { + return true + } + } + return false +} + +// StatusCollector exposes gauges for GitHub component status. +type StatusCollector struct { + client *github.Client + logger *slog.Logger + db store.Store + failures *prometheus.CounterVec + duration *prometheus.HistogramVec + config config.Target + + GitOperationsUp *prometheus.Desc + WebhooksUp *prometheus.Desc + APIRequestsUp *prometheus.Desc + IssuesUp *prometheus.Desc + PullRequestsUp *prometheus.Desc + ActionsUp *prometheus.Desc + PackagesUp *prometheus.Desc + PagesUp *prometheus.Desc + CodespacesUp *prometheus.Desc + CopilotUp *prometheus.Desc +} + +// NewStatusCollector returns a new StatusCollector with metric descriptors only. +func NewStatusCollector(logger *slog.Logger, client *github.Client, db store.Store, failures *prometheus.CounterVec, duration *prometheus.HistogramVec, cfg config.Target) *StatusCollector { + if failures != nil { + failures.WithLabelValues("status").Add(0) + } + + labels := []string{} + return &StatusCollector{ + client: client, + logger: logger.With("collector", "status"), + db: db, + failures: failures, + duration: duration, + config: cfg, + + GitOperationsUp: prometheus.NewDesc( + "github_status_git_operations_up", + "Current health status of Git Operations on githubstatus.com", + labels, + nil, + ), + WebhooksUp: prometheus.NewDesc( + "github_status_webhooks_up", + "Current health status of Webhooks on githubstatus.com", + labels, + nil, + ), + APIRequestsUp: prometheus.NewDesc( + "github_status_api_requests_up", + "Current health status of API Requests on githubstatus.com", + labels, + nil, + ), + IssuesUp: prometheus.NewDesc( + "github_status_issues_up", + "Current health status of Issues on githubstatus.com", + labels, + nil, + ), + PullRequestsUp: prometheus.NewDesc( + "github_status_pull_requests_up", + "Current health status of Pull Requests on githubstatus.com", + labels, + nil, + ), + ActionsUp: prometheus.NewDesc( + "github_status_actions_up", + "Current health status of Actions on githubstatus.com", + labels, + nil, + ), + PackagesUp: prometheus.NewDesc( + "github_status_packages_up", + "Current health status of Packages on githubstatus.com", + labels, + nil, + ), + PagesUp: prometheus.NewDesc( + "github_status_pages_up", + "Current health status of Pages on githubstatus.com", + labels, + nil, + ), + CodespacesUp: prometheus.NewDesc( + "github_status_codespaces_up", + "Current health status of Codespaces on githubstatus.com", + labels, + nil, + ), + CopilotUp: prometheus.NewDesc( + "github_status_copilot_up", + "Current health status of Copilot on githubstatus.com", + labels, + nil, + ), + } +} + +// Metrics returns descriptors for documentation generation. +func (c *StatusCollector) Metrics() []*prometheus.Desc { + return []*prometheus.Desc{ + c.GitOperationsUp, + c.WebhooksUp, + c.APIRequestsUp, + c.IssuesUp, + c.PullRequestsUp, + c.ActionsUp, + c.PackagesUp, + c.PagesUp, + c.CodespacesUp, + c.CopilotUp, + } +} + +// Describe sends all possible descriptors. +func (c *StatusCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.GitOperationsUp + ch <- c.WebhooksUp + ch <- c.APIRequestsUp + ch <- c.IssuesUp + ch <- c.PullRequestsUp + ch <- c.ActionsUp + ch <- c.PackagesUp + ch <- c.PagesUp + ch <- c.CodespacesUp + ch <- c.CopilotUp +} + +// Collect intentionally left empty for now; population will be implemented later. +func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { + // Perform a single scrape of the status page and populate all gauges. + // Follow redirects; treat "Operational" as up (1), everything else as down (0). + client := &http.Client{Timeout: c.config.Timeout} + + now := time.Now() + req, err := http.NewRequest("GET", "https://www.githubstatus.com/", nil) + if err != nil { + c.logger.Error("Failed to build status request", "err", err) + if c.failures != nil { + c.failures.WithLabelValues("status").Inc() + } + return + } + + resp, err := client.Do(req) + if err != nil { + c.logger.Error("Failed to fetch status page", "err", err) + if c.failures != nil { + c.failures.WithLabelValues("status").Inc() + } + return + } + defer func() { _ = resp.Body.Close() }() + + if c.duration != nil { + c.duration.WithLabelValues("status").Observe(time.Since(now).Seconds()) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + c.logger.Error("Failed to read status page", "err", err) + if c.failures != nil { + c.failures.WithLabelValues("status").Inc() + } + return + } + + body := string(bodyBytes) + statuses := extractStatusFromHTML(body) + + // Prepare the set of components to check → metric descriptors mapping. + components := []struct { + name statusComponent + desc *prometheus.Desc + }{ + {compGitOperations, c.GitOperationsUp}, + {compWebhooks, c.WebhooksUp}, + {compAPIRequests, c.APIRequestsUp}, + {compIssues, c.IssuesUp}, + {compPullRequests, c.PullRequestsUp}, + {compActions, c.ActionsUp}, + {compPackages, c.PackagesUp}, + {compPages, c.PagesUp}, + {compCodespaces, c.CodespacesUp}, + {compCopilot, c.CopilotUp}, + } + + // No labels for these metrics. + labels := []string{} + + // Emit metrics for each component. + for _, comp := range components { + var up float64 + if ok, exists := statuses[string(comp.name)]; exists { + if ok { + up = 1.0 + } else { + up = 0.0 + } + } else { + // If not found at all, consider as down and log for visibility. + c.logger.Warn("Component status not found on status page", "component", comp.name) + up = 0.0 + } + c.logger.Debug("Component status scraped", "component", string(comp.name), "up", up) + ch <- prometheus.MustNewConstMetric( + comp.desc, + prometheus.GaugeValue, + up, + labels..., + ) + } +} + +// extractStatusFromHTML parses the GitHub Status page HTML and returns a map of component name -> up (true if operational). +func extractStatusFromHTML(html string) map[string]bool { + result := make(map[string]bool) + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + return result + } + + doc.Find(".components-section .component-inner-container").Each(func(_ int, sel *goquery.Selection) { + // Extract and normalize fields + name := strings.TrimSpace(sel.Find(".name").Text()) + statusText := strings.ToLower(strings.TrimSpace(sel.Find(".component-status").Text())) + + if name == "" { + return + } + + // Only process names that are in our known list of components + if !isStatusComponent(name) { + return + } + + // Determine up/down strictly from the visible status text + if statusText == "operational" { + result[name] = true + return + } + + // Any other value or missing status is down + result[name] = false + }) + + return result +} diff --git a/pkg/exporter/status_test.go b/pkg/exporter/status_test.go new file mode 100644 index 00000000..dfa8cc69 --- /dev/null +++ b/pkg/exporter/status_test.go @@ -0,0 +1,137 @@ +package exporter + +import ( + "os" + "path/filepath" + "testing" +) + +func TestExtractStatusFromHTML_WithDataAttribute(t *testing.T) { + html := ` +
+
+ Git Operations + Operational +
+
+ Webhooks + Partial Outage +
+
` + + got := extractStatusFromHTML(html) + + if up, ok := got["Git Operations"]; !ok || !up { + t.Fatalf("expected Git Operations to be up, got ok=%v up=%v", ok, up) + } + if up, ok := got["Webhooks"]; !ok || up { + t.Fatalf("expected Webhooks to be down, got ok=%v up=%v", ok, up) + } +} + +func TestExtractStatusFromHTML_Fallbacks(t *testing.T) { + html := ` +
+
+ API Requests + Operational +
+
+ Issues + Operational +
+
+ Pull Requests + +
+
` + + got := extractStatusFromHTML(html) + + if up := got["API Requests"]; !up { + t.Fatalf("expected API Requests to be up via .component-status fallback") + } + if up := got["Issues"]; !up { + t.Fatalf("expected Issues to be up via component-status text") + } + if up := got["Pull Requests"]; up { + t.Fatalf("expected Pull Requests to be down when status missing") + } +} + +func TestExtractStatusFromHTML_NonOperationalVariants_TextStatuses(t *testing.T) { + html := ` +
+
+ Comp A + Degraded Performance +
+
+ Comp B + Partial Outage +
+
+ Comp C + Major Outage +
+
+ Comp D + Maintenance +
+
+ Comp E + under maintenance +
+
` + + got := extractStatusFromHTML(html) + + for _, name := range []string{"Comp A", "Comp B", "Comp C", "Comp D", "Comp E"} { + if up := got[name]; up { + t.Fatalf("expected %s to be down for non-operational data-component-status", name) + } + } +} +func TestExtractStatusFromFullPageIfPresent(t *testing.T) { + // Best-effort test: only execute if the provided page source exists. + // This verifies real-world parsing against the captured HTML. + // Location relative to this test file: repo root has githubstatus_page_source.html + root := filepath.Join("..", "..") + path := filepath.Join(root, "githubstatus_page_source.html") + data, err := os.ReadFile(path) + if err != nil { + t.Skipf("skipping full page test, source not found: %v", err) + return + } + + got := extractStatusFromHTML(string(data)) + + // Validate a subset of expected components are present and up + expectedUp := []string{ + "Git Operations", + "Webhooks", + "API Requests", + "Issues", + "Pull Requests", + "Actions", + "Packages", + "Pages", + "Codespaces", + "Copilot", + } + + for _, name := range expectedUp { + up, ok := got[name] + if !ok { + t.Fatalf("expected component %q to be present", name) + } + if !up { + t.Fatalf("expected component %q to be up (operational)", name) + } + } + + // Ensure the informational component is ignored + if _, ok := got["Visit www.githubstatus.com for more information"]; ok { + t.Fatalf("did not expect informational component in results") + } +} From 4014144aa25de6d9740ba729341496d7429cbaad Mon Sep 17 00:00:00 2001 From: Manjunath Basavaraj Bhadrannavar Date: Wed, 13 Aug 2025 00:09:40 +0200 Subject: [PATCH 2/4] add more test cases --- pkg/exporter/status.go | 1 - pkg/exporter/status_test.go | 104 ++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/pkg/exporter/status.go b/pkg/exporter/status.go index 4f1c5308..f4599a4e 100644 --- a/pkg/exporter/status.go +++ b/pkg/exporter/status.go @@ -183,7 +183,6 @@ func (c *StatusCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.CopilotUp } -// Collect intentionally left empty for now; population will be implemented later. func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { // Perform a single scrape of the status page and populate all gauges. // Follow redirects; treat "Operational" as up (1), everything else as down (0). diff --git a/pkg/exporter/status_test.go b/pkg/exporter/status_test.go index dfa8cc69..bdc814bb 100644 --- a/pkg/exporter/status_test.go +++ b/pkg/exporter/status_test.go @@ -92,6 +92,110 @@ func TestExtractStatusFromHTML_NonOperationalVariants_TextStatuses(t *testing.T) } } } + +func TestExtractStatusFromHTML_UnknownComponentsIgnored(t *testing.T) { + html := ` +
+
+ Not A Known Component + Operational +
+
+ API Requests + Operational +
+
` + + got := extractStatusFromHTML(html) + + if _, ok := got["Not A Known Component"]; ok { + t.Fatalf("expected unknown component to be ignored (not present)") + } + if up, ok := got["API Requests"]; !ok || !up { + t.Fatalf("expected known component 'API Requests' to be up and present, got ok=%v up=%v", ok, up) + } +} + +func TestExtractStatusFromHTML_TrimsNameAndStatus(t *testing.T) { + html := ` +
+
+ Git Operations + Operational +
+
` + + got := extractStatusFromHTML(html) + + if up, ok := got["Git Operations"]; !ok || !up { + t.Fatalf("expected trimmed name/status to be recognized as up, got ok=%v up=%v", ok, up) + } +} + +func TestExtractStatusFromHTML_NoComponentsSection(t *testing.T) { + html := `

No components here

` + got := extractStatusFromHTML(html) + if len(got) != 0 { + t.Fatalf("expected empty result when components section is missing, got len=%d", len(got)) + } +} + +func TestExtractStatusFromHTML_MissingNameSkipped(t *testing.T) { + html := ` +
+
+ Operational +
+
+ + Operational +
+
+ Pages + Operational +
+
` + + got := extractStatusFromHTML(html) + + if up, ok := got["Pages"]; !ok || !up { + t.Fatalf("expected 'Pages' to be present and up, got ok=%v up=%v", ok, up) + } + // Only the valid known component should be present + if len(got) != 1 { + t.Fatalf("expected only one valid component to be captured, got len=%d", len(got)) + } +} + +func TestExtractStatusFromHTML_NonOperationalVariants_KnownComponents(t *testing.T) { + html := ` +
+
+ Issues + Degraded Performance +
+
+ Pages + Partial Outage +
+
+ Copilot + Maintenance +
+
` + + got := extractStatusFromHTML(html) + + for _, name := range []string{"Issues", "Pages", "Copilot"} { + up, ok := got[name] + if !ok { + t.Fatalf("expected known component %q to be present", name) + } + if up { + t.Fatalf("expected %s to be down for non-operational status text", name) + } + } +} func TestExtractStatusFromFullPageIfPresent(t *testing.T) { // Best-effort test: only execute if the provided page source exists. // This verifies real-world parsing against the captured HTML. From 6bfe8474a764b7a0d69d416da02d4186dd7a0806 Mon Sep 17 00:00:00 2001 From: Manjunath Basavaraj Bhadrannavar Date: Wed, 13 Aug 2025 10:14:09 +0200 Subject: [PATCH 3/4] Use githubstatus json endpoint --- go.mod | 2 - go.sum | 7 -- pkg/exporter/status.go | 65 +++++----- pkg/exporter/status_test.go | 244 +++++++----------------------------- 4 files changed, 73 insertions(+), 245 deletions(-) diff --git a/go.mod b/go.mod index 65e5165e..130f2291 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.24.0 require ( github.com/GuiaBolso/darwin v0.0.0-20191218124601-fd6d2aa3d244 - github.com/PuerkitoBio/goquery v1.9.2 github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 github.com/cenkalti/backoff/v5 v5.0.3 github.com/chaisql/chai v0.16.0 @@ -48,7 +47,6 @@ require ( github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect github.com/ashanbrown/forbidigo/v2 v2.1.0 // indirect github.com/ashanbrown/makezero/v2 v2.0.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 76a7f863..b6e88aa5 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,6 @@ github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7r github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= -github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= -github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.19.0 h1:Im+SLRgT8maArxv81mULDWN8oKxkzboH07CHesxElq4= @@ -52,8 +50,6 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/ashanbrown/forbidigo/v2 v2.1.0 h1:NAxZrWqNUQiDz19FKScQ/xvwzmij6BiOw3S0+QUQ+Hs= github.com/ashanbrown/forbidigo/v2 v2.1.0/go.mod h1:0zZfdNAuZIL7rSComLGthgc/9/n2FqspBOH90xlCHdA= github.com/ashanbrown/makezero/v2 v2.0.1 h1:r8GtKetWOgoJ4sLyUx97UTwyt2dO7WkGFHizn/Lo8TY= @@ -624,7 +620,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -661,7 +656,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -670,7 +664,6 @@ golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= diff --git a/pkg/exporter/status.go b/pkg/exporter/status.go index f4599a4e..e7db3c1f 100644 --- a/pkg/exporter/status.go +++ b/pkg/exporter/status.go @@ -1,13 +1,13 @@ package exporter import ( + "encoding/json" "io" "log/slog" "net/http" "strings" "time" - "github.com/PuerkitoBio/goquery" "github.com/google/go-github/v74/github" "github.com/prometheus/client_golang/prometheus" "github.com/promhippie/github_exporter/pkg/config" @@ -184,14 +184,14 @@ func (c *StatusCollector) Describe(ch chan<- *prometheus.Desc) { } func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { - // Perform a single scrape of the status page and populate all gauges. - // Follow redirects; treat "Operational" as up (1), everything else as down (0). + // Perform a single scrape of the status JSON and populate all gauges. + // Treat "operational" as up (1), everything else as down (0). client := &http.Client{Timeout: c.config.Timeout} now := time.Now() - req, err := http.NewRequest("GET", "https://www.githubstatus.com/", nil) + req, err := http.NewRequest("GET", "https://www.githubstatus.com/api/v2/summary.json", nil) if err != nil { - c.logger.Error("Failed to build status request", "err", err) + c.logger.Error("Failed to build status summary request", "err", err) if c.failures != nil { c.failures.WithLabelValues("status").Inc() } @@ -200,7 +200,7 @@ func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { resp, err := client.Do(req) if err != nil { - c.logger.Error("Failed to fetch status page", "err", err) + c.logger.Error("Failed to fetch status summary", "err", err) if c.failures != nil { c.failures.WithLabelValues("status").Inc() } @@ -214,15 +214,14 @@ func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - c.logger.Error("Failed to read status page", "err", err) + c.logger.Error("Failed to read status summary", "err", err) if c.failures != nil { c.failures.WithLabelValues("status").Inc() } return } - body := string(bodyBytes) - statuses := extractStatusFromHTML(body) + statuses := extractStatusFromJSON(bodyBytes) // Prepare the set of components to check → metric descriptors mapping. components := []struct { @@ -255,7 +254,7 @@ func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { } } else { // If not found at all, consider as down and log for visibility. - c.logger.Warn("Component status not found on status page", "component", comp.name) + c.logger.Warn("Component status not found in status summary", "component", comp.name) up = 0.0 } c.logger.Debug("Component status scraped", "component", string(comp.name), "up", up) @@ -268,38 +267,32 @@ func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { } } -// extractStatusFromHTML parses the GitHub Status page HTML and returns a map of component name -> up (true if operational). -func extractStatusFromHTML(html string) map[string]bool { +// extractStatusFromJSON parses the GitHub Status summary JSON and returns +// a map of component name -> up (true if operational). +func extractStatusFromJSON(data []byte) map[string]bool { + type component struct { + Name string `json:"name"` + Status string `json:"status"` + } + type summary struct { + Components []component `json:"components"` + } + result := make(map[string]bool) - doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) - if err != nil { + var s summary + if err := json.Unmarshal(data, &s); err != nil { return result } - doc.Find(".components-section .component-inner-container").Each(func(_ int, sel *goquery.Selection) { - // Extract and normalize fields - name := strings.TrimSpace(sel.Find(".name").Text()) - statusText := strings.ToLower(strings.TrimSpace(sel.Find(".component-status").Text())) - - if name == "" { - return + for _, c := range s.Components { + name := strings.TrimSpace(c.Name) + if name == "" || !isStatusComponent(name) { + continue } - - // Only process names that are in our known list of components - if !isStatusComponent(name) { - return - } - - // Determine up/down strictly from the visible status text - if statusText == "operational" { - result[name] = true - return - } - - // Any other value or missing status is down - result[name] = false - }) + statusText := strings.ToLower(strings.TrimSpace(c.Status)) + result[name] = statusText == "operational" + } return result } diff --git a/pkg/exporter/status_test.go b/pkg/exporter/status_test.go index bdc814bb..8f9b0a45 100644 --- a/pkg/exporter/status_test.go +++ b/pkg/exporter/status_test.go @@ -1,25 +1,18 @@ package exporter import ( - "os" - "path/filepath" "testing" ) -func TestExtractStatusFromHTML_WithDataAttribute(t *testing.T) { - html := ` -
-
- Git Operations - Operational -
-
- Webhooks - Partial Outage -
-
` +func TestExtractStatusFromJSON_Basic(t *testing.T) { + data := []byte(`{ + "components": [ + {"name": "Git Operations", "status": "operational"}, + {"name": "Webhooks", "status": "partial_outage"} + ] + }`) - got := extractStatusFromHTML(html) + got := extractStatusFromJSON(data) if up, ok := got["Git Operations"]; !ok || !up { t.Fatalf("expected Git Operations to be up, got ok=%v up=%v", ok, up) @@ -29,84 +22,38 @@ func TestExtractStatusFromHTML_WithDataAttribute(t *testing.T) { } } -func TestExtractStatusFromHTML_Fallbacks(t *testing.T) { - html := ` -
-
- API Requests - Operational -
-
- Issues - Operational -
-
- Pull Requests - -
-
` +func TestExtractStatusFromJSON_NonOperationalVariants(t *testing.T) { + data := []byte(`{ + "components": [ + {"name": "Issues", "status": "degraded_performance"}, + {"name": "Pages", "status": "partial_outage"}, + {"name": "Copilot", "status": "major_outage"}, + {"name": "Actions", "status": "maintenance"} + ] + }`) - got := extractStatusFromHTML(html) + got := extractStatusFromJSON(data) - if up := got["API Requests"]; !up { - t.Fatalf("expected API Requests to be up via .component-status fallback") - } - if up := got["Issues"]; !up { - t.Fatalf("expected Issues to be up via component-status text") - } - if up := got["Pull Requests"]; up { - t.Fatalf("expected Pull Requests to be down when status missing") - } -} - -func TestExtractStatusFromHTML_NonOperationalVariants_TextStatuses(t *testing.T) { - html := ` -
-
- Comp A - Degraded Performance -
-
- Comp B - Partial Outage -
-
- Comp C - Major Outage -
-
- Comp D - Maintenance -
-
- Comp E - under maintenance -
-
` - - got := extractStatusFromHTML(html) - - for _, name := range []string{"Comp A", "Comp B", "Comp C", "Comp D", "Comp E"} { - if up := got[name]; up { - t.Fatalf("expected %s to be down for non-operational data-component-status", name) + for _, name := range []string{"Issues", "Pages", "Copilot", "Actions"} { + up, ok := got[name] + if !ok { + t.Fatalf("expected known component %q to be present", name) + } + if up { + t.Fatalf("expected %s to be down for non-operational status", name) } } } -func TestExtractStatusFromHTML_UnknownComponentsIgnored(t *testing.T) { - html := ` -
-
- Not A Known Component - Operational -
-
- API Requests - Operational -
-
` +func TestExtractStatusFromJSON_UnknownComponentsIgnored(t *testing.T) { + data := []byte(`{ + "components": [ + {"name": "Not A Known Component", "status": "operational"}, + {"name": "API Requests", "status": "operational"} + ] + }`) - got := extractStatusFromHTML(html) + got := extractStatusFromJSON(data) if _, ok := got["Not A Known Component"]; ok { t.Fatalf("expected unknown component to be ignored (not present)") @@ -116,126 +63,23 @@ func TestExtractStatusFromHTML_UnknownComponentsIgnored(t *testing.T) { } } -func TestExtractStatusFromHTML_TrimsNameAndStatus(t *testing.T) { - html := ` -
-
- Git Operations - Operational -
-
` - - got := extractStatusFromHTML(html) +func TestExtractStatusFromJSON_TrimsNameAndStatus(t *testing.T) { + data := []byte(`{ + "components": [ + {"name": " Git Operations ", "status": " operational "} + ] + }`) + got := extractStatusFromJSON(data) if up, ok := got["Git Operations"]; !ok || !up { t.Fatalf("expected trimmed name/status to be recognized as up, got ok=%v up=%v", ok, up) } } -func TestExtractStatusFromHTML_NoComponentsSection(t *testing.T) { - html := `

No components here

` - got := extractStatusFromHTML(html) +func TestExtractStatusFromJSON_Empty(t *testing.T) { + data := []byte(`{"components": []}`) + got := extractStatusFromJSON(data) if len(got) != 0 { - t.Fatalf("expected empty result when components section is missing, got len=%d", len(got)) - } -} - -func TestExtractStatusFromHTML_MissingNameSkipped(t *testing.T) { - html := ` -
-
- Operational -
-
- - Operational -
-
- Pages - Operational -
-
` - - got := extractStatusFromHTML(html) - - if up, ok := got["Pages"]; !ok || !up { - t.Fatalf("expected 'Pages' to be present and up, got ok=%v up=%v", ok, up) - } - // Only the valid known component should be present - if len(got) != 1 { - t.Fatalf("expected only one valid component to be captured, got len=%d", len(got)) - } -} - -func TestExtractStatusFromHTML_NonOperationalVariants_KnownComponents(t *testing.T) { - html := ` -
-
- Issues - Degraded Performance -
-
- Pages - Partial Outage -
-
- Copilot - Maintenance -
-
` - - got := extractStatusFromHTML(html) - - for _, name := range []string{"Issues", "Pages", "Copilot"} { - up, ok := got[name] - if !ok { - t.Fatalf("expected known component %q to be present", name) - } - if up { - t.Fatalf("expected %s to be down for non-operational status text", name) - } - } -} -func TestExtractStatusFromFullPageIfPresent(t *testing.T) { - // Best-effort test: only execute if the provided page source exists. - // This verifies real-world parsing against the captured HTML. - // Location relative to this test file: repo root has githubstatus_page_source.html - root := filepath.Join("..", "..") - path := filepath.Join(root, "githubstatus_page_source.html") - data, err := os.ReadFile(path) - if err != nil { - t.Skipf("skipping full page test, source not found: %v", err) - return - } - - got := extractStatusFromHTML(string(data)) - - // Validate a subset of expected components are present and up - expectedUp := []string{ - "Git Operations", - "Webhooks", - "API Requests", - "Issues", - "Pull Requests", - "Actions", - "Packages", - "Pages", - "Codespaces", - "Copilot", - } - - for _, name := range expectedUp { - up, ok := got[name] - if !ok { - t.Fatalf("expected component %q to be present", name) - } - if !up { - t.Fatalf("expected component %q to be up (operational)", name) - } - } - - // Ensure the informational component is ignored - if _, ok := got["Visit www.githubstatus.com for more information"]; ok { - t.Fatalf("did not expect informational component in results") + t.Fatalf("expected empty result when components list is empty, got len=%d", len(got)) } } From 30bc3f1bd60aca1634b1d8bd1f4863956dad0406 Mon Sep 17 00:00:00 2001 From: Manjunath Basavaraj Bhadrannavar Date: Mon, 18 Aug 2025 09:52:20 +0200 Subject: [PATCH 4/4] fix: fmt errors --- pkg/exporter/status.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/exporter/status.go b/pkg/exporter/status.go index e7db3c1f..a92580f4 100644 --- a/pkg/exporter/status.go +++ b/pkg/exporter/status.go @@ -183,6 +183,8 @@ func (c *StatusCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.CopilotUp } +// Collect gathers component status metrics from githubstatus.com and sends them +// to the provided channel. func (c *StatusCollector) Collect(ch chan<- prometheus.Metric) { // Perform a single scrape of the status JSON and populate all gauges. // Treat "operational" as up (1), everything else as down (0).