From 1668cb8bb4b7e6b60a0f80a5ec79486ab1bfa3bb Mon Sep 17 00:00:00 2001 From: firetiger-bot Date: Wed, 8 Jul 2026 00:21:07 +0000 Subject: [PATCH] chore: sync Firetiger skills from v1.1.2 --- skills/.firetiger-skills-source | 5 + skills/firetiger-create-agent/SKILL.md | 156 +++++++ skills/firetiger-instrument/SKILL.md | 102 +++++ .../references/custom-spans.md | 114 +++++ skills/firetiger-instrument/references/go.md | 73 ++++ .../firetiger-instrument/references/nodejs.md | 112 +++++ .../firetiger-instrument/references/python.md | 54 +++ .../firetiger-instrument/references/rust.md | 53 +++ skills/firetiger-investigate/SKILL.md | 153 +++++++ skills/firetiger-monitor-deploy/SKILL.md | 129 ++++++ skills/firetiger-query/SKILL.md | 134 ++++++ skills/firetiger-query/references/metrics.md | 129 ++++++ .../references/query-examples.md | 98 +++++ .../references/querying-connections.md | 125 ++++++ skills/firetiger-query/references/schema.md | 94 ++++ skills/firetiger-setup/SKILL.md | 151 +++++++ skills/firetiger-setup/references/aws.md | 55 +++ .../firetiger-setup/references/cloudflare.md | 64 +++ .../firetiger-setup/references/connections.md | 105 +++++ skills/firetiger-setup/references/datadog.md | 63 +++ skills/firetiger-setup/references/gcp.md | 54 +++ .../references/ingest-sources.md | 57 +++ skills/firetiger-setup/references/vercel.md | 51 +++ skills/firetiger/SKILL.md | 114 +++-- skills/instrument/SKILL.md | 408 ------------------ skills/investigate/SKILL.md | 168 -------- skills/plan/SKILL.md | 177 -------- skills/query/SKILL.md | 273 ------------ skills/run/SKILL.md | 209 --------- 29 files changed, 2185 insertions(+), 1295 deletions(-) create mode 100644 skills/.firetiger-skills-source create mode 100644 skills/firetiger-create-agent/SKILL.md create mode 100644 skills/firetiger-instrument/SKILL.md create mode 100644 skills/firetiger-instrument/references/custom-spans.md create mode 100644 skills/firetiger-instrument/references/go.md create mode 100644 skills/firetiger-instrument/references/nodejs.md create mode 100644 skills/firetiger-instrument/references/python.md create mode 100644 skills/firetiger-instrument/references/rust.md create mode 100644 skills/firetiger-investigate/SKILL.md create mode 100644 skills/firetiger-monitor-deploy/SKILL.md create mode 100644 skills/firetiger-query/SKILL.md create mode 100644 skills/firetiger-query/references/metrics.md create mode 100644 skills/firetiger-query/references/query-examples.md create mode 100644 skills/firetiger-query/references/querying-connections.md create mode 100644 skills/firetiger-query/references/schema.md create mode 100644 skills/firetiger-setup/SKILL.md create mode 100644 skills/firetiger-setup/references/aws.md create mode 100644 skills/firetiger-setup/references/cloudflare.md create mode 100644 skills/firetiger-setup/references/connections.md create mode 100644 skills/firetiger-setup/references/datadog.md create mode 100644 skills/firetiger-setup/references/gcp.md create mode 100644 skills/firetiger-setup/references/ingest-sources.md create mode 100644 skills/firetiger-setup/references/vercel.md delete mode 100644 skills/instrument/SKILL.md delete mode 100644 skills/investigate/SKILL.md delete mode 100644 skills/plan/SKILL.md delete mode 100644 skills/query/SKILL.md delete mode 100644 skills/run/SKILL.md diff --git a/skills/.firetiger-skills-source b/skills/.firetiger-skills-source new file mode 100644 index 0000000..7e532fc --- /dev/null +++ b/skills/.firetiger-skills-source @@ -0,0 +1,5 @@ +# Generated by scripts/sync-skills.sh from firetiger-oss/skills. Do not edit by hand. +# Update by merging the sync PR from firetiger-oss/skills, not by editing skills here. +source=firetiger-oss/skills +tag=v1.1.2 +hash=a8fcc6d3c62edff443d4bc050e29dffa6219b9cf99ee0d2a484eb1d7c7bdbd56 diff --git a/skills/firetiger-create-agent/SKILL.md b/skills/firetiger-create-agent/SKILL.md new file mode 100644 index 0000000..bbc8a97 --- /dev/null +++ b/skills/firetiger-create-agent/SKILL.md @@ -0,0 +1,156 @@ +--- +name: firetiger-create-agent +description: > + Use when creating a Firetiger monitoring agent, installing a prebuilt catalog + agent, or configuring an agent's triggers and schedules — monitoring something + automatically, scheduling recurring or one-off analysis, or wiring event-driven + automation. Always use this skill to create or automate an agent — + it carries the critical gotchas (prefer create_agent_with_goal and answer the + planner, agents must be AGENT_STATE_ON to fire, triggers are separate resources + with six config types, scheduled-agent-runs are one-shot not cron). +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Create or automate a monitoring agent in Firetiger" +metadata: + author: firetiger + version: "1.1.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +--- + +# Firetiger Create Agent + +Firetiger agents autonomously analyze telemetry, detect anomalies, investigate issues, and take actions through +connections (Slack, GitHub, Linear, databases, …). Three ways to get one — **prefer a goal or a catalog +template** over hand-building. + +## Quick Start + +``` +create_agent_with_goal with: + goal: "Monitor Next.js API routes for errors and slow responses; alert on error-rate spikes and p95 latency increases" + title: "Checkout Health Monitor" # optional +``` + +The agent-planner (a system agent, `agents/agent-planner`) configures the prompt, connections, and triggers +from the goal, then returns the planner conversation. **If the planner asked a question, you must answer it** +or the agent won't finish — reply with `send_agent_message` on the plan session. + +## Path A (recommended): create from a goal + +1. Call `create_agent_with_goal` with a clear `goal` (optional `title`, `agent_id`, `wait_for_plan` default true). +2. **Handle planner questions.** The planner asks sparingly ("act first, refine later"), via a + `request_user_input`. Answer on the user's behalf from the codebase/context — or ask the user if genuinely + unknown — by writing to the plan session: + ``` + send_agent_message with session: "" + message: "Monitor the Postgres connection at DATABASE_URL; alert threshold p99 > 500ms." + ``` +3. **Confirm.** When the planner completes, the agent flips to `AGENT_STATE_ON`. Share its name and focus. + +### Example goals +- "Monitor Next.js API routes for errors and slow responses" +- "Track database query latency and alert if p99 exceeds 500ms" +- "Watch for deployment failures and notify on Slack" +- "Track error rates across all services and open issues for spikes" + +## Path B: install a prebuilt catalog agent + +Firetiger ships tuned templates for common stacks. Browse and install them: +``` +list with resource: "catalog-agents" +create with resource: "catalog-agents/{catalog-agent}/..." # or the InstallCatalogAgent flow +``` +Templates include `postgres-performance`, `postgres-schema`, `mysql-performance`, `mysql-schema`, +`clickhouse-performance`, `temporal-performance`, `aws-cost-optimizer`, and `datadog-cost-expert`. + +**Key gotcha:** an installed catalog agent starts in **`AGENT_STATE_RECOMMENDED`**, not ON — it won't run +until a human turns it on. Send it a direct message (or set `state: AGENT_STATE_ON`) to activate. + +## Path C: configure manually + +Use the generic CRUD tools when you need explicit control. **Always run `schema` first** — fields drift. + +``` +schema with collection: "agents" +``` + +Agent fields worth knowing: `title`, `description`, `prompt` (guides every session), `state`, `connections[]` +(`{name: "connections/{id}", enabled_tools: [...]}` — what external tools the agent may use), +`mcp_connections[]` (external MCP servers), `tags[]`, `expire_time` (auto-archive after), `cache_ttl` +(`CACHE_TTL_5M` / `CACHE_TTL_1H`), `network_profile`. `skills` and `plan_session` are output-only. + +**`state` (`AgentState`) values:** `AGENT_STATE_ON`, `AGENT_STATE_OFF`, `AGENT_STATE_RECOMMENDED`, +`AGENT_STATE_REVIEW_REQUIRED`, `AGENT_STATE_UNSPECIFIED`. Only an **ON** agent fires from triggers or schedules. + +``` +create with resource: "agents" + body: { "title": "Checkout Health Monitor", + "description": "Daily health analysis of checkout", + "prompt": "You are a daily health check agent...", + "connections": [{"name": "connections/slack-prod", "enabled_tools": ["post_message"]}], + "state": "AGENT_STATE_ON" } +``` + +## Triggers — how agents fire (six kinds) + +Triggers are **separate resources** (`triggers/{id}`) referencing an agent. `InvokeTrigger` and automatic +firing require the target agent be `AGENT_STATE_ON`. Common fields: `display_name`, `agent` (required, +`agents/{id}`), `enabled`, `associated_resources`, and one `configuration`: + +| Config | Fires when | Key fields | +|--------|-----------|------------| +| `cron` | On a recurring schedule | `schedule` (5-field cron), `timezone` (IANA, default UTC) | +| `manual` | On `InvokeTrigger` or a POST to its webhook | `webhook_token`, `webhook_address` (output-only) | +| `post_deploy` | Once when a SHA (or descendant) deploys | `repository`, `environment`, `sha`, `delay` | +| `row` | When a matching row lands in a table | `table_name`, `predicate` (SQL), `cooldown` (default 15m) | +| `slack_message_posted` | On messages in Slack channels | `slack_connection`, `channels[]`, `include_thread_replies` | +| `slack_agent_mentioned` | When the agent's Slack handle is mentioned | `slack_handle`, `channels[]` | + +Cron example: +``` +create with resource: "triggers" + body: { "display_name": "Daily Checkout Health Check", + "agent": "agents/{agent-id}", "enabled": true, + "configuration": { "cron": { "schedule": "0 9 * * *", "timezone": "America/Los_Angeles" } } } +``` + +## Scheduled agent runs — one-shot, not cron + +`scheduled-agent-runs` are **single deferred invocations** ("run this prompt on agent X in 1 hour"), not +recurring schedules. Fields: `agent_id`, `prompt`, `time_delta` (delay from now), plus `use_latest_session` / +`incognito`; `scheduled_time` and `has_run` are output-only. For anything recurring, use a **cron trigger**. + +## Writing good agent prompts + +Specific, scoped, actionable — and say what the agent should NOT do. + +``` +You are a daily health check agent for the checkout service. + +Your job: +1. Query the last 24 hours of traces for checkout. +2. Calculate error rates and p99 latency; compare to the previous 24h. +3. If error rate rose >10% or p99 >20%, open an issue. +4. Summarize findings briefly. + +Focus only on checkout. Do not investigate other services. +``` + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **Ignoring the planner's question** | `create_agent_with_goal` isn't done until you answer via `send_agent_message` on the plan session. | +| 2 | **Expecting a catalog agent to run immediately** | It installs as `AGENT_STATE_RECOMMENDED` — turn it ON. | +| 3 | **Trigger created but agent is OFF** | Firing requires `AGENT_STATE_ON`. | +| 4 | **Using `scheduled-agent-runs` for recurring work** | Those are one-shot — use a `cron` trigger for recurrence. | +| 5 | **Manual `create` without `schema`** | Field names drift — call `schema` for `agents`/`triggers` first. | +| 6 | **Embedding the trigger in the agent** | Triggers are separate resources referencing `agents/{id}`. | +| 7 | **Unbounded prompt** | State what the agent should NOT do; scope it to specific services. | + +## Related + +- Run and steer an existing agent's session: `firetiger-monitor-deploy`. +- Manual investigations and the issues agents open: `firetiger-investigate`. +- Connect the integrations an agent acts through: `firetiger-setup`. diff --git a/skills/firetiger-instrument/SKILL.md b/skills/firetiger-instrument/SKILL.md new file mode 100644 index 0000000..ff5dc90 --- /dev/null +++ b/skills/firetiger-instrument/SKILL.md @@ -0,0 +1,102 @@ +--- +name: firetiger-instrument +description: > + Use when adding OpenTelemetry instrumentation to an application so it sends + telemetry to Firetiger — setting up tracing, configuring the OTLP exporter, or + connecting a Node.js, Next.js, Python, Go, or Rust app to Firetiger. Always use + this skill when instrumenting for Firetiger — it carries the critical gotchas + (init must load before any other import, Firetiger ingest uses HTTP Basic auth + not Bearer, credentials come from the get_ingest_credentials MCP tool) that make + telemetry actually show up. +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Instrument your application with OpenTelemetry for Firetiger" +metadata: + author: firetiger + version: "1.0.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +references: + - references/nodejs.md + - references/python.md + - references/go.md + - references/rust.md + - references/custom-spans.md +--- + +# Firetiger Instrument + +Instrument an application with OpenTelemetry so it exports traces (and logs/metrics) to Firetiger. For full +onboarding that also connects integrations and creates a monitoring agent, use `firetiger-setup` — this skill +is instrumentation only. + +## Workflow + +1. **Get credentials.** Call the Firetiger MCP tool **`get_ingest_credentials`** → OTLP endpoint + username + + password. If the MCP tools aren't available, tell the user to connect the Firetiger MCP server + (`https://api.cloud.firetiger.com/mcp/v1`) and sign in, then retry. +2. **Detect the framework** (table below) and open the matching reference. +3. **Check for existing instrumentation** — search for `@opentelemetry` (Node), `opentelemetry` (Python), or + `go.opentelemetry.io` (Go) imports. If present, just repoint the exporter at Firetiger instead of adding a + second SDK. +4. **Add the SDK init + env vars**, run, and verify data lands. + +| Detected file | Stack | Reference | +|---------------|-------|-----------| +| `next.config.js` / `next.config.ts` | Next.js (uses `@vercel/otel`) | [references/nodejs.md](references/nodejs.md) | +| `package.json` | Node.js / TypeScript | [references/nodejs.md](references/nodejs.md) | +| `requirements.txt` / `pyproject.toml` / `setup.py` | Python | [references/python.md](references/python.md) | +| `go.mod` | Go | [references/go.md](references/go.md) | +| `Cargo.toml` | Rust | [references/rust.md](references/rust.md) | + +For business-critical spans, log/trace correlation, and attribute best practices, see +[references/custom-spans.md](references/custom-spans.md). + +## Critical: initialization order + +**OpenTelemetry MUST initialize before any other import.** If instrumented libraries (Express, HTTP clients, +database drivers) load before the SDK starts, auto-instrumentation never attaches and you get no spans. Load +the instrumentation file first (`node --import ./instrumentation.js …`, `opentelemetry-instrument python …`, +or an init call at the very top of `main`). + +## Authentication & environment variables + +Firetiger authenticates ingest with **HTTP Basic auth** — the OTLP header is +`Authorization=Basic `, **not** a Bearer token. Configure these wherever the app +runs (`.env.local` for dev, the deploy platform for prod), using values from `get_ingest_credentials`: + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.cloud.firetiger.com" # use the value from get_ingest_credentials +OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic " +OTEL_SERVICE_NAME="your-service-name" +``` + +Add credential files to `.gitignore` (`.env.local`, `.env.*.local`). Never commit credentials. + +## Verify + +1. Generate some traffic. +2. Query with the `firetiger-query` skill: + `SELECT name, start_time FROM "opentelemetry/traces/your-service-name" WHERE start_time >= NOW() - INTERVAL '15 minutes' LIMIT 10` + (run `SHOW TABLES;` first to confirm your service's table appeared). + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **Init imported after app code** | Load the instrumentation file first — otherwise auto-instrumentation can't patch already-imported libraries. | +| 2 | **Using `Authorization=Bearer …`** | Firetiger ingest is HTTP Basic auth — `Authorization=Basic `. | +| 3 | **Hardcoding the endpoint** | Use the endpoint from `get_ingest_credentials`; it can differ per org/region. | +| 4 | **Adding a second SDK on top of existing OTEL** | Detect existing instrumentation and repoint the exporter instead. | +| 5 | **`SimpleSpanProcessor` in production** | Use `BatchSpanProcessor` with `gzip` compression for throughput. | +| 6 | **Committing `.env.local`** | Add credential files to `.gitignore`. | +| 7 | **Service name mismatch** | The `OTEL_SERVICE_NAME` you set is the `{service_name}` in the query table `"opentelemetry/traces/{service_name}"`. | +| 8 | **No graceful shutdown** | Call `sdk.shutdown()` on `SIGTERM` so buffered spans flush before exit. | + +## Related + +- **Verify the data landed and start querying it:** `firetiger-query` (`SHOW TABLES;`, then the per-service tables this instrumentation creates). +- **Add business-critical spans, log/trace correlation, and attribute hygiene:** [references/custom-spans.md](references/custom-spans.md). +- **Full onboarding beyond instrumentation** (connect integrations, register deploys, create a monitoring agent): `firetiger-setup`. +- **Not writing app code?** If you only have platform logs/traces (Vercel, AWS, GCP, Cloudflare) or a Datadog Agent, wire a drain via `firetiger-setup` instead of an SDK. +- **Once your change ships:** have Firetiger watch the deploy with `firetiger-monitor-deploy`. diff --git a/skills/firetiger-instrument/references/custom-spans.md b/skills/firetiger-instrument/references/custom-spans.md new file mode 100644 index 0000000..8d0c77f --- /dev/null +++ b/skills/firetiger-instrument/references/custom-spans.md @@ -0,0 +1,114 @@ +# Custom spans, log correlation & attribute hygiene + +Auto-instrumentation covers common libraries. Add manual spans for business-critical operations, and correlate +logs with traces so Firetiger can join them. + +## Custom spans + +### Node.js +```typescript +import { trace, SpanStatusCode } from '@opentelemetry/api'; + +const tracer = trace.getTracer('your-service-name'); + +async function processOrder(orderId: string, customerId: string) { + return await tracer.startActiveSpan('processOrder', async (span) => { + span.setAttribute('order.id', orderId); + span.setAttribute('customer.id', customerId); + try { + const result = await executeOrder(orderId); + span.addEvent('order.completed', { 'order.total': result.total }); + return result; + } catch (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + span.recordException(error); + throw error; + } finally { + span.end(); + } + }); +} +``` + +### Python +```python +from opentelemetry import trace + +tracer = trace.get_tracer("your-service-name") + +def process_order(order_id: str, customer_id: str): + with tracer.start_as_current_span("process_order") as span: + span.set_attribute("order.id", order_id) + span.set_attribute("customer.id", customer_id) + try: + result = execute_order(order_id) + span.add_event("order.completed", {"order.total": result.total}) + return result + except Exception as e: + span.set_status(trace.StatusCode.ERROR, str(e)) + span.record_exception(e) + raise +``` + +## Log correlation + +Inject `trace_id` / `span_id` into logs so `"opentelemetry/logs/{service}"` rows join to their traces. + +### Node.js (Winston) +```typescript +import winston from 'winston'; +import { trace } from '@opentelemetry/api'; + +const logger = winston.createLogger({ + format: winston.format.combine( + winston.format((info) => { + const span = trace.getActiveSpan(); + if (span) { + const ctx = span.spanContext(); + info.trace_id = ctx.traceId; + info.span_id = ctx.spanId; + } + return info; + })(), + winston.format.json() + ), + transports: [new winston.transports.Console()], +}); +``` + +### Python (structlog) +```python +import structlog +from opentelemetry import trace + +def add_trace_context(logger, method_name, event_dict): + span = trace.get_current_span() + if span.is_recording(): + ctx = span.get_span_context() + event_dict["trace_id"] = format(ctx.trace_id, "032x") + event_dict["span_id"] = format(ctx.span_id, "016x") + return event_dict + +structlog.configure(processors=[add_trace_context, structlog.processors.JSONRenderer()]) +``` + +## Attribute hygiene + +- **Semantic conventions:** `http.method`, `http.status_code`, `http.route`; `db.system`, `db.statement`, + `db.name`; `rpc.service`, `rpc.method`. Prefix custom business attributes with `app.`. +- **Cardinality:** high-cardinality *values* (user IDs) are fine; high-cardinality attribute *names* are not. +- **Never** put secrets, tokens, passwords, or PII in attributes. +- These attributes become queryable columns in Firetiger (`attributes.app.order_id`), inferred to depth 2 — + see the `firetiger-query` schema reference. + +## Performance + +- Use `BatchSpanProcessor` in production (not `SimpleSpanProcessor`). +- Enable `gzip` compression on exporters. +- Size batches to your traffic; set queue limits to bound memory. +- Add manual spans for checkout/payment and anything measured for SLOs. + +## Where to go next + +- **Query your custom spans and correlated logs** — the attributes you set become columns; see the `firetiger-query` skill (and its schema reference for the depth-2 inference rule). +- **Investigate with them** — once you have rich spans, `firetiger-investigate` can reason over them to diagnose incidents. diff --git a/skills/firetiger-instrument/references/go.md b/skills/firetiger-instrument/references/go.md new file mode 100644 index 0000000..a87c9a9 --- /dev/null +++ b/skills/firetiger-instrument/references/go.md @@ -0,0 +1,73 @@ +# Go instrumentation + +Assumes the `OTEL_EXPORTER_OTLP_*` env vars from the SKILL.md are set (Basic auth header from +`get_ingest_credentials`). + +**Note:** the gRPC exporter endpoint is `host:port` (no scheme) — `ingest.cloud.firetiger.com:443`. + +```bash +go get go.opentelemetry.io/otel \ + go.opentelemetry.io/otel/sdk \ + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc \ + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp +``` + +```go +package main + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" +) + +func initTracer(ctx context.Context) (*trace.TracerProvider, error) { + // Endpoint and auth headers come from OTEL_EXPORTER_OTLP_* env vars. + exporter, err := otlptracegrpc.New(ctx) + if err != nil { + return nil, err + } + + res, _ := resource.New(ctx, resource.WithAttributes( + semconv.ServiceName("your-service-name"), + )) + + tp := trace.NewTracerProvider( + trace.WithBatcher(exporter), + trace.WithResource(res), + ) + otel.SetTracerProvider(tp) + return tp, nil +} +``` + +Call `initTracer` at the start of `main`, and defer `tp.Shutdown(ctx)` so buffered spans flush on exit: + +```go +func main() { + ctx := context.Background() + tp, err := initTracer(ctx) + if err != nil { + log.Fatal(err) + } + defer tp.Shutdown(ctx) + // ... start your server, wrapping handlers with otelhttp.NewHandler(...) +} +``` + +Environment: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="ingest.cloud.firetiger.com:443" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic " +``` + +## Next steps + +- **Verify** — generate traffic, then query `"opentelemetry/traces/your-service-name"` with `firetiger-query` (`SHOW TABLES;` first). +- **Custom spans & log correlation** — instrument business-critical operations and join logs to traces in [custom-spans.md](custom-spans.md). +- **Full onboarding** — connect integrations and create a monitoring agent with `firetiger-setup`. diff --git a/skills/firetiger-instrument/references/nodejs.md b/skills/firetiger-instrument/references/nodejs.md new file mode 100644 index 0000000..336c2d4 --- /dev/null +++ b/skills/firetiger-instrument/references/nodejs.md @@ -0,0 +1,112 @@ +# Node.js & Next.js instrumentation + +All snippets assume the `OTEL_EXPORTER_OTLP_*` env vars from the SKILL.md are set (Basic auth header from +`get_ingest_credentials`). + +## Next.js + +Next.js has first-class OpenTelemetry support via `@vercel/otel`. + +```bash +npm install @vercel/otel @opentelemetry/sdk-logs @opentelemetry/api-logs @opentelemetry/instrumentation +``` + +Create `instrumentation.ts` in the **project root** (or `src/` if you use a src folder): + +```typescript +import { registerOTel } from '@vercel/otel' + +export function register() { + registerOTel({ serviceName: 'your-service-name' }) +} +``` + +The `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` env vars drive the exporter. Next.js loads +`instrumentation.ts` before app code automatically — no `--import` flag needed. + +## Node.js / TypeScript (non-Next.js) + +```bash +npm install @opentelemetry/api \ + @opentelemetry/sdk-node \ + @opentelemetry/auto-instrumentations-node \ + @opentelemetry/exporter-trace-otlp-proto \ + @opentelemetry/exporter-logs-otlp-proto \ + @opentelemetry/exporter-metrics-otlp-proto \ + @opentelemetry/sdk-metrics \ + @opentelemetry/sdk-logs +``` + +Create `instrumentation.ts` — it MUST load before any application code: + +```typescript +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'; +import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'; + +const traceExporter = new OTLPTraceExporter({ compression: 'gzip' }); + +const sdk = new NodeSDK({ + serviceName: process.env.OTEL_SERVICE_NAME || 'my-service', + spanProcessor: new BatchSpanProcessor(traceExporter, { + maxQueueSize: 2048, + maxExportBatchSize: 512, + scheduledDelayMillis: 5000, + }), + metricReader: new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter() }), + logRecordProcessor: new SimpleLogRecordProcessor(new OTLPLogExporter()), + instrumentations: [getNodeAutoInstrumentations()], +}); + +sdk.start(); + +// Flush buffered spans before exit +process.on('SIGTERM', () => { sdk.shutdown().then(() => process.exit(0)); }); +``` + +Run with the instrumentation loaded first: + +```bash +node --import ./instrumentation.js app.js +# TypeScript with tsx: +node --import tsx --import ./instrumentation.ts app.ts +``` + +## Simpler traces-only setup + +If you only need traces, `@opentelemetry/exporter-trace-otlp-http` plus auto-instrumentations is enough: + +```bash +npm install @opentelemetry/api @opentelemetry/sdk-node \ + @opentelemetry/auto-instrumentations-node @opentelemetry/exporter-trace-otlp-http +``` + +```typescript +import { NodeSDK } from '@opentelemetry/sdk-node' +import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node' +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' + +const sdk = new NodeSDK({ + serviceName: process.env.OTEL_SERVICE_NAME || 'my-service', + traceExporter: new OTLPTraceExporter({ + url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + headers: { + Authorization: process.env.OTEL_EXPORTER_OTLP_HEADERS?.replace('Authorization=', '') || '', + }, + }), + instrumentations: [getNodeAutoInstrumentations()], +}) + +sdk.start() +``` + +## Next steps + +- **Verify** — generate traffic, then query `"opentelemetry/traces/your-service-name"` with `firetiger-query` (`SHOW TABLES;` first). +- **Custom spans & log correlation** — instrument business-critical operations and join logs to traces in [custom-spans.md](custom-spans.md). +- **Full onboarding** — connect integrations and create a monitoring agent with `firetiger-setup`. diff --git a/skills/firetiger-instrument/references/python.md b/skills/firetiger-instrument/references/python.md new file mode 100644 index 0000000..e1d782b --- /dev/null +++ b/skills/firetiger-instrument/references/python.md @@ -0,0 +1,54 @@ +# Python instrumentation + +Assumes the `OTEL_EXPORTER_OTLP_*` env vars from the SKILL.md are set (Basic auth header from +`get_ingest_credentials`). + +## Auto-instrumentation (recommended) + +```bash +pip install opentelemetry-distro opentelemetry-exporter-otlp +opentelemetry-bootstrap -a install # installs instrumentations for detected libraries +``` + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.cloud.firetiger.com" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic " +export OTEL_SERVICE_NAME="your-service-name" + +opentelemetry-instrument python app.py +``` + +`opentelemetry-instrument` wraps your process and initializes the SDK before your code runs — no code changes +required for auto-instrumented libraries (Flask, Django, FastAPI, requests, psycopg, etc.). + +## Manual setup (more control) + +```python +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.semconv.resource import ResourceAttributes + +resource = Resource.create({ + ResourceAttributes.SERVICE_NAME: "your-service-name", + ResourceAttributes.DEPLOYMENT_ENVIRONMENT: "production", +}) + +provider = TracerProvider(resource=resource) +processor = BatchSpanProcessor(OTLPSpanExporter( + endpoint="https://ingest.cloud.firetiger.com", + headers={"Authorization": "Basic "}, +)) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) +``` + +Call this at the very top of your entrypoint, before importing instrumented libraries. + +## Next steps + +- **Verify** — generate traffic, then query `"opentelemetry/traces/your-service-name"` with `firetiger-query` (`SHOW TABLES;` first). +- **Custom spans & log correlation** — instrument business-critical operations and join logs to traces in [custom-spans.md](custom-spans.md). +- **Full onboarding** — connect integrations and create a monitoring agent with `firetiger-setup`. diff --git a/skills/firetiger-instrument/references/rust.md b/skills/firetiger-instrument/references/rust.md new file mode 100644 index 0000000..ba324a1 --- /dev/null +++ b/skills/firetiger-instrument/references/rust.md @@ -0,0 +1,53 @@ +# Rust instrumentation + +Uses `tracing` + `tracing-opentelemetry` + the OTLP exporter. Assumes the `OTEL_EXPORTER_OTLP_*` env vars +from the SKILL.md are set (Basic auth header from `get_ingest_credentials`). + +```toml +[dependencies] +opentelemetry = "0.21" +opentelemetry_sdk = { version = "0.21", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.14", features = ["tonic"] } +tracing = "0.1" +tracing-opentelemetry = "0.22" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +``` + +```rust +use opentelemetry::global; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::TracerProvider; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +fn init_tracing() -> Result<(), Box> { + let exporter = opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint("https://ingest.cloud.firetiger.com"); + + let provider = TracerProvider::builder() + .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio) + .build(); + + global::set_tracer_provider(provider.clone()); + + let telemetry = tracing_opentelemetry::layer() + .with_tracer(provider.tracer("your-service-name")); + + tracing_subscriber::registry() + .with(telemetry) + .init(); + + Ok(()) +} +``` + +Call `init_tracing()` early in `main` (inside the Tokio runtime). Instrument functions with +`#[tracing::instrument]` or emit spans with `tracing::span!` — they flow through the OpenTelemetry layer to +Firetiger. + +## Next steps + +- **Verify** — generate traffic, then query `"opentelemetry/traces/your-service-name"` with `firetiger-query` (`SHOW TABLES;` first). +- **Custom spans & log correlation** — the [custom-spans.md](custom-spans.md) patterns (Node/Python shown) map directly onto `tracing` spans and fields. +- **Full onboarding** — connect integrations and create a monitoring agent with `firetiger-setup`. diff --git a/skills/firetiger-investigate/SKILL.md b/skills/firetiger-investigate/SKILL.md new file mode 100644 index 0000000..b58e27f --- /dev/null +++ b/skills/firetiger-investigate/SKILL.md @@ -0,0 +1,153 @@ +--- +name: firetiger-investigate +description: > + Use when diagnosing an issue or incident with Firetiger — launching or reading a + Firetiger investigation, troubleshooting with telemetry, or working the issues an + investigation surfaces. Always use this skill for incident diagnosis — it carries + the critical gotchas (an investigation is an AI agent session, not a findings form; + its status is server-set; issues are managed by an expert agent you steer rather + than mutate) that keep you working with the product instead of against it. +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Start or read a Firetiger investigation to diagnose issues" +metadata: + author: firetiger + version: "1.1.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +--- + +# Firetiger Investigate + +A Firetiger **investigation is an autonomous AI agent session**, not a document you fill in. You give it a +problem statement; the `investigate` agent grounds itself in the affected services and open issues, runs +telemetry queries, and reasons toward a root cause. You read and steer that session — you don't hand-edit +findings fields. + +There are two ways to work an incident: **let the investigation agent drive** (this skill), or **query the +data yourself** (the `firetiger-query` skill). Use both — launch an investigation for breadth, drop into direct +SQL when you want to dig a specific thread. + +## Quick Start + +**Launch an investigation** — `create` on the `investigations` collection with a clear problem statement: +``` +create with resource: "investigations" + body: { + "display_name": "Checkout 5xx spike", + "description": "Error rate on checkout-service jumped ~4x starting 14:20 UTC; find the cause and blast radius." + } +``` +The resource ID *is* the agent session ID. Creating it starts the `investigate` agent. + +**Read an existing investigation** — you have a Firetiger URL or ID: +``` +resolve_url with url: "https://ui..firetigerapi.com/investigations/" # → the resource name +read_agent_messages with session: "agents/investigate/sessions/" # the agent's reasoning + findings +``` +(There is also a built-in MCP **prompt** `read-investigation` that takes a `url_or_id` and walks this for you.) + +## The investigation resource + +Deliberately thin — it wraps a session; the findings live in the transcript, not in fields: + +| Field | Notes | +|-------|-------| +| `display_name` | Short title | +| `description` | The problem statement — what to investigate and why | +| `status` | **Server-set**: `EXECUTING` (agent working) or `WAITING` (agent paused, needs input). You do not set it. | +| `created_by`, `create_time` | Output-only | + +There is **no** `findings`, `root_cause`, `resolution`, `time_range`, or `services` field — don't try to write +them. To progress or redirect the investigation, message its session. + +## Steering an investigation + +``` +# See where it's at (blocks until the agent is WAITING if you pass wait_for_completion) +read_agent_messages with session: "agents/investigate/sessions/" + +# Redirect or answer its question +send_agent_message with session: "agents/investigate/sessions/" + message: "Focus on the payment dependency — compare error rate to the deploy at 14:15 and check DB latency." +``` + +`send_agent_message` waits for the agent to finish its turn by default. A `WAITING` status means it wants your +input. + +## Diagnosing directly with SQL + +When you'd rather drive, use the `query` tool (full mechanics in `firetiger-query`). Telemetry lives in +per-service Iceberg tables; discover them with `SHOW TABLES;`. Duration = `end_time - start_time`; errors = +`status.code = 2`; filter the time column first. Bound results with a `LIMIT`. + +**Errors in the window:** +```sql +SELECT trace_id, name, status.code, status.message, start_time +FROM "opentelemetry/traces/checkout-service" +WHERE status.code = 2 + AND start_time BETWEEN TIMESTAMPTZ '{start}' AND TIMESTAMPTZ '{end}' +ORDER BY start_time DESC +LIMIT 100; +``` + +**Latency by operation:** +```sql +SELECT name, COUNT(*) AS count, + AVG(EXTRACT(EPOCH FROM (end_time - start_time))) * 1e3 AS avg_ms, + MAX(EXTRACT(EPOCH FROM (end_time - start_time))) * 1e3 AS max_ms +FROM "opentelemetry/traces/checkout-service" +WHERE start_time BETWEEN TIMESTAMPTZ '{start}' AND TIMESTAMPTZ '{end}' +GROUP BY name +ORDER BY avg_ms DESC +LIMIT 100; +``` + +**Error logs → then pull the trace by ID** (`x'...'` literal): +```sql +SELECT time, severity_text, body, trace_id +FROM "opentelemetry/logs/checkout-service" +WHERE time BETWEEN TIMESTAMPTZ '{start}' AND TIMESTAMPTZ '{end}' + AND severity_number >= 13 -- WARN and above +ORDER BY time DESC +LIMIT 100; +``` + +## Issues — the durable output of a diagnosis + +A Firetiger **Issue** (a "Known Issue") is the record an agent files when it finds a real, human-worthy +problem. Issue IDs are call signs: **`FT-{number}`**, resource name `issues/FT-{number}` (server-generated). +Full CRUD is available on the `issues` collection. + +**Workflow state** (`workflow_state`): `INVESTIGATING → ACTIONABLE → VERIFYING_FIX → CLOSED`. New issues start +`INVESTIGATING`. Closing requires a `closure.reason`, one of: `RESOLVED`, `ACCEPTED_RISK`, `FALSE_POSITIVE`, +`DUPLICATE`, `NOT_USEFUL`. There is no severity field. + +**Key gotcha:** issues are triaged and enriched by an autonomous **Issue Expert agent** (the issue's +`expert_session`). Don't hand-edit issue fields to steer triage — `send_agent_message` to the `expert_session` +and let the expert own `workflow_state`, `details`, and closure. Read the issue with `get`, note its +`source`, `services`, `pull_requests`, and `observations`. + +To find or review issues: +``` +list with resource: "issues" # filter by workflow_state, service, etc. +get with name: "issues/FT-42" +``` + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **Treating an investigation as a form** | It's an agent session — set `description` and read/steer via the session; there are no findings/root_cause fields to write. | +| 2 | **Setting `status: resolved`** | `status` is server-set (`EXECUTING`/`WAITING`) — you can't mark an investigation resolved. Close the loop by filing/closing an **issue**. | +| 3 | **Editing issue fields to drive triage** | Message the issue's `expert_session` instead; the expert owns state and enrichment. | +| 4 | **Inventing an issue ID** | `FT-{number}` call signs are server-generated on create — don't pass one. | +| 5 | **Closing an issue without a reason** | `closure.reason` is required (RESOLVED / ACCEPTED_RISK / FALSE_POSITIVE / DUPLICATE / NOT_USEFUL). | +| 6 | **Querying a generic `traces` table** | Data is per-service — `SHOW TABLES;`, then `"opentelemetry/traces/{service}"`. See `firetiger-query`. | +| 7 | **`status_code = 'ERROR'` / `duration_ns`** | Use `status.code = 2` and `end_time - start_time`. | + +## Related + +- Query mechanics and full schema: `firetiger-query`. +- Create a standing agent to investigate on a schedule or trigger: `firetiger-create-agent`. +- Tie an investigation to a specific deploy: `firetiger-monitor-deploy`. diff --git a/skills/firetiger-monitor-deploy/SKILL.md b/skills/firetiger-monitor-deploy/SKILL.md new file mode 100644 index 0000000..b67c7e9 --- /dev/null +++ b/skills/firetiger-monitor-deploy/SKILL.md @@ -0,0 +1,129 @@ +--- +name: firetiger-monitor-deploy +description: > + Use when monitoring a PR or deployment with Firetiger and reviewing what it found + — watching a deploy, setting up the @firetiger comment flow, registering + deployments from CI/CD, or reading a monitoring agent's results. Always use this + skill for deploy monitoring — it carries the critical gotchas (monitor_pr needs + the full PR URL, the @firetiger flow needs a connected GitHub App, checks run at + 10m/1h/24h/72h anchored to the real deploy, register deploys with Basic auth so + checkpoints fire on the right release). +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Monitor a PR or deployment and review what Firetiger found" +metadata: + author: firetiger + version: "1.1.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +--- + +# Firetiger Monitor Deploy + +When a change ships, Firetiger builds a monitoring plan from the PR diff, waits for the deploy, then compares +telemetry against a pre-deploy baseline at fixed checkpoints — reporting regressions on the PR and as issues. + +## Quick Start + +**From your coding session** — you have the PR URL: +``` +monitor_pr with: + pr_url: "https://github.com/org/repo/pull/123" + initial_message: "Watch checkout error rate and DB latency; this changes the payment flow" # optional +``` + +**From GitHub** — comment on the PR (requires the Firetiger GitHub App connected): +```bash +gh pr comment --body "@firetiger watch for errors and latency spikes in checkout" +``` + +After either, Firetiger reacts 👀 to acknowledge, posts a monitoring plan as a PR comment, and — after merge + +deploy — runs checks at **10 minutes, 1 hour, 24 hours, and 72 hours** past the deploy. + +**Key gotcha:** the checkpoints anchor to the *actual deploy time*, not the comment. If Firetiger can't tell +when you deployed, register deployments from CI/CD (below) so the schedule fires correctly. + +## A. The `monitor_pr` MCP tool + +Use `monitor_pr` when you have a PR URL in your session. It analyzes the PR, creates the monitoring plan, and +watches for problems after deploy. It **bypasses** the auto-monitor PR filter (so it always monitors, even for +PRs the auto-filter would skip). Seed `initial_message` from the diff (`gh pr diff`) and the user's concerns — +it steers the plan. + +## B. The `@firetiger` GitHub comment flow + +Commenting `@firetiger` on a PR enables monitoring with no coding session. The flow is asynchronous: + +1. Firetiger's GitHub App sees the mention, reacts **👀** on your comment, and queues the work. +2. A per-PR deploy-monitor agent drafts a monitoring plan and posts it as a PR comment. +3. **Steer it by commenting again** — further `@firetiger` comments are appended to the live planner session. +4. After merge + deploy, checks run at 10m / 1h / 24h / 72h; findings post on the PR and as issues. + +Prerequisites & setup: +- **GitHub connection** with an installed app (`installation_id`) — this is the linchpin for reactions, + comments, and merge-ancestry matching. Connect it via `firetiger-setup` (the `onboard_github` tool or the + `connections` collection) if `list with resource: "connections", filter: connection_type = "GITHUB"` is empty. +- Optional: a **Slack connection** + per-user preferences to DM the PR author when issues are found. +- **PR-opened auto-monitoring** is a separate path: once enabled, Firetiger can auto-monitor qualifying PRs on + open (author allowlist + a relevance judge) without any comment. + +### Example comments +``` +@firetiger +``` +``` +@firetiger monitor for error-rate increases in /api/orders and any 5xx from the new endpoint +``` + +## Registering deployments from CI/CD (deployments API) + +So checkpoints anchor to the real release, POST a deployment event from your pipeline. Get credentials with the +**`get_deploy_credentials`** MCP tool (endpoint + username/password, auto-provisioned): + +```bash +curl -X POST "$FIRETIGER_API_URL/deployments" \ + -H "Authorization: Basic $(printf '%s:%s' "$FT_DEPLOY_USER" "$FT_DEPLOY_PASS" | base64)" \ + -H "Content-Type: application/json" \ + -d '{"repository":"owner/repo","environment":"production","sha":"'"$GIT_SHA"'","deploy_time":"'"$(date -u +%FT%TZ)"'"}' +# → {"name":"deployments/"} +``` + +Fields: `repository` (`owner/repo`), `environment`, `sha` (hex, 6–64), `deploy_time` (RFC 3339, optional — +defaults to now). Inject the credentials as pipeline secrets; never commit them. GitHub Deployment webhooks +also drive this automatically when the GitHub App is connected, so manual registration is only needed when +Firetiger isn't already seeing your deploys. + +## Reading & steering the monitoring agent + +Monitoring runs as an agent with sessions and a plan. + +``` +list with resource: "monitoring-plans" # find the plan (read/delete only via MCP) +get with name: "monitoring-plans/" # plan_content, deployments[], outcome, timeline +list with resource: "agents" # the per-PR deploy-monitor agent (tag firetiger:deploy-monitor) +read_agent_messages with session: "agents//sessions/" +send_agent_message with session: "agents//sessions/" + message: "Compare the 503 rate on /api/orders to the previous version and widen the window to 2h." +``` + +Per-deployment outcomes are `NO_ISSUE` or `ISSUE_DETECTED`. A **failed** deploy (not just a regression) is +itself filed as a tracked issue tagged `deploy-failure`. `monitoring-plans` is created via `monitor_pr` / +`@firetiger` — the generic `create`/`update` tools don't apply (read and delete only). + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **Partial PR ref to `monitor_pr`** | `pr_url` must be the full URL: `https://github.com/org/repo/pull/123`. | +| 2 | **`@firetiger` with no GitHub App connected** | Connect the GitHub integration first, or the mention is ignored. | +| 3 | **Expecting the wrong checkpoints** | The schedule is 10m / 1h / 24h / 72h after deploy — not 30m/2h. | +| 4 | **Not registering deploys** | Without deploy events, checkpoints can't anchor — POST to `/deployments` from CI (or connect the GitHub App). | +| 5 | **Bearer auth on the deployments API** | It's HTTP Basic auth from `get_deploy_credentials`. | +| 6 | **`create` on `monitoring-plans`** | Plans are created by `monitor_pr`/`@firetiger`; the collection is read/delete only. | +| 7 | **Empty `initial_message`** | Seed it from the diff — a focused hint yields a much better plan. | + +## Related + +- Deep-dive an issue the monitor filed: `firetiger-investigate`. +- Create a standing (non-PR) monitoring agent: `firetiger-create-agent`. +- Connect GitHub/Slack and register deploys during onboarding: `firetiger-setup`. diff --git a/skills/firetiger-query/SKILL.md b/skills/firetiger-query/SKILL.md new file mode 100644 index 0000000..aabdf32 --- /dev/null +++ b/skills/firetiger-query/SKILL.md @@ -0,0 +1,134 @@ +--- +name: firetiger-query +description: > + Use when querying Firetiger telemetry with SQL — finding traces, searching logs, + inspecting metrics, locating errors or slow requests, computing latency + percentiles, aggregating observability data, or querying a connected source + (Postgres, MySQL, ClickHouse, Datadog, Prometheus, GCP Monitoring) through the + Firetiger MCP `query` tool. Always use this skill before writing a Firetiger query + — it carries the critical gotchas (slashed table names must be quoted, duration is + computed not stored, status codes are integers, attributes are structs not maps, + fully-qualify "connections/{name}".table to reach a connected source) that make + queries succeed. +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Query traces, logs, and metrics with SQL" +metadata: + author: firetiger + version: "1.0.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +references: + - references/schema.md + - references/query-examples.md + - references/metrics.md + - references/querying-connections.md +--- + +# Firetiger Query + +Firetiger stores telemetry in Apache Iceberg tables with dynamic schema inference. Run **DuckDB SQL** against +it with the Firetiger MCP server's **`query`** tool (parameter: `sql`). + +## Quick Start + +```sql +-- 1. Discover which tables exist +SHOW TABLES; + +-- 2. Inspect a table's columns before querying it +DESCRIBE "opentelemetry/traces/checkout-service"; + +-- 3. Query it — time-filter first; add a LIMIT to bound the result +SELECT trace_id, name, end_time - start_time AS duration, start_time +FROM "opentelemetry/traces/checkout-service" +WHERE start_time >= NOW() - INTERVAL '1 hour' +ORDER BY start_time DESC +LIMIT 100; +``` + +**Key gotcha:** the `query` tool auto-injects `USE "connections/iceberg-gateway"`, so you reference tables by +their quoted name (`"opentelemetry/traces/checkout-service"`) — you don't prefix a catalog path. There is **no +enforced row cap**, but the tool returns every row as JSON, so always add a `LIMIT` (and explicit columns, not +`SELECT *`) to avoid dumping huge nested rows. + +## Table naming + +Tables are namespaced by signal type and service name. Names with slashes **must be double-quoted**. The +`{service_name}` layout is the default shard key; a deployment can configure a different one, so always +`SHOW TABLES;` to see the real names. + +| Signal | Table | +|--------|-------| +| Traces | `"opentelemetry/traces/{service_name}"` (one row per span) | +| Logs | `"opentelemetry/logs/{service_name}"` | +| Metrics catalog | `"opentelemetry/metrics"` (one row per metric — discover names here) | +| Metric data | `"opentelemetry/metrics/{metric_name}"` (one table per metric; values inline) | + +Metrics use a per-metric table with values inline — there is **no** `series` join and no +`opentelemetry_metrics_gauges`/`_series` tables. There is no separate "spans" or "events" table — a span's +`events` and `links` are nested `LIST` columns inside the traces table. + +## Essential facts + +| Fact | Detail | +|------|--------| +| **Time columns** | `start_time` (traces), `time` (logs/metrics), all UTC. Filter on these first — tables are day-partitioned on them. Use a `Z`/`+00:00` suffix on literal timestamps. | +| **Duration** | Not stored — compute `end_time - start_time`. For a number, `EXTRACT(EPOCH FROM (end_time - start_time))` gives seconds. | +| **Status codes** | Integers: `0`=UNSET, `1`=OK, `2`=ERROR. Filter errors with `status.code = 2`. | +| **Severity** | `severity_number` integer: TRACE=1-4, DEBUG=5-8, INFO=9-12, WARN=13-16, ERROR=17-20, FATAL=21-24. | +| **Attributes** | STRUCTs, dot-accessed (`attributes.http.route`, `resource.attributes.service.name`) — **not** maps, so no `attributes['key']`. Keys are snake_case-normalized. Inferred to depth 2; level 3+ is a JSON-typed column — extract with DuckDB JSON funcs (e.g. `json_extract_string(attributes.request.context, '$.tenant_id')`). | +| **IDs** | `trace_id` is 16 bytes, `span_id`/`parent_span_id` 8 bytes (BLOB in DuckDB). Match with `x'...'` hex literals: `WHERE trace_id = x'0123...'`. | +| **Sampling** | Traces may carry a nullable `sample_rate` (double). When present, multiply counts by `1 / sample_rate` to estimate true volume. | + +## Querying connected sources + +The same `query` tool can also reach sources you've **connected** to Firetiger — and join across them in one +query. Firetiger's own telemetry is the default source; other sources are addressed two ways: + +- **SQL sources** (Postgres, MySQL, ClickHouse) — **fully-qualify the table** with the connection name. + `"connections/{name}"` is a quoted identifier (it contains a `/`), followed by schema/database and table: + ```sql + SELECT id, email FROM "connections/prod-postgres".public.users WHERE created_at >= NOW() - INTERVAL '1 day' LIMIT 100; + ``` + Prefer this to `USE` — it's self-contained and lets one query join across connections. (`USE + "connections/{name}";` is an optional shorthand that sets the default for *unqualified* names.) +- **Observability backends** (Datadog, Prometheus, GCP Monitoring) — call a **function** with the connection + name as a string argument and the vendor's own query language: + ```sql + SELECT * FROM datadog_search_logs('connections/prod-datadog', 'status:error service:api', NOW() - INTERVAL '1 hour', NOW(), max_rows => 500); + ``` + +Run `list with resource: "connections"` to see configured sources, and `SELECT * FROM confit_functions();` to +see which functions your connections unlock. Full patterns, PromQL/GCP examples, cross-source joins, and gotchas +are in [references/querying-connections.md](references/querying-connections.md). (Trino and Elasticsearch aren't +reachable through the `query` tool.) + +## What do you need? + +| Task | Reference | +|------|-----------| +| **Full column reference** for traces, logs, and deeply-nested attributes | [references/schema.md](references/schema.md) | +| **Ready-to-run queries** — recent/slow/error spans, latency percentiles, error logs, cross-service traces | [references/query-examples.md](references/query-examples.md) | +| **Metrics** — the metadata catalog, per-metric data tables, gauge/sum/histogram value columns | [references/metrics.md](references/metrics.md) | +| **Querying connected sources** — Postgres/MySQL/ClickHouse via fully-qualified `"connections/{name}".table`, Datadog/Prometheus/GCP via functions, cross-source joins | [references/querying-connections.md](references/querying-connections.md) | + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **`SELECT *` / no `LIMIT`** | Telemetry rows are wide (nested structs/lists) and every row is returned as JSON. Select explicit columns and add a `LIMIT`. | +| 2 | **Unquoted table name with slashes** | `"opentelemetry/traces/my-service"` — bare `opentelemetry/traces/...` is a parse error. | +| 3 | **Selecting a `duration` column** | There is none. Use `end_time - start_time`. | +| 4 | **Filtering `status.code = 'ERROR'`** | Status codes are integers — use `status.code = 2`. | +| 5 | **No time filter** | Always constrain `start_time`/`time` first — tables are day-partitioned, so this is what prunes the scan. | +| 6 | **Averaging an INTERVAL** | Wrap in `EXTRACT(EPOCH FROM (end_time - start_time))` to aggregate durations as numbers. | +| 7 | **Guessing service/table names** | Run `SHOW TABLES;` and `DESCRIBE "table"` first — schemas are inferred and vary per service. | +| 8 | **Map syntax on attributes** | Attributes are structs — `attributes.http.route`, not `attributes['http.route']`. | +| 9 | **Reading a level-3+ attribute directly** | Beyond nesting depth 2 it's a JSON-typed column — `json_extract_string(attributes.request.context, '$.key')`. | +| 10 | **Wrong connection-name form when federating** | In `FROM`, `"connections/pg".schema.table` — a **quoted identifier**; in a vendor function, `datadog_search_logs('connections/pg', …)` — a **string literal**. Don't swap them. | + +## Related + +- No data to query? Verify instrumentation with `firetiger-instrument`. +- Diagnosing an incident? `firetiger-investigate` wraps these queries in a tracked workflow. diff --git a/skills/firetiger-query/references/metrics.md b/skills/firetiger-query/references/metrics.md new file mode 100644 index 0000000..9096bbe --- /dev/null +++ b/skills/firetiger-query/references/metrics.md @@ -0,0 +1,129 @@ +# Querying metrics + +Metrics use the same per-service-style layout as traces and logs: **one table per metric name**, with the data +points' values inline. There is also a single **metadata catalog** table listing every metric. All table names +contain slashes, so they must be **double-quoted**. + +| Table | Contents | +|-------|----------| +| `"opentelemetry/metrics"` | Metadata catalog — one row per unique metric definition. Use it to discover metrics. | +| `"opentelemetry/metrics/{metric_name}"` | Data points for a single metric. The metric name is used verbatim as the path segment, dots included — e.g. `"opentelemetry/metrics/http.server.request.duration"`. | + +> There is **no** `opentelemetry_metrics_series` table and **no** `opentelemetry_metrics_gauges` / +> `_counters_cumulative` / `_histograms_cumulative` tables, and **no `series` join**. Values live directly in +> each metric's table. + +## Discover available metrics (the catalog) + +The `"opentelemetry/metrics"` table is your catalog. Query it first to find metric names, then query the +matching per-metric data table. + +| Column | Type | Description | +|--------|------|-------------| +| `name` | STRING | Metric name (e.g. `http.server.request.duration`) — this is the `{metric_name}` in the data table path | +| `type` | STRING | `gauge`, `sum`, `histogram`, or `exponential_histogram` | +| `unit` | STRING | Unit of measurement (`ms`, `bytes`, `1` for dimensionless) | +| `description` | STRING | Human-readable description | +| `aggregation_temporality` | STRING | `delta` (rate metrics) or `cumulative` (counters) | +| `time` | TIMESTAMPTZ | When the metric metadata was first observed (day-partition key) | + +```sql +-- List all metrics +SELECT DISTINCT name, type, unit, description +FROM "opentelemetry/metrics" +ORDER BY name +LIMIT 100; +``` + +```sql +-- Find latency histograms +SELECT name, description, unit +FROM "opentelemetry/metrics" +WHERE type IN ('histogram', 'exponential_histogram') + AND unit = 'ms' +LIMIT 100; +``` + +## Per-metric data tables + +Every metric's data table shares a common set of columns, plus type-specific value columns. `time` is the +day-partition key — **filter on it first**. + +### Common columns + +| Column | Type | Description | +|--------|------|-------------| +| `time` | TIMESTAMPTZ | Data point timestamp (partition key) | +| `start_time` | TIMESTAMPTZ | Start of the aggregation window | +| `name` | STRING | Metric name | +| `type` | STRING | `gauge`, `sum`, `histogram`, `exponential_histogram` | +| `unit` | STRING | Unit | +| `description` | STRING | Description | +| `aggregation_temporality` | STRING | `delta` or `cumulative` | +| `attributes` | STRUCT | Data-point attributes (dot-accessed, depth-2 inference — see [schema.md](schema.md)) | +| `resource.attributes.*` | STRUCT | Resource attributes (e.g. `resource.attributes.service.name`) | +| `scope.name` / `scope.version` | STRING | Instrumentation scope | + +### Value columns by metric type + +| `type` | Value columns | +|--------|---------------| +| `gauge`, `sum` | `value` DOUBLE | +| `histogram` | `count` LONG, `sum` DOUBLE, `min` DOUBLE, `max` DOUBLE, `bucket_counts` LIST, `explicit_bounds` LIST | +| `exponential_histogram` | `count`, `sum`, `min`, `max`, `scale` INT, `zero_count` LONG, `zero_threshold` DOUBLE, `positive` STRUCT{`offset` INT, `bucket_counts` LIST}, `negative` STRUCT{…} | + +Always `DESCRIBE "opentelemetry/metrics/{metric_name}"` to confirm the exact columns — attributes are inferred +and vary per metric. + +## Examples + +### Gauge / sum — read values directly + +```sql +SELECT time, value, + resource.attributes.service.name AS service, + attributes.http.route AS route +FROM "opentelemetry/metrics/http.server.active_requests" +WHERE time >= NOW() - INTERVAL '1 hour' +ORDER BY time DESC +LIMIT 100; +``` + +### Sum — aggregate a counter over a window + +```sql +SELECT resource.attributes.service.name AS service, + SUM(value) AS total +FROM "opentelemetry/metrics/http.server.request.count" +WHERE time >= NOW() - INTERVAL '1 hour' +GROUP BY service +ORDER BY total DESC +LIMIT 100; +``` + +### Histogram — average latency from count and sum + +```sql +SELECT attributes.http.route AS route, + SUM(count) AS requests, + SUM(sum) / NULLIF(SUM(count), 0) AS avg_ms +FROM "opentelemetry/metrics/http.server.request.duration" +WHERE time >= NOW() - INTERVAL '1 hour' + AND unit = 'ms' +GROUP BY route +ORDER BY requests DESC +LIMIT 100; +``` + +## Notes + +- Metric data and catalog tables use `time` (not `start_time`) as the day-partition key — filter on it first. +- Attributes follow the same depth-2 inference rule as traces/logs — see [schema.md](schema.md). +- Add a `LIMIT` to bound results (no fixed cap is enforced, but every row is returned as JSON). +- A legacy `"opentelemetry/metrics/metadata"` table may exist in older deployments — prefer `"opentelemetry/metrics"`. + +## Where to go next + +- **Trace & log columns** and the attribute inference rules: [schema.md](schema.md). +- **Copy-paste trace/log queries** to pair with these metrics: [query-examples.md](query-examples.md). +- **Prometheus / GCP Monitoring metrics** from a connected source (PromQL functions): [querying-connections.md](querying-connections.md). diff --git a/skills/firetiger-query/references/query-examples.md b/skills/firetiger-query/references/query-examples.md new file mode 100644 index 0000000..e8c4195 --- /dev/null +++ b/skills/firetiger-query/references/query-examples.md @@ -0,0 +1,98 @@ +# Firetiger query examples + +Copy-paste patterns for traces and logs. Every query time-filters first (day-partition pruning) and adds a +`LIMIT` to bound the result. Replace service names with ones from `SHOW TABLES;`. + +## Recent spans + +```sql +SELECT trace_id, name, resource.attributes.service.name AS service, + end_time - start_time AS duration, start_time +FROM "opentelemetry/traces/checkout-service" +WHERE start_time >= NOW() - INTERVAL '1 hour' +ORDER BY start_time DESC +LIMIT 100; +``` + +## Slow spans + +```sql +SELECT trace_id, name, resource.attributes.service.name AS service, + end_time - start_time AS duration +FROM "opentelemetry/traces/api-gateway" +WHERE end_time - start_time > INTERVAL '1 second' + AND start_time >= NOW() - INTERVAL '1 hour' +ORDER BY (end_time - start_time) DESC +LIMIT 50; +``` + +## Error spans + +```sql +SELECT trace_id, name, status.code, status.message, start_time +FROM "opentelemetry/traces/payment-service" +WHERE status.code = 2 -- ERROR + AND start_time >= NOW() - INTERVAL '1 hour' +ORDER BY start_time DESC +LIMIT 100; +``` + +## Aggregate by span name + +```sql +SELECT name, COUNT(*) AS span_count, + AVG(EXTRACT(EPOCH FROM (end_time - start_time))) AS avg_duration_sec, + MAX(EXTRACT(EPOCH FROM (end_time - start_time))) AS max_duration_sec +FROM "opentelemetry/traces/checkout-service" +WHERE start_time >= NOW() - INTERVAL '1 hour' +GROUP BY name +ORDER BY span_count DESC +LIMIT 50; +``` + +## Latency percentiles by HTTP route + +```sql +SELECT attributes.http.route AS route, COUNT(*) AS requests, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p50_sec, + PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p95_sec, + PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p99_sec +FROM "opentelemetry/traces/api-gateway" +WHERE start_time >= NOW() - INTERVAL '1 hour' + AND attributes.http.route IS NOT NULL +GROUP BY attributes.http.route +ORDER BY requests DESC +LIMIT 50; +``` + +## Error logs + +```sql +SELECT time, severity_text, body, trace_id, resource.attributes.service.name AS service +FROM "opentelemetry/logs/checkout-service" +WHERE time >= NOW() - INTERVAL '1 hour' + AND severity_number >= 17 -- ERROR and above +ORDER BY time DESC +LIMIT 100; +``` + +## Trace a request across a service + +```sql +SELECT name, resource.attributes.service.name AS service, start_time, + end_time - start_time AS duration, status.code +FROM "opentelemetry/traces/api-gateway" +WHERE trace_id = x'0123456789abcdef0123456789abcdef' +ORDER BY start_time +LIMIT 100; +``` + +To follow a trace across *multiple* services, run this against each service table for the same `trace_id` +(discover the tables with `SHOW TABLES;`). + +## Where to go next + +- **Column reference** for any field used above (traces, logs, nested attributes): [schema.md](schema.md). +- **Metric queries** (gauge/sum/histogram value columns): [metrics.md](metrics.md). +- **Join these tables to a connected database or Datadog/Prometheus**: [querying-connections.md](querying-connections.md). +- **Chasing an incident?** `firetiger-investigate` wraps these same queries in a tracked, agent-driven workflow. diff --git a/skills/firetiger-query/references/querying-connections.md b/skills/firetiger-query/references/querying-connections.md new file mode 100644 index 0000000..da5ad4c --- /dev/null +++ b/skills/firetiger-query/references/querying-connections.md @@ -0,0 +1,125 @@ +# Querying connected sources (federation) + +The `query` tool isn't limited to Firetiger's own telemetry — it runs **Confit SQL**, which can also reach the +data sources you've connected (databases, Datadog, Prometheus, GCP Monitoring) and even **join across them in a +single query**. Confit SQL is DuckDB dialect plus connection-aware `FROM` resolution plus vendor functions; +only `SELECT` / `SHOW` / `DESCRIBE` are allowed. + +The connections available to you are whatever is configured on the org — `list with resource: "connections"` +to see them. Each is addressed by its **full resource name**, `connections/{name}`. + +## Two addressing patterns + +### 1. SQL sources → fully-qualify the table + +Firetiger's own telemetry lake, **`connections/iceberg-gateway`**, is the **default** connection: the `query` +tool auto-selects it (it prepends `USE "connections/iceberg-gateway";` when your SQL has no leading `USE`), so +plain telemetry queries just name the table — `FROM "opentelemetry/traces/checkout-service"` is implicitly +`FROM "connections/iceberg-gateway"."opentelemetry/traces/checkout-service"`. To query a *different* connected +SQL source, **fully-qualify the table with the connection name** — `"connections/{name}"` is the leading +identifier (quoted, because it contains a `/`), followed by the schema/database and table: + +```sql +-- Postgres / MySQL: "connections/{name}".. +SELECT id, email, created_at +FROM "connections/prod-postgres".public.users +WHERE created_at >= NOW() - INTERVAL '1 day' +ORDER BY created_at DESC +LIMIT 100; + +-- ClickHouse: "connections/{name}"..
+SELECT event_type, count(*) FROM "connections/prod-clickhouse".default.events +WHERE timestamp >= NOW() - INTERVAL '1 hour' +GROUP BY event_type LIMIT 100; +``` + +Prefer this fully-qualified form: each query is self-contained (no hidden default state), and it lets you +**join across connections in one query**. The engine federates the source in server-side, so your SQL never +contains its credentials. Connected **Postgres, MySQL, and ClickHouse** are reachable this way; connected +**Trino** and **Elasticsearch** are *not* reachable through the `query` tool — query those from the Firetiger +dashboard or via a generic HTTP connection. + +> **`USE` is optional.** `USE "connections/{name}";` at the start of a statement just sets the default +> connection for *unqualified* table names — handy when a whole query targets one source. It's not required, +> and fully-qualifying is clearer for one-offs and mandatory when a single query spans two connections. + +### 2. Observability backends → vendor functions + +Datadog, Prometheus, and GCP Monitoring are **not** SQL databases — you reach them through table functions, +passing the **connection name as the first string argument** (a literal, not a `USE`) and the vendor's own +query language as a string. + +**Datadog** (native Datadog search syntax / MQL): +```sql +SELECT * FROM datadog_search_logs( + 'connections/prod-datadog', + 'status:error service:api', -- Datadog log query syntax + NOW() - INTERVAL '1 hour', NOW(), + max_rows => 500); +``` +Other Datadog functions: `datadog_search_spans`, `datadog_query_metrics`, `datadog_list_metrics`, +`datadog_list_monitors` / `datadog_get_monitor`, `datadog_list_dashboards` / `datadog_get_dashboard`, and +`datadog_cost_analysis` (reads Datadog **billing** metadata — custom-metric series counts, indexed log volume, +tag cardinality — to find what's driving Datadog spend). + +**Prometheus / PromQL:** +```sql +SELECT timestamp, value FROM promql_query_range( + 'connections/prod-prom', + 'sum by (status) (rate(http_requests_total{env="prod"}[5m]))', -- native PromQL + NOW() - INTERVAL '1 hour', NOW(), '1m') +LIMIT 200; +``` +Also `promql_query` (instant), and discovery: `promql_metadata`, `promql_labels`, `promql_label_values`, +`promql_series`. Use `rate()`/`increase()` on counters, aggregate *outside* the rate, and keep `[range]` ≥ 2× +the scrape interval. + +**GCP Cloud Monitoring** uses the same `promql_*` functions pointed at a GCP-type connection. GCP metric names +carry a kind prefix — e.g. `compute_googleapis_com:instance_cpu_utilization`, +`kubernetes_io:container_memory_used_bytes` — resolve the exact name (with its `:` and underscores) first, and +filter by `project_id` + `location`. + +## Cross-source joins + +A single query can join a vendor function's output to Firetiger telemetry, or two connections to each other — +e.g. compare CPU across two Prometheus environments: + +```sql +SELECT p.timestamp, p.value AS prod_cpu, s.value AS staging_cpu +FROM promql_query_range('connections/prod-prom', 'rate(process_cpu_seconds_total[5m])', + NOW() - INTERVAL '1 hour', NOW(), '1m') p +JOIN promql_query_range('connections/staging-prom', 'rate(process_cpu_seconds_total[5m])', + NOW() - INTERVAL '1 hour', NOW(), '1m') s + ON p.timestamp = s.timestamp +ORDER BY p.timestamp +LIMIT 100; +``` + +## Discovering what's queryable + +The available functions depend on which connection types are in scope. List them from SQL: + +```sql +SELECT * FROM confit_functions(); -- functions available for your active connections +SELECT * FROM confit_signatures(); -- their argument signatures +SELECT * FROM confit_examples(); -- worked examples +``` + +## Gotchas + +| Gotcha | Detail | +|--------|--------| +| **Connection name form differs by pattern** | In `FROM`, `"connections/pg".schema.table` — the connection is a **quoted identifier**. In a vendor function, `datadog_search_logs('connections/pg', …)` — it's a **string literal argument**. Don't swap them. | +| **ClickHouse needs database + table** | `"connections/ch"..
` — both parts after the connection are required. | +| **Time arguments** | Pass `TIMESTAMPTZ` literals or `NOW() - INTERVAL 'N unit'` — never bare unix timestamps. | +| **`max_rows` only on search/list functions** | Adding `max_rows` to a point-query function (e.g. `promql_query`) is a signature error. | +| **No `SELECT *` on `promql_query_range`** | Its `labels` MAP inflates output — select `timestamp, value`. | +| **Vendor APIs bill on window × resolution** | Start with a 15-minute window and widen only after a hypothesis; Datadog/GCP charge per query. | +| **Telemetry lags ~5 min** | For fresher-than-5-minute data, query the source connection directly instead of Firetiger's lake. | +| **Trino / Elasticsearch** | Not reachable via the `query` tool — no federation path. Use the dashboard or a generic HTTP connection. | +| **`LIMIT` still applies** | Add a `LIMIT` to every SELECT, federated or not. | + +## Where to go next + +- **No connection to query yet?** Create one — per-type config (DSN, API key, region) and the connect model are in the `firetiger-setup` skill's "Connections" reference. +- **Querying Firetiger's own telemetry** (traces/logs/metrics tables): [schema.md](schema.md), [query-examples.md](query-examples.md), [metrics.md](metrics.md). diff --git a/skills/firetiger-query/references/schema.md b/skills/firetiger-query/references/schema.md new file mode 100644 index 0000000..77e484f --- /dev/null +++ b/skills/firetiger-query/references/schema.md @@ -0,0 +1,94 @@ +# Firetiger telemetry schema + +Firetiger infers schema dynamically from incoming OpenTelemetry data, so exact columns vary per service. Run +`DESCRIBE "table_name"` to see the live shape. The columns below are always present. + +One row per **span** (there is no separate spans table). `trace_id`/`span_id` surface as BLOB in DuckDB. + +## Traces — `"opentelemetry/traces/{service_name}"` + +| Column | Type | Description | +|--------|------|-------------| +| `trace_id` | fixed[16] (BLOB) | W3C Trace Context trace ID (16 bytes). Match with `x'...'` literals. | +| `span_id` | fixed[8] (BLOB) | Span ID within the trace (8 bytes) | +| `parent_span_id` | fixed[8] (BLOB) | Parent span ID (8 bytes; empty for root spans) | +| `name` | STRING | Logical operation name (span name) | +| `kind` | INT | 0=UNSPECIFIED, 1=INTERNAL, 2=SERVER, 3=CLIENT, 4=PRODUCER, 5=CONSUMER | +| `start_time` | TIMESTAMPTZ | Span start, UTC (day-partition key — filter on this first) | +| `end_time` | TIMESTAMPTZ | Span end, UTC | +| `status.code` | INT | 0=UNSET, 1=OK, 2=ERROR | +| `status.message` | STRING | Status message | +| `attributes` | STRUCT | Span attributes (dynamically inferred) | +| `scope.name` / `scope.version` | STRING | Instrumentation scope | +| `resource.attributes.service.name` | STRING | Service name | +| `resource.attributes.service.namespace` | STRING | Service namespace | +| `resource.attributes.service.version` | STRING | Service version | +| `events` | LIST | Span events (nested; event attributes are dropped at ingest) | +| `links` | LIST | Span links (nested) | +| `sample_rate` | DOUBLE (nullable) | Head sampling rate when present — multiply counts by `1 / sample_rate` to estimate true volume | +| `trace_state`, `flags`, `dropped_*_count` | — | Less-common OTLP fields; `DESCRIBE` to see them | + +**Duration is calculated, not stored:** `end_time - start_time AS duration`. + +## Logs — `"opentelemetry/logs/{service_name}"` + +| Column | Type | Description | +|--------|------|-------------| +| `time` | TIMESTAMPTZ | Log timestamp (partition key — filter on this first) | +| `observed_time` | TIMESTAMPTZ | When the log was observed | +| `severity_number` | INT | TRACE=1-4, DEBUG=5-8, INFO=9-12, WARN=13-16, ERROR=17-20, FATAL=21-24 | +| `severity_text` | STRING | TRACE, DEBUG, INFO, WARN, ERROR, FATAL | +| `body` | inferred (STRING or STRUCT) | Log message body — string, or a struct for structured logs | +| `trace_id` | fixed[16] (BLOB) | Trace ID for correlation (optional) | +| `span_id` | fixed[8] (BLOB) | Span ID for correlation (optional) | +| `attributes` | STRUCT | Log attributes (dynamically inferred) | +| `scope.name` / `scope.version` | STRING | Instrumentation scope | +| `event_name`, `flags`, `dropped_attributes_count` | — | Less-common OTLP fields | +| `resource.attributes.service.name` | STRING | Service name | + +Structured-log signals (records carrying `__type__`/`__name__`) also land in per-name sub-tables: +`"opentelemetry/logs/{service}/events/{name}"`, `.../counters/{name}`, `.../gauges/{name}`, +`.../histograms/{name}`, `.../metrics/{name}`. + +## Accessing attributes + +Attributes and resource attributes are nested structs — access with dot notation: + +```sql +-- Span attributes (semantic conventions) +attributes.http.method +attributes.http.route +attributes.http.status_code +attributes.db.system +attributes.db.statement + +-- Resource attributes +resource.attributes.service.name +resource.attributes.service.namespace +resource.attributes.telemetry.sdk.language +``` + +Keys are snake_case-normalized at ingest, and dotted keys become nested struct paths. Attributes are structs, +not maps — use dot access, never `attributes['key']`. + +### Type inference depth limit + +Firetiger infers attribute types to a **depth of 2 levels**: + +- **Levels 1–2** — expanded into queryable struct columns (`attributes.http.route`). +- **Level 3+** — collapsed into a single JSON-typed column at that path. Extract with DuckDB JSON functions: + +```sql +SELECT + json_extract_string(attributes.request.context, '$.tenant_id') AS tenant_id +FROM "opentelemetry/traces/api" +WHERE start_time >= NOW() - INTERVAL '1 hour' +LIMIT 100; +``` + +## Where to go next + +- **Ready-to-run queries** using these columns — recent/slow/error spans, latency percentiles, error logs: [query-examples.md](query-examples.md). +- **Metrics tables** (catalog + per-metric value columns), which follow the same depth-2 attribute rule: [metrics.md](metrics.md). +- **Querying connected sources** (Postgres/MySQL/ClickHouse, Datadog/Prometheus/GCP) and cross-source joins: [querying-connections.md](querying-connections.md). +- **No rows coming back?** Confirm the service is instrumented and exporting with `firetiger-instrument`. diff --git a/skills/firetiger-setup/SKILL.md b/skills/firetiger-setup/SKILL.md new file mode 100644 index 0000000..cf52d34 --- /dev/null +++ b/skills/firetiger-setup/SKILL.md @@ -0,0 +1,151 @@ +--- +name: firetiger-setup +description: > + Use when onboarding a project to Firetiger end-to-end — authenticate, subscribe, + detect the stack, wire telemetry from any source (OTLP SDK, platform log/trace + drains, Datadog Agent, Prometheus, Vector, or a generic HTTP sink), connect + integrations (GitHub, Slack, databases, cloud, observability backends), register + deployments, let discovery map services, and create a monitoring agent. Always use + this skill for first-time setup — it carries the gotchas (credentials auto-provision, + Basic-auth ingest, connect GitHub + telemetry to unlock discovery) that finish + onboarding in one pass. +license: Apache-2.0 +user_invocable: true +user_invocable_description: "Set up Firetiger for this project — detect the stack, wire telemetry, connect integrations, create a monitoring agent" +metadata: + author: firetiger + version: "1.1.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills +references: + - references/ingest-sources.md + - references/connections.md + - references/vercel.md + - references/aws.md + - references/gcp.md + - references/cloudflare.md + - references/datadog.md +--- + +# Firetiger Setup + +Onboard this project to Firetiger with **minimal user interaction**: subscribe, wire up telemetry, connect +integrations, register deployments, let discovery map the system, and create a monitoring agent. Make changes +automatically; pause only when genuinely uncertain. Firetiger also ships two built-in MCP **prompts** — +`onboard-firetiger` (this whole flow) and `integrate-firetiger` (SDK instrumentation) — that mirror these steps. + +## Step 1 — Authenticate & provision + +Call **`get_ingest_credentials`**. It auto-provisions the org backend (credentials, storage) if needed and +returns the OTLP endpoint + username/password. If the MCP tools aren't available, tell the user to connect the +Firetiger MCP server (`https://api.cloud.firetiger.com/mcp/v1`) and sign in (Claude Code: `/mcp` → Firetiger → +sign in), then retry. Build the Basic auth header as `base64(username:password)` for Step 3. + +## Step 2 — Subscribe + +Check **`get_subscription_status`**: +- `active` / `trialing` → continue. +- `none` / `canceled` / `past_due` → call **`get_checkout_url`** (plan `bootstrap` is the free tier; `growth` + is paid), open it for the user, and wait for them to finish before continuing. + +(These tools appear only on deployments with billing enabled; skip this step if they're absent.) + +## Step 3 — Detect the stack + +Explore the codebase to find telemetry sources and services Firetiger can connect to: + +- **Deployment platforms** (log/trace drains): Vercel, Cloudflare, AWS, GCP. +- **Telemetry sources**: OpenTelemetry (`@opentelemetry/*`, `opentelemetry-*`, `go.opentelemetry.io`), Datadog + (`dd-trace`, `datadog`), Prometheus (`prometheus.yml`), Vector (`vector.toml`). +- **Databases**: PostgreSQL / MySQL (Prisma, SQLAlchemy, connection strings), ClickHouse, Trino, Elasticsearch. +- **Event & incident sources**: GitHub (`.git/config`), SendGrid, Convex, Kafka, incident.io, PagerDuty. +- **Language/framework** (for OTEL auto-instrumentation): `package.json` → Node.js; `requirements.txt` / + `pyproject.toml` → Python; `go.mod` → Go. + +## Step 4 — Wire up telemetry ingestion + +Pick the path that matches what you detected. The ingest endpoint is `https://ingest.cloud.firetiger.com`, and +all ingest is **HTTP Basic auth** with the Step 1 credentials (`$USERNAME` / `$PASSWORD`, or +`$AUTH_HEADER = base64(user:pass)`). The full source menu — with endpoint paths — is in +**[references/ingest-sources.md](references/ingest-sources.md)**. + +| Detected | Recipe | +|----------|--------| +| Vercel | [references/vercel.md](references/vercel.md) — log + trace drains via the Vercel API | +| AWS (`which aws`) | [references/aws.md](references/aws.md) — CloudWatch/ALB/CloudFront/ECS via CloudFormation + Firehose | +| GCP (`which gcloud`) | [references/gcp.md](references/gcp.md) — Cloud Logging sink → Pub/Sub → forwarder, or HTTP push | +| Cloudflare (`which wrangler`) | [references/cloudflare.md](references/cloudflare.md) — Workers observability + Logpush | +| Datadog Agent already deployed | [references/datadog.md](references/datadog.md) — repoint `DD_URL` at Firetiger (logs+metrics+traces) | +| Prometheus / Vector / generic HTTP | [references/ingest-sources.md](references/ingest-sources.md) — `remote_write`, Vector sink, `/datapoints/` | +| None of the above | Hand off to **`firetiger-instrument`** — OTLP SDK for Node/Next.js/Python/Go/Rust | + +## Step 5 — Connect integrations (incl. pull-based sources) + +Integrations are **Connections**. Many customers rely on **pull-based** connections most of all — Firetiger +*queries into* their existing Datadog, Prometheus, GCP metrics, and databases rather than ingesting from them. +The full catalog, per-type config, and connect mechanism is in +**[references/connections.md](references/connections.md)**. + +- **A client agent creates most connections directly** — `schema` (collection `connections`), then `create` + with the `connection_type` and its config (DSN, API key, region…). Only GitHub/Slack/Linear differ: they use + the OAuth tools `onboard_github` / `onboard_slack` / `onboard_linear`. +- **GitHub first** — required for deploy monitoring *and* discovery. +- **Proactively connect pull-based sources** you detected: databases (Postgres/MySQL/ClickHouse), observability + backends (Datadog/PromQL/GCP Monitoring), cloud (AWS/GCP). Once connected, agents query them through the + `query` tool — see the `firetiger-query` "Querying connected sources" reference. +- **Action integrations:** ask once — "Which do you use? Slack, Linear, PagerDuty, incident.io" — then connect. +- **Any vendor without a dedicated connector** (e.g. Axiom): use a generic `OPENAPI`/`HTTP` connection with a + bearer token. **Private** databases need a NetworkTransport (Tailscale). Both in the reference. +- **Warn about gaps.** No integrations → the agent can monitor and analyze but can't act (Slack alerts, GitHub + issues). Note specifics: no Slack → no alerts; no GitHub → no codebase search, no deploy tracking, no discovery. + +## Step 6 — Register deployments + +Call **`get_deploy_credentials`** and wire a CI/CD step that POSTs deploy events (`repository`, `environment`, +`sha`) to the returned endpoint with Basic auth, so monitors verify each release. Details in +`firetiger-monitor-deploy`. + +## Step 7 — Let discovery run + +Once **GitHub + a telemetry/query source** are connected (the "qualifying" pair), Firetiger auto-discovers +**Services** (the customer's own software boundaries) and **Providers** (external dependencies — Postgres, +Vercel, OpenAI, etc.) by triangulating connections, OTEL signals, and the connected repos, and recommends what +to monitor. You don't build this by hand — connecting the qualifying pair unlocks it. + +## Step 8 — Create a monitoring agent + +Use **`create_agent_with_goal`** with a goal tailored to the stack (or install a prebuilt catalog agent — see +`firetiger-create-agent`): + +- **Next.js/React:** "Monitor this Next.js app for API route errors, slow page loads, and DB query issues. + Alert on error-rate spikes and p95 latency increases." +- **Python API:** "Monitor this Python API for request errors, slow endpoints, and exception patterns. Track + DB performance and alert on anomalies." +- **Go service:** "Monitor this Go service for errors, goroutine issues, and latency; alert on degradation." + +If the planner asks a question, answer from what you detected via `send_agent_message` on the plan session. + +## Step 9 — Summary + +Show: files changed, env vars needed in prod, connections configured, telemetry sources wired, deploys +registered, the agent created and its focus, discovered services/providers, and a dashboard link. + +## Common Mistakes + +| # | Mistake | Fix | +|---|---------|-----| +| 1 | **Assuming setup means ingesting telemetry** | Many customers connect *existing* systems (Datadog, Prometheus, GCP metrics, databases) as **pull-based** connections and never ingest — Firetiger queries into them. Offer this path, not just ingestion. | +| 2 | **Only considering the OTLP SDK** | Datadog Agent, Prometheus `remote_write`, Vector, platform drains, and `/datapoints/` are all first-class — match the source you actually have. | +| 3 | **Skipping GitHub** | GitHub is required for deploy monitoring and to unlock discovery — connect it early. | +| 4 | **Bearer auth on ingest/drains** | Everything ingest-side is HTTP Basic auth (`base64(user:pass)`). | +| 5 | **Creating the agent before connecting integrations** | Connect first, or the agent can't act (alerts, issues). | +| 6 | **Hand-hardcoding auth for query connections** | Connection credentials are proxy-injected by host — reference the connection, don't set `Authorization` yourself. | +| 7 | **Expecting discovery with only telemetry** | Discovery needs the qualifying pair: GitHub + a telemetry/query source. | +| 8 | **Committing credentials** | Add credential/env files to `.gitignore`; use platform secret stores in CI. | + +## Related + +- OTLP SDK instrumentation: `firetiger-instrument`. +- Deploy monitoring & the deployments API: `firetiger-monitor-deploy`. +- Agent & catalog configuration: `firetiger-create-agent`. +- Firetiger integration docs: . diff --git a/skills/firetiger-setup/references/aws.md b/skills/firetiger-setup/references/aws.md new file mode 100644 index 0000000..fc3abc4 --- /dev/null +++ b/skills/firetiger-setup/references/aws.md @@ -0,0 +1,55 @@ +# AWS CloudWatch logs + +Forward CloudWatch logs to Firetiger by deploying the onboarding CloudFormation stack (ingest + IAM role). The +ingest endpoint is `https://ingest.cloud.firetiger.com`; use the Step 1 credentials `$USERNAME` / `$PASSWORD`. +Requires `which aws` and a chosen `$REGION`. + +## Deploy the onboarding stack + +```bash +aws cloudformation create-stack \ + --stack-name firetiger-cloudwatch-logs \ + --template-url https://firetiger-public-$REGION.s3.$REGION.amazonaws.com/ingest/aws/cloudwatch/logs/ingest-and-iam-onboarding.yaml \ + --parameters \ + ParameterKey=FiretigerEndpoint,ParameterValue=https://ingest.cloud.firetiger.com \ + ParameterKey=FiretigerUsername,ParameterValue=$USERNAME \ + ParameterKey=FiretigerPassword,ParameterValue=$PASSWORD \ + ParameterKey=FiretigerExternalId,ParameterValue=$(uuidgen) \ + --capabilities CAPABILITY_NAMED_IAM \ + --region $REGION +``` + +## Wait, then read the outputs + +```bash +aws cloudformation wait stack-create-complete --stack-name firetiger-cloudwatch-logs --region $REGION +aws cloudformation describe-stacks --stack-name firetiger-cloudwatch-logs \ + --query 'Stacks[0].Outputs' --region $REGION +``` + +The outputs include the created IAM Role ARN — surface it in the summary so the user can confirm the +cross-account role in the Firetiger dashboard if prompted. + +## Other AWS sources + +CloudWatch Logs is the common case, but Firetiger ingests several other AWS streams (all Basic auth on the +`https://ingest.cloud.firetiger.com` host, typically delivered via **Kinesis Firehose**, which also supports a dedicated Firehose +auth mode): + +| Source | Endpoint | Delivery | +|--------|----------|----------| +| CloudWatch Logs | `/aws/cloudwatch/logs` | CloudWatch subscription → Firehose | +| ALB access logs | `/AWSLogs/...` | S3 access-log delivery | +| CloudFront logs | `/aws/cloudfront/kinesis` | Kinesis Firehose | +| ECS task state changes | `/aws/eventbridge/ecs-task-state-change`, `/aws/events` | EventBridge rule | +| Kafka / MSK broker logs | `/aws/kinesis/kafka/logs`, `/aws/kinesis/msk/logs` | Kinesis Firehose | + +Point a Firehose delivery stream (HTTP endpoint destination) or EventBridge API destination at the relevant +path with the Basic auth header. For richer app telemetry, prefer the OTLP SDK (`firetiger-instrument`) +alongside these infra logs. + +## Next steps + +- **Verify** — after some traffic, run `SHOW TABLES;` with `firetiger-query` (data can lag a few minutes right after setup). +- **Query AWS without ingesting** — AWS is also a **pull-based** connection (CloudWatch metrics, cost data) you query in place; see [connections.md](connections.md). +- **Other sources** — back to [ingest-sources.md](ingest-sources.md) for the full drain menu. diff --git a/skills/firetiger-setup/references/cloudflare.md b/skills/firetiger-setup/references/cloudflare.md new file mode 100644 index 0000000..e07560e --- /dev/null +++ b/skills/firetiger-setup/references/cloudflare.md @@ -0,0 +1,64 @@ +# Cloudflare Workers observability + +Send Cloudflare Workers traces and logs to Firetiger via Workers observability destinations. The ingest +endpoint is `https://ingest.cloud.firetiger.com`; auth uses the Step 1 credentials as `$AUTH_HEADER` = +`base64(username:password)`. Requires `which wrangler`. + +## Prerequisites + +- Account ID (`$ACCOUNT_ID`) and API credentials (`$CF_EMAIL`, `$CF_API_KEY`) — try `wrangler whoami`, or ask + the user. + +## Create the destinations + +```bash +# Traces +curl -X POST "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/observability/destinations" \ + -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_API_KEY" -H "Content-Type: application/json" \ + -d '{"name":"firetiger-traces","enabled":true,"configuration":{"type":"logpush", + "logpushDataset":"opentelemetry-traces","url":"https://ingest.cloud.firetiger.com/v1/traces", + "headers":{"Authorization":"Basic '$AUTH_HEADER'"}}}' + +# Logs +curl -X POST "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/observability/destinations" \ + -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_API_KEY" -H "Content-Type: application/json" \ + -d '{"name":"firetiger-logs","enabled":true,"configuration":{"type":"logpush", + "logpushDataset":"opentelemetry-logs","url":"https://ingest.cloud.firetiger.com/v1/logs", + "headers":{"Authorization":"Basic '$AUTH_HEADER'"}}}' +``` + +## Enable observability in `wrangler.toml` + +```toml +[observability.traces] +enabled = true +head_sampling_rate = 1.0 +destinations = ["firetiger-traces"] + +[observability.logs] +enabled = true +head_sampling_rate = 1.0 +destinations = ["firetiger-logs"] +``` + +Then deploy: + +```bash +wrangler deploy +``` + +## Logpush (beyond Workers) + +For zone-level HTTP request logs (not just Workers), create a **Logpush** job pointing at Firetiger. Firetiger +accepts Logpush three ways: +- **Direct HTTP** → `https://ingest.cloud.firetiger.com/cloudflare/logs` (or `/cloudflare/logpush/http_requests`). +- **Via S3** → `/cloudflare/logpush/s3` (S3 + EventBridge object notifications). +- **Via GCS** → `/cloudflare/logpush/gcs` (GCS + Pub/Sub notifications). + +Use direct HTTP for the simplest path; use the S3/GCS variants when you already push Logpush to object storage. + +## Next steps + +- **Verify** — after some traffic, run `SHOW TABLES;` with `firetiger-query` (data can lag a few minutes right after setup). +- **App-level traces** — Workers observability captures request logs/traces; for spans from your own code add the OTLP SDK via `firetiger-instrument`. +- **Other sources & integrations** — back to [ingest-sources.md](ingest-sources.md) for more drains, or [connections.md](connections.md) to connect GitHub/Slack and pull-based sources. diff --git a/skills/firetiger-setup/references/connections.md b/skills/firetiger-setup/references/connections.md new file mode 100644 index 0000000..b09deed --- /dev/null +++ b/skills/firetiger-setup/references/connections.md @@ -0,0 +1,105 @@ +# Connections: pull-based sources, integrations & the connect model + +A **Connection** is a configured integration Firetiger reaches an external system through. There are three +shapes, and most customers rely heavily on the **pull-based** ones — Firetiger queries *into* their existing +Datadog, Prometheus, databases, and cloud metrics rather than (or in addition to) ingesting from them. + +| Shape | What Firetiger does | Examples | +|-------|---------------------|----------| +| **Pull-based data sources** | Queries *into* the system on demand (no data copied) | Postgres, MySQL, ClickHouse, Datadog, Prometheus (PromQL), GCP Monitoring, AWS, Trino, Elasticsearch, web search | +| **Action / tool integrations** | Reads and takes actions | GitHub, Slack, Linear, PagerDuty, incident.io, Pylon, email | +| **Identity / compliance** | Reads identity/compliance data | Clerk, WorkOS, Vanta | + +(Push/ingest sources — where the customer *sends* telemetry to Firetiger — are covered in +[ingest-sources.md](ingest-sources.md). AWS, GCP, and Convex appear on both sides.) + +## How connections are created + +**A client agent can create most connection types directly** with `create` on the `connections` collection — +run `schema` (collection `connections`) first to get the proto shape, then `create` with the `connection_type`, +`display_name`, `description`, and the matching details (host/DSN, API key, region, …). Secrets are input-only +and stored in a secrets backend — they're never echoed back on `get`/`list`. + +The exceptions are the three **OAuth** integrations — **GitHub, Slack, Linear** — whose credentials can only +come from an OAuth handshake. Use the `onboard_github` / `onboard_slack` / `onboard_linear` MCP tools; each +returns a browser URL, the user completes OAuth, and Firetiger creates the connection. + +**Key gotcha:** once a connection exists, its credentials are **injected by a host-keyed egress proxy**. Agents +query or call the target by connection name and never handle the secret — never hand-set an `Authorization` +header for a connected service, and never query with an embedded password. + +## Pull-based data sources + +Once connected, these are queried through the `query` tool — see the `firetiger-query` skill's +"Querying connected sources" reference for the exact SQL (SQL databases via fully-qualified +`"connections/{name}".schema.table`, Datadog/Prometheus/GCP via vendor functions, cross-source joins). + +| Type | Config (via `create` on `connections`) | Auth | +|------|------------------------------------------|------| +| `POSTGRES` | `host`, `port` (5432), `database`, `username`, `password`, `ssl_mode`, `read_only` | password | +| `MYSQL` | `host`, `port` (3306), `database`, `username`, `password`, `ssl_mode`, `read_only` | password | +| `CLICKHOUSE` | `host`, `port` (9440/8443), `database`, `username`, `password`, `secure` | password | +| `TRINO` | `host`, `port`, `catalog`, `schema`, `username`, `password`, `secure` | password (optional) | +| `ELASTICSEARCH` | `url`, auth: `basic` / `api_key` / `none`, `tls_skip_verify` | basic or Elastic API key | +| `DATADOG` | `site`, `api_key`, `application_key` | API key + app key | +| `PROMQL` | `base_url`, `timeout`, auth: `basic` / `bearer` / `sigv4` (for AMP) | basic / bearer / SigV4 | +| `GCP` | `service_account_key` (JSON), `project_id`, `region` | service account | +| `AWS` | auth: `assume_role` (`role_arn`, `external_id`) **or** `static_credentials`, `region` | STS AssumeRole (preferred) or IAM keys | +| `ICEBERG` | `catalog_uri`, auth: `basic` / `bearer` / `context` | Firetiger's own lake is the default `iceberg-gateway` | +| `WEB_SEARCH` | (none per-connection) | org-level (Brave) | + +> Note: `DATADOG` here is the *query* side (agents read your Datadog). Forwarding the Datadog **Agent** into +> Firetiger is the separate push path in [datadog.md](datadog.md). Similarly `AWS`/`GCP` are both query +> connections and log-forwarding ingest sources. + +## Action / tool integrations + +| Type | Config | Create path | +|------|--------|-------------| +| `GITHUB` | owner, repositories, permissions, `auto_monitor_pull_requests` | **`onboard_github`** (OAuth) | +| `SLACK` | workspace, bot token, allowed channels | **`onboard_slack`** (OAuth) | +| `LINEAR` | org, access/refresh token | **`onboard_linear`** (OAuth) | +| `PAGERDUTY` | `api_token` | `create` | +| `INCIDENT_IO` | `api_key`, `signing_secret` | `create` | +| `PYLON` | `api_token` | `create` | +| `EMAIL_WEBHOOK` | HTTP base URL + auth + signing secret | `create` | +| `GOOGLE_POSTMASTER` | service account (domain-wide delegation) | `create` | +| `CLERK` / `WORKOS` / `VANTA` | bearer token / OAuth client-credentials, `read_only` | `create` | + +What they unlock: **GitHub** — deploy monitoring, codebase search, PR/issue actions, and (with telemetry) +discovery; **Slack** — alerts and PR-author DMs; **PagerDuty / incident.io / Linear** — opening and managing +incidents/tickets. + +## Generic connectors — reaching any API (including Axiom) + +For a vendor with **no dedicated connector**, use a generic protocol connection. Firetiger's agents then call +it with proxy-injected auth: + +| Type | Config | Use for | +|------|--------|---------| +| `OPENAPI` | `spec_url`, `server_url`, auth: `bearer` / `basic` / `oauth_client_credentials`, `read_only` | Any API that publishes an OpenAPI spec (introspectable, read-only-gateable) | +| `HTTP` | `base_url`, `allowed_routes[]`, `headers`, auth: `static_headers` / `bearer` / `basic` / `oauth_client_credentials` | Any raw REST API | +| `GRPC` | `address` (host:port), `protocol`, auth: `basic` / `bearer` (injected on :443) | gRPC services with server reflection | +| `GRAPHQL` | `url`, auth: `bearer` / `basic` / `static_headers` | GraphQL endpoints | + +**Axiom** has no dedicated connector. Connect it as an **`OPENAPI`** connection to `https://api.axiom.co` with a +bearer token (Axiom publishes an OpenAPI spec) — or as `HTTP` with a bearer header. (Axiom's +Elasticsearch-compatible endpoint could also back an `ELASTICSEARCH` connection.) + +## Private networks (NetworkTransports) + +A connection whose target isn't publicly reachable references a **NetworkTransport** (`network_transport` +field) — currently **Tailscale** (OAuth client credentials). Firetiger stands up an on-demand tunnel and dials +the private host through it. The transport's route `domain` must cover the target host **exactly** — an exact +host or a `*.prefix` wildcard, no implicit subdomains; a host no route covers is dialed publicly and silently +times out. Required for any database or service on a private network (e.g. RDS behind a VPC). + +## Connectivity & health + +A connection is **healthy** when Firetiger can resolve its credentials. Firetiger validates with a read-only +probe per type — `SELECT 1` (SQL databases), instant `up` (PromQL), list metrics (Datadog), STS +`GetCallerIdentity` (AWS), list repos (GitHub), etc. A 2xx/success means the credentials authenticate; 401/403 +means they don't; a timeout means a reachability/transport problem, not auth. For **ingest** sources the real +check is instead "is data flowing" — query the landing tables with `firetiger-query`. + +See for per-integration setup detail. diff --git a/skills/firetiger-setup/references/datadog.md b/skills/firetiger-setup/references/datadog.md new file mode 100644 index 0000000..78c11c9 --- /dev/null +++ b/skills/firetiger-setup/references/datadog.md @@ -0,0 +1,63 @@ +# Datadog with Firetiger + +There are two ways to use Datadog with Firetiger, and they're equally valid — pick by what the customer wants, +or do both: + +- **Query your existing Datadog** (pull-based connection) — Firetiger's agents read your Datadog logs, spans, + metrics, monitors, and dashboards on demand. Nothing moves; your data stays in Datadog. +- **Forward the Datadog Agent** (push/ingest) — repoint the agent so its telemetry lands in Firetiger's lake. + Useful when consolidating onto Firetiger or migrating off Datadog. + +## Query your existing Datadog (pull) + +Add a `DATADOG` **Connection** — `schema` then `create` on the `connections` collection with `site`, +`api_key`, and `application_key`. Once connected, agents query it through the `query` tool with the Datadog +functions (`datadog_search_logs`, `datadog_query_metrics`, `datadog_cost_analysis`, …) — see the +`firetiger-query` "Querying connected sources" reference, and [connections.md](connections.md) for the +connection config. This needs no changes to your Datadog setup at all. + +## Forward the Datadog Agent (push) + +If the project already runs the **Datadog Agent**, repoint it at Firetiger's Datadog-compatible ingest to send +logs, metrics, and traces with almost no code change (uses the Step 1 ingest credentials). + +Firetiger accepts Datadog-agent traffic at `https://ingest.cloud.firetiger.com` on these paths: + +| Signal | Endpoint(s) | +|--------|-------------| +| Logs | `/datadog/logs` | +| Metrics | `/datadog/api/v1/series`, `/datadog/api/v2/series` | +| Traces | `/datadog/api/v0.2/traces`, `/datadog/api/v0.7/traces` | + +Authentication accepts the Datadog **`DD-API-KEY`** header (use the Firetiger-issued key) as well as HTTP Basic +auth. Discovery/validation endpoints (`/datadog/info`, `/datadog/api/v1/validate`) are public. + +### Repoint the agent + +Set the base URL (and API key) in `datadog.yaml` or via environment: + +```yaml +# datadog.yaml +dd_url: "https://ingest.cloud.firetiger.com/datadog" # metrics +logs_config: + logs_dd_url: "ingest.cloud.firetiger.com:443" + use_http: true +apm_config: + apm_dd_url: "https://ingest.cloud.firetiger.com/datadog" # traces +api_key: "" +``` + +Or with environment variables: + +```bash +DD_DD_URL="https://ingest.cloud.firetiger.com/datadog" +DD_APM_DD_URL="https://ingest.cloud.firetiger.com/datadog" +DD_LOGS_CONFIG_LOGS_DD_URL="ingest.cloud.firetiger.com:443" +DD_API_KEY="" +``` + +### Dual-shipping + +To validate before cutting over, run both backends in parallel (Datadog + Firetiger) using the agent's +multi-destination config, compare the data in Firetiger with `firetiger-query`, then remove the Datadog +destination once confident. diff --git a/skills/firetiger-setup/references/gcp.md b/skills/firetiger-setup/references/gcp.md new file mode 100644 index 0000000..7d0c990 --- /dev/null +++ b/skills/firetiger-setup/references/gcp.md @@ -0,0 +1,54 @@ +# GCP Cloud Logging + +Forward GCP Cloud Logging to Firetiger via a logging sink → Pub/Sub topic → Cloud Function forwarder. The +ingest endpoint is `https://ingest.cloud.firetiger.com`; use the Step 1 credentials `$USERNAME` / `$PASSWORD`. +Requires `which gcloud` and a chosen `$REGION`. + +```bash +# Current project +PROJECT=$(gcloud config get-value project) + +# Enable required APIs +gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com logging.googleapis.com \ + run.googleapis.com cloudbuild.googleapis.com artifactregistry.googleapis.com eventarc.googleapis.com + +# Pub/Sub topic +gcloud pubsub topics create firetiger-cloud-logs + +# Logging sink → topic +gcloud logging sinks create firetiger-cloud-logs \ + pubsub.googleapis.com/projects/$PROJECT/topics/firetiger-cloud-logs + +# Grant the sink's writer identity publish permission +SINK_SA=$(gcloud logging sinks describe firetiger-cloud-logs --format='value(writerIdentity)') +gcloud pubsub topics add-iam-policy-binding firetiger-cloud-logs \ + --member="$SINK_SA" --role="roles/pubsub.publisher" + +# Deploy the forwarder Cloud Function +gcloud functions deploy firetiger-cloud-logs-forwarder \ + --gen2 --runtime=python313 \ + --trigger-topic=firetiger-cloud-logs \ + --entry-point=process_log_entry \ + --set-env-vars="FT_EXPORTER_ENDPOINT=https://ingest.cloud.firetiger.com,FT_EXPORTER_BASIC_AUTH_USERNAME=$USERNAME,FT_EXPORTER_BASIC_AUTH_PASSWORD=$PASSWORD" \ + --source=gs://firetiger-public/ingest/gcp/cloud-logging/function.zip \ + --region=$REGION +``` + +By default the sink forwards all project logs. Add a `--log-filter='...'` to the `gcloud logging sinks create` +command to scope which logs are exported. + +## Delivery options + +Firetiger accepts GCP logs two ways: +- **Pub/Sub pull** — the forwarder function above, driven by the logging sink → Pub/Sub topic (shown here). +- **HTTP push** — a Log Router sink or Cloud Function can POST Cloud Logging `LogEntry` payloads directly to + `https://ingest.cloud.firetiger.com/gcp/cloud-logging` with the Basic auth header, skipping the pull subscriber. + +Choose HTTP push for the simplest setup; use the Pub/Sub path when you want buffering/replay in front of +Firetiger. For app traces/metrics, add the OTLP SDK (`firetiger-instrument`). + +## Next steps + +- **Verify** — after some traffic, run `SHOW TABLES;` with `firetiger-query` (data can lag a few minutes right after setup). +- **Query GCP without ingesting** — GCP is also a **pull-based** connection (Cloud Monitoring metrics via PromQL) you query in place; see [connections.md](connections.md). +- **Other sources** — back to [ingest-sources.md](ingest-sources.md) for the full drain menu. diff --git a/skills/firetiger-setup/references/ingest-sources.md b/skills/firetiger-setup/references/ingest-sources.md new file mode 100644 index 0000000..1e99091 --- /dev/null +++ b/skills/firetiger-setup/references/ingest-sources.md @@ -0,0 +1,57 @@ +# Telemetry ingest sources + +Every way to get telemetry into Firetiger. All endpoints are paths under the ingest host +`https://ingest.cloud.firetiger.com` and use **HTTP Basic auth** (`Authorization: Basic base64(username:password)`, +the username/password from `get_ingest_credentials`), except the self-authenticating webhooks noted below. +Everything lands in Apache Iceberg tables you can then query with `firetiger-query`. + +## OpenTelemetry (OTLP) — the default + +Signals: logs, metrics, traces. Endpoints: `https://ingest.cloud.firetiger.com/v1/traces`, `/v1/logs`, +`/v1/metrics` (HTTP), plus OTLP gRPC on `:443`. Point any OTEL SDK or Collector at the ingest endpoint with the +Basic auth header. For SDK setup per language, use the **`firetiger-instrument`** skill. + +## Platform log/trace drains + +| Platform | Signals | How | Recipe | +|----------|---------|-----|--------| +| Vercel | logs, traces | Log Drain + trace drain via Vercel API | [vercel.md](vercel.md) | +| AWS | logs, access logs, ECS events | CloudFormation → CloudWatch subscription / Kinesis Firehose / EventBridge / S3 (ALB, CloudFront, MSK, Kafka) | [aws.md](aws.md) | +| GCP | logs | Cloud Logging sink → Pub/Sub → forwarder, or HTTP push to `/gcp/cloud-logging` | [gcp.md](gcp.md) | +| Cloudflare | request logs, traces | Workers observability destinations + Logpush (direct, or via S3/GCS notifications) | [cloudflare.md](cloudflare.md) | + +## Agent / collector forwarders + +| Source | Signals | Endpoint(s) | Notes | +|--------|---------|-------------|-------| +| **Datadog Agent** | logs, metrics, traces | `/datadog/logs`, `/datadog/api/v1/series`, `/datadog/api/v2/series`, `/datadog/api/v0.2/traces`, `/datadog/api/v0.7/traces` | Repoint `DD_URL` at Firetiger. Uses `DD-API-KEY` header auth as well as Basic. A drop-in migration path — see [datadog.md](datadog.md). | +| **Prometheus** | metrics | `/api/v1/write` | Add Firetiger as a `remote_write` target. | +| **Vector** | logs, metrics, traces | native Vector sink, or the HTTP sink → `/datapoints/` | Basic auth. | +| **Generic HTTP sink** | arbitrary events | `/datapoints/{table}` | Table chosen from the URL suffix — for Fastly, custom shippers, anything that can POST JSON. | + +## Event & product webhooks + +| Source | Signals | Endpoint | Auth | +|--------|---------|----------|------| +| **GitHub** | PR/deploy/issue events, `@firetiger` mentions | `/github/events` (GitHub App) | GitHub App (Clerk M2M) | +| **SendGrid** | email delivery/engagement | `/sendgrid/`, `/sendgrid/delivery`, `/sendgrid/engagement` | Self-auth via HMAC signature | +| **Convex** | function/console/audit logs | `/convex/` | Self-auth via HMAC | +| **incident.io** | incident events | `/incident-io/events` | webhook | + +## Choosing + +- Already running the **Datadog Agent**? Repoint it — you keep all three signals with near-zero code change. +- Already exporting **Prometheus**? Add a `remote_write` target. +- On **Vercel/AWS/GCP/Cloudflare**? Use the platform drain — no app changes. +- Greenfield or want traces from your own code? Instrument with the **OTLP SDK** (`firetiger-instrument`). +- Odd source that can POST JSON? Use **`/datapoints/`**. + +After wiring a source, verify data is arriving with `firetiger-query` (`SHOW TABLES;`). Data can lag a few +minutes right after setup — absence immediately after isn't a failure. + +## Where to go next + +- **Don't want to ship telemetry at all?** Many sources (Datadog, Prometheus, GCP metrics, databases) can be + **pull-based** connections Firetiger queries *into* instead — see [connections.md](connections.md). +- **Instrumenting your own code** for OTLP: the `firetiger-instrument` skill (per-language SDK setup). +- **Querying what landed**, including joins across connected sources: the `firetiger-query` skill. diff --git a/skills/firetiger-setup/references/vercel.md b/skills/firetiger-setup/references/vercel.md new file mode 100644 index 0000000..7cd6a01 --- /dev/null +++ b/skills/firetiger-setup/references/vercel.md @@ -0,0 +1,51 @@ +# Vercel log & trace drains + +Forward Vercel logs and traces to Firetiger via the Vercel API. The ingest endpoint is +`https://ingest.cloud.firetiger.com`; auth uses the Step 1 credentials as `$AUTH_HEADER` = +`base64(username:password)`. + +## Prerequisites + +- `which vercel` succeeds. +- A Vercel token (check `vercel whoami`, or ask the user for one). + +## Find the project and team + +```bash +curl -H "Authorization: Bearer $TOKEN" "https://api.vercel.com/v9/projects" +``` + +Grab the `$PROJECT_ID` for the project being onboarded. + +## Create the logs drain + +```bash +curl -X POST "https://api.vercel.com/v1/drains" \ + -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ + -d '{"name":"Send logs to Firetiger","projects":"some","projectIds":["'$PROJECT_ID'"], + "schemas":{"log":{"version":"v1"}}, + "delivery":{"type":"http","endpoint":"https://ingest.cloud.firetiger.com/vercel/logs","encoding":"json", + "headers":{"Authorization":"Basic '$AUTH_HEADER'"}}, + "filter":{"version":"v2","filter":{"type":"basic","log":{"sources":["lambda","edge"]}, + "deployment":{"environments":["production"]}}}}' +``` + +## Create the traces drain + +```bash +curl -X POST "https://api.vercel.com/v1/drains" \ + -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \ + -d '{"name":"Send traces to Firetiger","projects":"some","projectIds":["'$PROJECT_ID'"], + "schemas":{"trace":{"version":"v1"}}, + "delivery":{"type":"otlphttp","endpoint":{"traces":"https://ingest.cloud.firetiger.com/v1/traces"},"encoding":"json", + "headers":{"Authorization":"Basic '$AUTH_HEADER'"}}}' +``` + +The logs drain targets production `lambda` and `edge` sources; adjust the `filter` block to include other +environments or sources as needed. + +## Next steps + +- **Verify** — after some traffic, run `SHOW TABLES;` with `firetiger-query` (data can lag a few minutes right after setup). +- **App-level traces** — drains capture platform logs/traces; for spans from your own code add the OTLP SDK via `firetiger-instrument`. +- **Other sources & integrations** — back to [ingest-sources.md](ingest-sources.md) for more drains, or [connections.md](connections.md) to connect GitHub/Slack and pull-based sources. diff --git a/skills/firetiger/SKILL.md b/skills/firetiger/SKILL.md index 743e658..7f278c2 100644 --- a/skills/firetiger/SKILL.md +++ b/skills/firetiger/SKILL.md @@ -1,78 +1,72 @@ --- name: firetiger -description: "Route Firetiger observability tasks to the appropriate specialized skill" +description: > + Use when the user mentions Firetiger or wants to work with their observability + data — setting up Firetiger, instrumenting an app with OpenTelemetry, querying + traces/logs/metrics with SQL, investigating an incident, monitoring a PR or + deployment, or creating a monitoring agent. Always use this skill when the user + says "Firetiger", even for simple asks — it routes to the specialized skill that + carries the critical gotchas (Basic-auth ingest, DuckDB SQL over per-service + tables, the @firetiger comment flow) that prevent common mistakes. +license: Apache-2.0 user_invocable: true -user_invocable_description: "Firetiger observability toolkit - instrumentation and queries" +user_invocable_description: "Firetiger observability toolkit — setup, instrumentation, queries, investigations, deploy monitoring, and agents" +metadata: + author: firetiger + version: "1.0.0" + homepage: https://firetiger.com + source: https://github.com/firetiger-oss/skills --- -# Firetiger Observability Toolkit +# Firetiger -You are a routing agent for Firetiger observability tasks. Analyze the user's request and delegate to the appropriate specialized skill. +[Firetiger](https://firetiger.com) is an AI-powered observability platform. Telemetry (traces, logs, metrics) +lands in Apache Iceberg tables you query with SQL, and autonomous agents monitor your services, investigate +issues, and watch deployments. -## Task Routing +This skill is a **router**. Identify what the user wants and invoke the matching specialized skill — each one +is self-contained and carries the gotchas for its task. -Evaluate what the user wants to accomplish and use the appropriate skill: +## How you talk to Firetiger -### Instrumentation → `firetiger-instrument` -Use when the user wants to: -- Add OpenTelemetry instrumentation to their application -- Install OTEL SDKs (Node.js, Python, Go, Rust) -- Configure environment variables for telemetry export -- Set up automatic instrumentation libraries -- Connect their application to Firetiger +Two mechanisms, referenced throughout the skills: -**Trigger phrases:** "instrument my app", "add observability", "set up tracing", "configure OpenTelemetry", "send traces to Firetiger" +- **The Firetiger MCP server** — `https://api.cloud.firetiger.com/mcp/v1` (OAuth 2.0 bearer; the first tool + call opens a browser to sign in). Tools: `get_ingest_credentials`, `get_deploy_credentials`, `query`, + `monitor_pr`, `create_agent_with_goal`, `send_agent_message`, `read_agent_messages`, `resolve_url`, + `get_subscription_status`/`get_checkout_url` (billing), `onboard_github`/`onboard_slack`/`onboard_linear` + (OAuth connect), and generic CRUD (`schema`/`list`/`get`/`create`/`update`/`delete`). The collections a + client agent works with: `agents` + `sessions` (create/run agents), `triggers` + `scheduled-agent-runs` + (automate them), `investigations` (AI diagnosis sessions), `issues` (Known Issues, IDs are `FT-{n}` call + signs), `monitoring-plans` (deploy monitoring, read/delete), and `connections` (integrations). Always + `schema` a collection before `create`/`update`. Some tools and collections are gated by deployment config + and API-key policy. +- **The `@firetiger` GitHub flow** — comment `@firetiger` on a pull request to have Firetiger monitor the + deployment that PR produces. -### Querying Data → `firetiger-query` -Use when the user wants to: -- Query traces, logs, or metrics with SQL -- Analyze telemetry data -- Find errors or slow requests -- Aggregate or summarize observability data +**Key gotcha:** if the MCP tools aren't available, the user hasn't connected the Firetiger MCP server yet. The +target skill handles this — it tells the user to connect `https://api.cloud.firetiger.com/mcp/v1` and sign in, +then retries. -**Trigger phrases:** "find traces", "search logs", "query data", "show me errors", "analyze latency" +## Routing -### Investigations → `firetiger-investigate` -Use when the user wants to: -- Start a new investigation -- Diagnose an issue or incident -- Analyze a problem with observability data -- Track findings during troubleshooting - -**Trigger phrases:** "investigate", "start investigation", "diagnose", "what's wrong with", "troubleshoot" - -### Agent Planning → `firetiger-plan` -Use when the user wants to: -- Create a new agent -- Define agent prompts and capabilities -- Set up scheduled triggers (cron) for agents -- Create manual triggers for on-demand invocation - -**Trigger phrases:** "create agent", "new agent", "plan agent", "schedule agent", "create trigger" - -### Run Agent → `firetiger-run` -Use when the user wants to: -- Run an existing agent -- Start an agent session -- Interact with an agent -- Execute an agent task - -**Trigger phrases:** "run agent", "start agent", "talk to agent", "agent session", "execute agent" +| The user wants to… | Skill | Trigger phrases | +|---|---|---| +| Onboard a project end-to-end (detect stack → instrument → connect → agent) | **`firetiger-setup`** | "set up Firetiger", "onboard this project", "connect my app" | +| Add OpenTelemetry instrumentation (Node/Next.js/Python/Go/Rust) | **`firetiger-instrument`** | "instrument my app", "add OpenTelemetry", "send traces" | +| Query traces, logs, or metrics with SQL | **`firetiger-query`** | "find traces", "search logs", "show me errors", "analyze latency" | +| Investigate an incident and track findings | **`firetiger-investigate`** | "investigate", "diagnose", "what's wrong with", "troubleshoot" | +| Monitor a PR/deployment | **`firetiger-monitor-deploy`** | "monitor this PR", "watch this deploy", "@firetiger" | +| Create a monitoring agent, or configure agents/triggers | **`firetiger-create-agent`** | "create an agent", "monitor X automatically", "schedule an agent" | ## Execution -1. Identify the category of the user's request -2. Use the Skill tool to invoke the appropriate skill: - - `firetiger-instrument` for instrumentation tasks - - `firetiger-query` for querying and analysis - - `firetiger-investigate` for investigations and diagnosis - - `firetiger-plan` for creating new agents - - `firetiger-run` for running existing agents -3. If the request spans multiple categories, handle them sequentially +1. Classify the request into one row above. +2. Invoke the corresponding skill with the Skill tool (e.g. `firetiger-query`). +3. If a request spans categories ("set up Firetiger and monitor my next PR"), handle them sequentially — + usually `firetiger-setup` first, then the follow-up. -## MCP Server +## Resources -The Firetiger MCP server is available for direct API interactions. Use it when you need to: -- Execute SQL queries against the data warehouse -- Fetch trace or log data -- Interact with Firetiger's API directly +- [Firetiger Documentation](https://docs.firetiger.com) +- [OpenTelemetry Documentation](https://opentelemetry.io/docs/) diff --git a/skills/instrument/SKILL.md b/skills/instrument/SKILL.md deleted file mode 100644 index 11587db..0000000 --- a/skills/instrument/SKILL.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -name: firetiger-instrument -description: "Add OpenTelemetry instrumentation to applications for Firetiger observability" -user_invocable: true -user_invocable_description: "Instrument your application with OpenTelemetry for Firetiger" ---- - -# OpenTelemetry Instrumentation for Firetiger - -You are an expert at instrumenting applications with OpenTelemetry to send telemetry data to Firetiger. - -## Critical: Initialization Order - -**OpenTelemetry MUST be initialized before any other imports.** If instrumented libraries (Express, HTTP, database clients) are imported before OpenTelemetry initializes, auto-instrumentation will not work. Always load the instrumentation file first. - -## Framework Detection - -Detect the application's technology stack by examining: -- `package.json` → Node.js/TypeScript -- `next.config.js` or `next.config.ts` → Next.js (use `@vercel/otel`) -- `requirements.txt`, `pyproject.toml`, `setup.py` → Python -- `go.mod` → Go -- `Cargo.toml` → Rust - -## Next.js Applications - -For Next.js apps, use the `@vercel/otel` package for simplified setup. - -### Install Dependencies - -```bash -npm install @vercel/otel @opentelemetry/sdk-logs @opentelemetry/api-logs @opentelemetry/instrumentation -``` - -### Create Instrumentation File - -Create `instrumentation.ts` in the **root directory** (or `src/` if using src folder): - -```typescript -import { registerOTel } from '@vercel/otel' - -export function register() { - registerOTel({ serviceName: 'your-service-name' }) -} -``` - -### Environment Variables - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.cloud.firetiger.com" -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " -``` - -## Node.js / TypeScript (Non-Next.js) - -### Install Dependencies - -```bash -npm install @opentelemetry/api \ - @opentelemetry/sdk-node \ - @opentelemetry/auto-instrumentations-node \ - @opentelemetry/exporter-trace-otlp-proto \ - @opentelemetry/exporter-logs-otlp-proto \ - @opentelemetry/exporter-metrics-otlp-proto \ - @opentelemetry/sdk-metrics \ - @opentelemetry/sdk-logs -``` - -### Create Instrumentation File - -Create `instrumentation.ts` (or `instrumentation.js`). This file MUST be loaded before any application code: - -```typescript -import { NodeSDK } from '@opentelemetry/sdk-node'; -import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; -import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'; -import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'; - -const traceExporter = new OTLPTraceExporter({ - compression: 'gzip', // Reduce bandwidth usage -}); - -const sdk = new NodeSDK({ - serviceName: process.env.OTEL_SERVICE_NAME || 'my-service', - spanProcessor: new BatchSpanProcessor(traceExporter, { - maxQueueSize: 2048, - maxExportBatchSize: 512, - scheduledDelayMillis: 5000, - }), - metricReader: new PeriodicExportingMetricReader({ - exporter: new OTLPMetricExporter(), - }), - logRecordProcessor: new SimpleLogRecordProcessor(new OTLPLogExporter()), - instrumentations: [getNodeAutoInstrumentations()], -}); - -sdk.start(); - -// Graceful shutdown -process.on('SIGTERM', () => { - sdk.shutdown().then(() => process.exit(0)); -}); -``` - -### Environment Variables - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.cloud.firetiger.com" -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " -export OTEL_SERVICE_NAME="your-service-name" -``` - -### Run with Instrumentation - -```bash -node --import ./instrumentation.js app.js -# or for TypeScript with tsx -node --import tsx --import ./instrumentation.ts app.ts -``` - -## Python - -### Install Dependencies - -```bash -pip install opentelemetry-distro opentelemetry-exporter-otlp -opentelemetry-bootstrap -a install # Installs instrumentations for detected libraries -``` - -### Auto-Instrumentation (Recommended) - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.cloud.firetiger.com" -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " -export OTEL_SERVICE_NAME="your-service-name" - -opentelemetry-instrument python app.py -``` - -### Manual Setup (for more control) - -```python -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.semconv.resource import ResourceAttributes - -resource = Resource.create({ - ResourceAttributes.SERVICE_NAME: "your-service-name", - ResourceAttributes.DEPLOYMENT_ENVIRONMENT: "production", -}) - -provider = TracerProvider(resource=resource) -processor = BatchSpanProcessor(OTLPSpanExporter( - endpoint="https://ingest.cloud.firetiger.com", - headers={"Authorization": "Bearer "}, -)) -provider.add_span_processor(processor) -trace.set_tracer_provider(provider) -``` - -## Go - -### Install Dependencies - -```bash -go get go.opentelemetry.io/otel \ - go.opentelemetry.io/otel/sdk \ - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc \ - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp -``` - -### Setup Code - -```go -package main - -import ( - "context" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/sdk/resource" - "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.21.0" -) - -func initTracer(ctx context.Context) (*trace.TracerProvider, error) { - exporter, err := otlptracegrpc.New(ctx) - if err != nil { - return nil, err - } - - res, _ := resource.New(ctx, - resource.WithAttributes( - semconv.ServiceName("your-service-name"), - ), - ) - - tp := trace.NewTracerProvider( - trace.WithBatcher(exporter), - trace.WithResource(res), - ) - otel.SetTracerProvider(tp) - return tp, nil -} -``` - -### Environment Variables - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT="ingest.cloud.firetiger.com:443" -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer " -``` - -## Rust - -### Add Dependencies to Cargo.toml - -```toml -[dependencies] -opentelemetry = "0.21" -opentelemetry_sdk = { version = "0.21", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.14", features = ["tonic"] } -tracing = "0.1" -tracing-opentelemetry = "0.22" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -``` - -### Setup Code - -```rust -use opentelemetry::global; -use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::trace::TracerProvider; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; - -fn init_tracing() -> Result<(), Box> { - let exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint("https://ingest.cloud.firetiger.com"); - - let provider = TracerProvider::builder() - .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio) - .build(); - - global::set_tracer_provider(provider.clone()); - - let telemetry = tracing_opentelemetry::layer() - .with_tracer(provider.tracer("your-service-name")); - - tracing_subscriber::registry() - .with(telemetry) - .init(); - - Ok(()) -} -``` - -## Adding Custom Spans - -Auto-instrumentation covers common libraries, but add manual spans for business-critical operations. - -### Node.js Custom Spans - -```typescript -import { trace, SpanStatusCode } from '@opentelemetry/api'; - -const tracer = trace.getTracer('your-service-name'); - -async function processOrder(orderId: string, customerId: string) { - return await tracer.startActiveSpan('processOrder', async (span) => { - // Add business context as attributes - span.setAttribute('order.id', orderId); - span.setAttribute('customer.id', customerId); - - try { - const result = await executeOrder(orderId); - span.addEvent('order.completed', { 'order.total': result.total }); - return result; - } catch (error) { - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - span.recordException(error); - throw error; - } finally { - span.end(); - } - }); -} -``` - -### Python Custom Spans - -```python -from opentelemetry import trace - -tracer = trace.get_tracer("your-service-name") - -def process_order(order_id: str, customer_id: str): - with tracer.start_as_current_span("process_order") as span: - span.set_attribute("order.id", order_id) - span.set_attribute("customer.id", customer_id) - - try: - result = execute_order(order_id) - span.add_event("order.completed", {"order.total": result.total}) - return result - except Exception as e: - span.set_status(trace.StatusCode.ERROR, str(e)) - span.record_exception(e) - raise -``` - -## Log Correlation - -Inject trace context into logs to correlate logs with traces in Firetiger. - -### Node.js with Winston - -```typescript -import winston from 'winston'; -import { trace } from '@opentelemetry/api'; - -const logger = winston.createLogger({ - format: winston.format.combine( - winston.format((info) => { - const span = trace.getActiveSpan(); - if (span) { - const context = span.spanContext(); - info.trace_id = context.traceId; - info.span_id = context.spanId; - } - return info; - })(), - winston.format.json() - ), - transports: [new winston.transports.Console()], -}); -``` - -### Python with structlog - -```python -import structlog -from opentelemetry import trace - -def add_trace_context(logger, method_name, event_dict): - span = trace.get_current_span() - if span.is_recording(): - ctx = span.get_span_context() - event_dict["trace_id"] = format(ctx.trace_id, "032x") - event_dict["span_id"] = format(ctx.span_id, "016x") - return event_dict - -structlog.configure(processors=[add_trace_context, structlog.processors.JSONRenderer()]) -``` - -## Best Practices - -### Semantic Conventions - -Use OpenTelemetry semantic conventions for consistent attribute naming: -- `http.method`, `http.status_code`, `http.route` for HTTP -- `db.system`, `db.statement`, `db.name` for databases -- `rpc.service`, `rpc.method` for RPC calls -- Use `app.` prefix for custom business attributes - -### Performance Optimization - -1. **Use BatchSpanProcessor** in production (not SimpleSpanProcessor) -2. **Enable compression** (`gzip`) to reduce bandwidth -3. **Configure appropriate batch sizes** based on your traffic volume -4. **Set queue limits** to prevent memory issues under load - -### What to Instrument - -1. Start with auto-instrumentation for broad coverage -2. Add manual spans for: - - Business-critical operations (checkout, payment) - - Custom code not covered by auto-instrumentation - - Operations you need to measure for SLOs - -### Attribute Guidelines - -- Only include relevant attributes for each span -- Avoid high-cardinality attributes (user IDs as attribute values are fine, but not as attribute names) -- Never include sensitive data (passwords, tokens, PII) in attributes - -## Verification - -After setup, verify telemetry is flowing: -1. Generate some traffic to your application -2. Check the Firetiger console for incoming traces -3. Use SQL to query: `SELECT * FROM traces WHERE service_name = 'your-service-name' LIMIT 10` - -## Common Issues - -- **No data appearing**: Check that `OTEL_EXPORTER_OTLP_ENDPOINT` is set correctly -- **Authentication errors**: Verify your API key in `OTEL_EXPORTER_OTLP_HEADERS` -- **Missing spans**: Ensure auto-instrumentation libraries are installed for your frameworks -- **Spans not connected**: Verify context propagation is working (W3C Trace Context headers) -- **High memory usage**: Reduce batch sizes or enable sampling diff --git a/skills/investigate/SKILL.md b/skills/investigate/SKILL.md deleted file mode 100644 index 73f60c5..0000000 --- a/skills/investigate/SKILL.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -name: firetiger-investigate -description: "Start and interact with Firetiger investigations" -user_invocable: true -user_invocable_description: "Start an investigation to diagnose issues in Firetiger" ---- - -# Firetiger Investigation Guide - -You are an expert at running investigations in Firetiger to diagnose issues and analyze telemetry data. - -## Overview - -Firetiger investigations are time-bounded analysis sessions that help you systematically diagnose issues using observability data. Investigations track your findings, queries, and conclusions in one place. - -## Using MCP Tools - -The Firetiger MCP server provides tools for managing investigations: - -### Discover Investigation Schema - -First, understand the available fields: - -``` -schema with collection: "investigations" -``` - -This returns the field definitions for investigations including required fields and their types. - -### List Existing Investigations - -Find ongoing or past investigations: - -``` -list with resource: "investigations" -``` - -You can filter by status, time range, or other criteria to find relevant investigations. - -### Create a New Investigation - -Start a new investigation: - -``` -create with resource: "investigations" -``` - -Include: -- **title**: Brief description of what you're investigating -- **description**: Detailed context about the issue -- **time_range**: The time window to focus on -- **services**: Relevant services to investigate - -### Get Investigation Details - -Retrieve a specific investigation: - -``` -get with name: "investigations/{id}" -``` - -### Update Investigation - -Add findings or update status: - -``` -update with name: "investigations/{id}" -``` - -Update fields like: -- **findings**: Document what you discovered -- **status**: "in_progress", "resolved", "closed" -- **root_cause**: When identified - -## Investigation Workflow - -### 1. Start the Investigation - -1. Use `schema` to understand available fields -2. Create a new investigation with clear title and context -3. Note the investigation ID for reference - -### 2. Analyze Telemetry Data - -Use the `query` tool to run SQL queries against Firetiger's data warehouse: - -**Find errors in the time window:** -```sql -SELECT trace_id, service_name, span_name, status_code, timestamp -FROM traces -WHERE status_code = 'ERROR' - AND timestamp BETWEEN '{start_time}' AND '{end_time}' -ORDER BY timestamp DESC; -``` - -**Analyze latency patterns:** -```sql -SELECT - service_name, - span_name, - count(*) as count, - avg(duration_ns) / 1e6 as avg_ms, - max(duration_ns) / 1e6 as max_ms -FROM traces -WHERE timestamp BETWEEN '{start_time}' AND '{end_time}' -GROUP BY service_name, span_name -ORDER BY avg_ms DESC; -``` - -**Correlate logs with traces:** -```sql -SELECT l.timestamp, l.severity, l.body, t.span_name -FROM logs l -JOIN traces t ON l.trace_id = t.trace_id -WHERE l.timestamp BETWEEN '{start_time}' AND '{end_time}' - AND l.severity IN ('ERROR', 'WARN') -ORDER BY l.timestamp; -``` - -### 3. Document Findings - -Update the investigation with your findings: - -``` -update with name: "investigations/{id}" -``` - -Include: -- Patterns you identified -- Root cause analysis -- Affected services or endpoints -- Recommendations - -### 4. Close the Investigation - -When complete, update the status: - -``` -update with name: "investigations/{id}" - status: "resolved" - root_cause: "Description of the root cause" - resolution: "How it was fixed or mitigated" -``` - -## Best Practices - -1. **Define a clear time window**: Focus your analysis on a specific period -2. **Start broad, then narrow**: Begin with high-level queries and drill down -3. **Document as you go**: Update the investigation with findings incrementally -4. **Link related traces**: Note specific trace IDs that demonstrate the issue -5. **Consider dependencies**: Check upstream and downstream services - -## Common Investigation Scenarios - -### High Latency -1. Find slow traces in the time window -2. Identify which service/span is contributing most to latency -3. Check for database queries, external API calls, or resource contention - -### Error Spike -1. Query for errors grouped by service and error type -2. Find the first occurrence of the error pattern -3. Correlate with deployments or configuration changes - -### Missing Data -1. Check span counts by service over time -2. Look for gaps in the data -3. Verify instrumentation is working correctly diff --git a/skills/plan/SKILL.md b/skills/plan/SKILL.md deleted file mode 100644 index 257b3eb..0000000 --- a/skills/plan/SKILL.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -name: firetiger-plan -description: "Plan and create a new Firetiger agent" -user_invocable: true -user_invocable_description: "Create a new AI agent in Firetiger" ---- - -# Firetiger Agent Planning Guide - -You are an expert at designing and creating AI agents in Firetiger that automate observability workflows. - -## Overview - -Firetiger agents are AI-powered automation workers that can: -- Execute investigations and analyze telemetry data -- Interact with external systems via configured connections -- Be triggered on schedules or invoked manually - -## Using MCP Tools - -The Firetiger MCP server provides tools for creating agents and triggers. - -**Important:** Always use `schema` first to discover the current field definitions before creating resources. The examples below are illustrative - verify actual field names with the schema tool. - -### Discover Schemas - -``` -schema with collection: "agents" -schema with collection: "triggers" -``` - -### List Existing Resources - -``` -list with resource: "agents" -list with resource: "triggers" -``` - -### Create Resources - -``` -create with resource: "agents" -create with resource: "triggers" -``` - -## Agent Structure - -Agents have these core fields (verify with `schema`): - -- **name**: Resource name (format: `agents/{agent-id}`) -- **title**: Human-readable title -- **description**: What the agent does -- **prompt**: The initial prompt that guides agent behavior -- **connections**: List of enabled tool connections -- **state**: Agent state (`AGENT_STATE_ON`, `AGENT_STATE_OFF`, etc.) - -## Trigger Structure - -Triggers are **separate resources** that invoke agents. They are not embedded in the agent definition. - -Trigger fields: -- **name**: Resource name (format: `triggers/{trigger-id}`) -- **display_name**: Human-readable name (required) -- **description**: What the trigger does -- **agent**: Target agent (format: `agents/{agent-id}`, required) -- **enabled**: Boolean flag -- **configuration**: One of the following: - -### Cron Triggers - -Schedule agents to run periodically: - -``` -configuration: - cron: - schedule: "0 9 * * *" # Standard 5-field cron expression - timezone: "America/New_York" # IANA timezone (default: UTC) -``` - -Cron examples: -- `0 9 * * *` - Daily at 9 AM -- `*/15 * * * *` - Every 15 minutes -- `0 0 * * 1` - Weekly on Monday at midnight - -### Manual Triggers - -For on-demand invocation only: - -``` -configuration: - manual: {} -``` - -## Agent Design Process - -### 1. Define the Agent's Purpose - -Before creating an agent, clearly define: -- **What problem does it solve?** (e.g., "Analyze latency trends daily") -- **How will it be triggered?** (cron schedule or manual invocation) -- **What connections does it need?** (query, notifications, tickets) - -### 2. Write Clear Prompts - -Agent prompts should be: -- **Specific**: Define exactly what the agent should do -- **Scoped**: Limit the agent's responsibilities -- **Actionable**: Include concrete steps to follow - -Example prompt: -``` -You are a daily health check agent for the checkout service. - -Your job: -1. Query the last 24 hours of traces for the checkout service -2. Calculate error rates and p99 latency -3. Compare against the previous 24-hour period -4. If error rate increased >10% or p99 increased >20%, create an investigation -5. Summarize findings in a brief report - -Focus only on the checkout service. Do not investigate other services. -``` - -### 3. Create the Agent - -``` -create with resource: "agents" - title: "Checkout Health Monitor" - description: "Daily health analysis of the checkout service" - prompt: "You are a daily health check agent..." - state: "AGENT_STATE_ON" -``` - -### 4. Create a Trigger - -Create a separate trigger to schedule or enable invocation: - -``` -create with resource: "triggers" - display_name: "Daily Checkout Health Check" - description: "Runs checkout health analysis every morning" - agent: "agents/{agent-id}" - enabled: true - configuration: - cron: - schedule: "0 9 * * *" - timezone: "America/Los_Angeles" -``` - -## Agent Patterns - -### Daily Health Monitor - -Runs scheduled health checks: -- **Trigger**: Cron schedule (e.g., daily at 9 AM) -- **Actions**: Query metrics, compare to baseline, report anomalies - -### On-Demand Investigator - -Available for manual invocation when issues arise: -- **Trigger**: Manual (invoked by users or external systems) -- **Actions**: Deep-dive analysis, correlation, root cause identification - -### Periodic Report Generator - -Creates regular summary reports: -- **Trigger**: Cron schedule (e.g., weekly on Monday) -- **Actions**: Aggregate data, generate insights, send to stakeholders - -## Best Practices - -1. **Start simple**: Create focused agents that do one thing well -2. **Test manually first**: Use manual triggers to validate agent behavior before enabling cron schedules -3. **Set boundaries**: Define what agents should NOT do in the prompt -4. **Use appropriate schedules**: Don't run agents more frequently than needed -5. **Monitor agents**: Review agent sessions and outcomes regularly -6. **Iterate on prompts**: Improve prompts based on agent performance diff --git a/skills/query/SKILL.md b/skills/query/SKILL.md deleted file mode 100644 index ee8f1c4..0000000 --- a/skills/query/SKILL.md +++ /dev/null @@ -1,273 +0,0 @@ ---- -name: firetiger-query -description: "Query telemetry data in Firetiger using SQL via MCP" -user_invocable: true -user_invocable_description: "Query traces, logs, and metrics with SQL" ---- - -# Firetiger Query Guide - -You are an expert at querying observability data in Firetiger using SQL via the MCP server. - -## Overview - -Firetiger stores telemetry data in Apache Iceberg tables with dynamic schema inference. Use the Firetiger MCP server's `query` tool to execute DuckDB SQL against the data warehouse. - -**Important:** All SELECT queries MUST include `LIMIT < 200`. - -## Discovering Schema - -Before writing queries, discover available tables and columns: - -```sql --- List all tables -SHOW TABLES; - --- Show columns for a specific table -DESCRIBE "opentelemetry/traces/api-gateway"; - --- Alternative syntax -SHOW COLUMNS FROM "opentelemetry/logs/checkout-service"; -``` - -## Table Names - -Tables are namespaced by signal type and service name: - -- **Traces**: `"opentelemetry/traces/{service_name}"` -- **Logs**: `"opentelemetry/logs/{service_name}"` -- **Metrics Series**: `opentelemetry_metrics_series` -- **Metrics Data**: `opentelemetry_metrics_gauges`, `opentelemetry_metrics_counters_cumulative`, `opentelemetry_metrics_counters_delta`, `opentelemetry_metrics_histograms_cumulative`, etc. - -**Note:** Table names with slashes must be quoted: `"opentelemetry/traces/my-service"` - -## Traces Schema - -Key columns in trace tables: - -| Column | Type | Description | -|--------|------|-------------| -| `trace_id` | BINARY(16) | W3C Trace Context trace ID | -| `span_id` | BINARY(8) | Span ID within trace | -| `parent_span_id` | BINARY(8) | Parent span ID (optional) | -| `name` | STRING | Logical operation name (span name) | -| `kind` | INT | Span kind: 0=UNSPECIFIED, 1=INTERNAL, 2=SERVER, 3=CLIENT, 4=PRODUCER, 5=CONSUMER | -| `start_time` | TIMESTAMPTZ | Span start timestamp (partition key) | -| `end_time` | TIMESTAMPTZ | Span end timestamp | -| `status.code` | INT | Status: 0=UNSET, 1=OK, 2=ERROR | -| `status.message` | STRING | Status message | -| `attributes` | STRUCT | Span attributes (dynamically inferred) | -| `resource.attributes.service.name` | STRING | Service name | -| `resource.attributes.service.namespace` | STRING | Service namespace | -| `resource.attributes.service.version` | STRING | Service version | - -**Duration is calculated**, not stored: -```sql -end_time - start_time AS duration -``` - -## Logs Schema - -Key columns in log tables: - -| Column | Type | Description | -|--------|------|-------------| -| `time` | TIMESTAMPTZ | Log timestamp (partition key) | -| `observed_time` | TIMESTAMPTZ | When log was observed | -| `severity_number` | INT | Numeric severity: TRACE=1-4, DEBUG=5-8, INFO=9-12, WARN=13-16, ERROR=17-20, FATAL=21-24 | -| `severity_text` | STRING | Human-readable: TRACE, DEBUG, INFO, WARN, ERROR, FATAL | -| `body` | ANY | Log message body | -| `trace_id` | BINARY(16) | Trace ID for correlation (optional) | -| `span_id` | BINARY(8) | Span ID for correlation (optional) | -| `attributes` | STRUCT | Log attributes (dynamically inferred) | -| `resource.attributes.service.name` | STRING | Service name | - -## Accessing Attributes - -Attributes are stored as nested structs. Access them with dot notation: - -```sql --- Span attributes (up to 2 levels deep) -attributes.http.method -attributes.http.route -attributes.http.status_code -attributes.db.system -attributes.db.statement - --- Resource attributes -resource.attributes.service.name -resource.attributes.service.namespace -resource.attributes.telemetry.sdk.language -``` - -### Type Inference and Deeply Nested Attributes - -Firetiger infers attribute types with a **depth limit of 2 levels**: -- **Levels 1-2**: Expanded into queryable struct columns -- **Level 3+**: Stored as hex-encoded binary JSON - -For deeply nested attributes (3+ levels), decode and extract: - -```sql -SELECT - json_extract_string( - decode(attributes.request.context), - '$.tenant_id' - ) AS tenant_id -FROM "opentelemetry/traces/api" -WHERE start_time >= NOW() - INTERVAL '1 hour' -LIMIT 100; -``` - -## Query Examples - -### Find Recent Spans - -```sql -SELECT - trace_id, - name, - resource.attributes.service.name AS service, - end_time - start_time AS duration, - start_time -FROM "opentelemetry/traces/checkout-service" -WHERE start_time >= NOW() - INTERVAL '1 hour' -ORDER BY start_time DESC -LIMIT 100; -``` - -### Find Slow Spans - -```sql -SELECT - trace_id, - name, - resource.attributes.service.name AS service, - end_time - start_time AS duration -FROM "opentelemetry/traces/api-gateway" -WHERE end_time - start_time > INTERVAL '1 second' - AND start_time >= NOW() - INTERVAL '1 hour' -ORDER BY (end_time - start_time) DESC -LIMIT 50; -``` - -### Find Error Spans - -```sql -SELECT - trace_id, - name, - status.code, - status.message, - start_time -FROM "opentelemetry/traces/payment-service" -WHERE status.code = 2 -- ERROR - AND start_time >= NOW() - INTERVAL '1 hour' -ORDER BY start_time DESC -LIMIT 100; -``` - -### Aggregate by Span Name - -```sql -SELECT - name, - COUNT(*) AS span_count, - AVG(EXTRACT(EPOCH FROM (end_time - start_time))) AS avg_duration_sec, - MAX(EXTRACT(EPOCH FROM (end_time - start_time))) AS max_duration_sec -FROM "opentelemetry/traces/checkout-service" -WHERE start_time >= NOW() - INTERVAL '1 hour' -GROUP BY name -ORDER BY span_count DESC -LIMIT 50; -``` - -### Latency Percentiles by HTTP Route - -```sql -SELECT - attributes.http.route AS route, - COUNT(*) AS requests, - PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p50_sec, - PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p95_sec, - PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (end_time - start_time))) AS p99_sec -FROM "opentelemetry/traces/api-gateway" -WHERE start_time >= NOW() - INTERVAL '1 hour' - AND attributes.http.route IS NOT NULL -GROUP BY attributes.http.route -ORDER BY requests DESC -LIMIT 50; -``` - -### Search Error Logs - -```sql -SELECT - time, - severity_text, - body, - trace_id, - resource.attributes.service.name AS service -FROM "opentelemetry/logs/checkout-service" -WHERE time >= NOW() - INTERVAL '1 hour' - AND severity_number >= 17 -- ERROR and above -ORDER BY time DESC -LIMIT 100; -``` - -### Trace a Request Across Services - -```sql -SELECT - name, - resource.attributes.service.name AS service, - start_time, - end_time - start_time AS duration, - status.code -FROM "opentelemetry/traces/api-gateway" -WHERE trace_id = x'0123456789abcdef0123456789abcdef' -ORDER BY start_time -LIMIT 100; -``` - -### Query Metrics with Series Join - -```sql -SELECT - s.name AS metric_name, - s.resource.attributes.service.name AS service, - s.attributes.http.route AS route, - g.time, - g.value -FROM opentelemetry_metrics_gauges g -JOIN opentelemetry_metrics_series s ON g.series = s.series -WHERE g.time >= NOW() - INTERVAL '1 hour' - AND s.name = 'http.server.active_requests' -ORDER BY g.time DESC -LIMIT 100; -``` - -### Discover Available Metrics - -```sql -SELECT DISTINCT - name, - type, - unit, - description -FROM opentelemetry_metrics_series -WHERE time >= NOW() - INTERVAL '1 day' -ORDER BY name -LIMIT 100; -``` - -## Tips - -1. **Always include LIMIT < 200**: Required for all SELECT queries -2. **Quote table names with slashes**: `"opentelemetry/traces/my-service"` -3. **Filter by time first**: Use `start_time` (traces) or `time` (logs/metrics) for partition pruning -4. **Calculate duration**: Use `end_time - start_time`, not a duration column -5. **Use EXTRACT for aggregations**: `EXTRACT(EPOCH FROM (end_time - start_time))` gives seconds as a number -6. **Status codes are integers**: 0=UNSET, 1=OK, 2=ERROR -7. **Discover schema first**: Run `DESCRIBE "table_name"` to see available columns -8. **Handle hex-encoded fields**: Use `decode()` + `json_extract_string()` for deeply nested attributes diff --git a/skills/run/SKILL.md b/skills/run/SKILL.md deleted file mode 100644 index 8da7cf2..0000000 --- a/skills/run/SKILL.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -name: firetiger-run -description: "Run an existing Firetiger agent" -user_invocable: true -user_invocable_description: "Start a session with a Firetiger agent" ---- - -# Firetiger Agent Execution Guide - -You are an expert at running and interacting with Firetiger agents to automate observability workflows. - -## Overview - -Firetiger agents run in sessions - interactive conversations where you can give the agent tasks and receive responses. Sessions maintain context across multiple messages, allowing for complex multi-step workflows. - -## Using MCP Tools - -The Firetiger MCP server provides tools for running agents: - -### List Available Agents - -Find agents you can run: - -``` -list with resource: "agents" -``` - -This returns all configured agents with their descriptions and capabilities. - -### Get Agent Details - -Learn more about a specific agent: - -``` -get with name: "agents/{agent-id}" -``` - -This shows the agent's instructions, available tools, and configuration. - -### Create a New Session - -Start a conversation with an agent: - -``` -create with resource: "agents/{agent-id}/sessions" -``` - -This returns a session ID that you'll use for all subsequent interactions. - -### Send a Message - -Interact with the agent: - -``` -send_agent_message with session: "agents/{agent-id}/sessions/{session-id}" - message: "Your message here" -``` - -The agent will process your message and respond. This is a synchronous operation that waits for the agent's response. - -### Read Session History - -Review the conversation: - -``` -read_agent_messages with session: "agents/{agent-id}/sessions/{session-id}" -``` - -This returns all messages in the session, useful for reviewing what the agent has done. - -## Running an Agent - -### 1. Find the Right Agent - -List available agents to find one that matches your need: - -``` -list with resource: "agents" -``` - -Look for: -- **name**: The agent's resource name -- **title**: Human-readable title -- **description**: What the agent does - -### 2. Review Agent Capabilities - -Get details about the agent: - -``` -get with name: "agents/{agent-id}" -``` - -Understand: -- What connections (tools) the agent has access to -- What prompt guides its behavior -- Whether the agent is enabled (`state: AGENT_STATE_ON`) - -### 3. Start a Session - -Create a new session: - -``` -create with resource: "agents/{agent-id}/sessions" -``` - -Note the session ID returned - you'll need it for all interactions. - -### 4. Give the Agent a Task - -Send your first message: - -``` -send_agent_message with session: "agents/{agent-id}/sessions/{session-id}" - message: "Investigate the spike in 5xx errors from the checkout service in the last hour" -``` - -The agent will: -- Parse your request -- Plan its approach -- Execute necessary queries and actions -- Return a response with findings or results - -### 5. Continue the Conversation - -Send follow-up messages to refine or expand: - -``` -send_agent_message with session: "agents/{agent-id}/sessions/{session-id}" - message: "Can you dig deeper into the payment processing errors specifically?" -``` - -The agent maintains context from previous messages. - -### 6. Review the Session - -Read the full conversation history: - -``` -read_agent_messages with session: "agents/{agent-id}/sessions/{session-id}" -``` - -This shows all messages and agent actions for review or documentation. - -## Example Workflows - -### Error Investigation - -``` -# 1. Start session with error investigator agent -create with resource: "agents/error-investigator/sessions" -# Returns: session-id: "sess_abc123" - -# 2. Ask the agent to investigate -send_agent_message with session: "agents/error-investigator/sessions/sess_abc123" - message: "There's been an increase in errors from the API gateway in the last 30 minutes. Can you investigate?" - -# 3. Agent responds with initial findings, ask for more detail -send_agent_message with session: "agents/error-investigator/sessions/sess_abc123" - message: "What's causing the 503 errors specifically?" - -# 4. Ask for remediation -send_agent_message with session: "agents/error-investigator/sessions/sess_abc123" - message: "Please create an investigation ticket and notify the on-call engineer" -``` - -### Performance Analysis - -``` -# 1. Start session with performance agent -create with resource: "agents/perf-analyzer/sessions" - -# 2. Request analysis -send_agent_message - message: "Compare the p99 latency of the checkout flow this week vs last week" - -# 3. Drill down -send_agent_message - message: "Which specific endpoint is contributing most to the latency increase?" - -# 4. Get recommendations -send_agent_message - message: "What optimizations would you recommend?" -``` - -### Deployment Validation - -``` -# 1. Start session with deployment validator -create with resource: "agents/deploy-validator/sessions" - -# 2. Request validation -send_agent_message - message: "We just deployed version 2.3.1 to the payment service. Can you validate the deployment is healthy?" - -# 3. Agent runs checks and reports back -# 4. Ask for specific metrics -send_agent_message - message: "How do the error rates compare to the previous version?" -``` - -## Tips for Effective Agent Interactions - -1. **Be specific**: Give clear context about what you want investigated -2. **Include time ranges**: Specify when the issue occurred -3. **Name services**: Mention specific services or endpoints -4. **Ask follow-ups**: Drill down into findings with additional questions -5. **Request actions**: Ask the agent to create investigations, send alerts, etc. -