From 91b5dbdfc74cbc82e883ecb88d42d7991238ba8c Mon Sep 17 00:00:00 2001 From: binamkayastha Date: Tue, 4 Aug 2026 20:22:47 -0400 Subject: [PATCH 1/2] Add SQLite metrics.sqlite DB for storing post engagement metrics Creates a metrics.sqlite artifact (persisted across workflow runs) with posts and post_metrics tables. collect_metrics now writes to SQLite instead of database.json; record_publish_results syncs new posts to both. Workflows download/upload metrics.sqlite alongside database.json. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/dev.yml | 17 ++- .github/workflows/prod.yml | 17 ++- main.py | 158 ++++++++++++++------ tests/test_collect_metrics_orchestration.py | 71 +++++---- tests/test_main.py | 7 +- 5 files changed, 192 insertions(+), 78 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 4443e69..e143b47 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -65,7 +65,15 @@ jobs: name: database.json github-token: ${{ secrets.GITHUB_TOKEN }} run-id: ${{ steps.get_id.outputs.previous_run_id }} - continue-on-error: true + continue-on-error: true + + - name: Download previous metrics artifact + uses: actions/download-artifact@v8 + with: + name: metrics.sqlite + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ steps.get_id.outputs.previous_run_id }} + continue-on-error: true - name: Call RescueGroups API env: @@ -92,6 +100,13 @@ jobs: path: database.json retention-days: 1 archive: false + + - name: Upload metrics artifact + uses: actions/upload-artifact@v7 + with: + path: metrics.sqlite + retention-days: 1 + archive: false - name: Upload Log artifact if: '!cancelled()' #This ensures this step runs even if the previous steps failed only if manually cancelled it doesnt run diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index b4f0cd8..d1528df 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -52,7 +52,15 @@ jobs: name: database.json github-token: ${{ secrets.GITHUB_TOKEN }} run-id: ${{ steps.get_id.outputs.previous_run_id }} - continue-on-error: true + continue-on-error: true + + - name: Download previous metrics artifact + uses: actions/download-artifact@v8 + with: + name: metrics.sqlite + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ steps.get_id.outputs.previous_run_id }} + continue-on-error: true - name: Call RescueGroups API env: @@ -73,6 +81,13 @@ jobs: retention-days: 14 archive: false + - name: Upload metrics artifact + uses: actions/upload-artifact@v7 + with: + path: metrics.sqlite + retention-days: 14 + archive: false + - name: Upload API Log artifact if: '!cancelled()' #This ensures this step runs even if the previous steps failed only if manually cancelled it doesnt run uses: actions/upload-artifact@v7 diff --git a/main.py b/main.py index 46e43bf..088e542 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ from pathlib import Path import pprint import random +import sqlite3 import sys import traceback @@ -83,7 +84,9 @@ def create_sources(debug=False): return [SourceRescueGroups()] -def run(sources, posters, collectors=None, database_path="database.json"): +def run(sources, posters, collectors=None, database_path="database.json", metrics_db_path="metrics.sqlite"): + _init_metrics_db(metrics_db_path) + pets = [] for source in sources: try: @@ -118,9 +121,9 @@ def run(sources, posters, collectors=None, database_path="database.json"): else: logger.info("%s post published.", poster.platform_name) - record_publish_results(pet, publish_results, database_path=database_path) + record_publish_results(pet, publish_results, database_path=database_path, metrics_db_path=metrics_db_path) - collect_metrics(collectors or [], database_path=database_path) + collect_metrics(collectors or [], database_path=database_path, metrics_db_path=metrics_db_path) return results @@ -142,7 +145,7 @@ def pick_pet(pets, database_path="database.json"): return random.choice(eligible) -def record_publish_results(pet, results, database_path="database.json"): +def record_publish_results(pet, results, database_path="database.json", metrics_db_path="metrics.sqlite"): data = _read_database(database_path) posted_pets = data.setdefault("posted_pets", []) posts = data.setdefault("posts", []) @@ -151,19 +154,20 @@ def record_publish_results(pet, results, database_path="database.json"): posted_pets.append( {"name": pet.name, "pet_id": pet.pet_id, "posted_at": posted_at} ) + new_posts = [] for poster, result in results: if not result.success: continue - posts.append( - { - "pet_id": pet.pet_id, - "platform": poster.platform_name, - "post_id": result.post_id, - "post_url": result.post_url, - "posted_at": posted_at, - "metrics": [], - } - ) + post_entry = { + "pet_id": pet.pet_id, + "platform": poster.platform_name, + "post_id": result.post_id, + "post_url": result.post_url, + "posted_at": posted_at, + "metrics": [], + } + posts.append(post_entry) + new_posts.append(post_entry) cutoff = datetime.now(timezone.utc) - timedelta(weeks=12) data["posted_pets"] = [ @@ -178,55 +182,119 @@ def record_publish_results(pet, results, database_path="database.json"): ] _write_database(database_path, data) + if new_posts: + _upsert_posts_to_db(new_posts, metrics_db_path) + -def collect_metrics(collectors, database_path="database.json", window_days=14): +def collect_metrics(collectors, database_path="database.json", metrics_db_path="metrics.sqlite", window_days=14): try: data = _read_database(database_path) posts = data.get("posts", []) if not posts: return + _init_metrics_db(metrics_db_path) + _upsert_posts_to_db(posts, metrics_db_path) + collectors_by_platform = { collector.platform_name: collector for collector in collectors } cutoff = datetime.now(timezone.utc) - timedelta(days=window_days) - updated = False - - for entry in posts: - try: - if datetime.fromisoformat(entry["posted_at"]) < cutoff: - continue - collector = collectors_by_platform.get(entry.get("platform")) - if collector is None: - continue + with sqlite3.connect(metrics_db_path) as conn: + for entry in posts: + try: + if datetime.fromisoformat(entry["posted_at"]) < cutoff: + continue - metrics = collector.fetch_metrics( - entry["post_id"], entry.get("post_url") - ) - if metrics is None: - continue - - snapshot = asdict(metrics) - snapshot["collected_at"] = datetime.now(timezone.utc).isoformat() - entry.setdefault("metrics", []).append(snapshot) - updated = True - except Exception as exc: - platform = entry.get("platform", "unknown platform") - post_id = entry.get("post_id", "unknown post") - logger.error( - "%s metric collection failed for %s: %s", - platform, - post_id, - exc, - ) + collector = collectors_by_platform.get(entry.get("platform")) + if collector is None: + continue - if updated: - _write_database(database_path, data) + metrics = collector.fetch_metrics( + entry["post_id"], entry.get("post_url") + ) + if metrics is None: + continue + + snapshot = asdict(metrics) + collected_at = datetime.now(timezone.utc).isoformat() + conn.execute( + """ + INSERT INTO post_metrics (post_id, collected_at, likes, reposts, comments) + VALUES (?, ?, ?, ?, ?) + """, + ( + entry["post_id"], + collected_at, + snapshot.get("likes"), + snapshot.get("reposts"), + snapshot.get("comments"), + ), + ) + except Exception as exc: + platform = entry.get("platform", "unknown platform") + post_id = entry.get("post_id", "unknown post") + logger.error( + "%s metric collection failed for %s: %s", + platform, + post_id, + exc, + ) except Exception as exc: logger.error("Metric collection failed: %s", exc) +def _init_metrics_db(metrics_db_path): + with sqlite3.connect(metrics_db_path) as conn: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pet_id TEXT NOT NULL, + platform TEXT NOT NULL, + post_id TEXT NOT NULL UNIQUE, + post_url TEXT NOT NULL, + posted_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS post_metrics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + post_id TEXT NOT NULL, + collected_at TEXT NOT NULL, + likes INTEGER, + reposts INTEGER, + comments INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (post_id) REFERENCES posts (post_id) + ); + """) + + +def _upsert_posts_to_db(posts, metrics_db_path): + with sqlite3.connect(metrics_db_path) as conn: + conn.executemany( + """ + INSERT INTO posts (pet_id, platform, post_id, post_url, posted_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(post_id) DO UPDATE SET + updated_at = datetime('now') + """, + [ + ( + post["pet_id"], + post["platform"], + post["post_id"], + post["post_url"], + post["posted_at"], + ) + for post in posts + ], + ) + + def _read_database(database_path): path = Path(database_path) if not path.exists() or path.stat().st_size == 0: diff --git a/tests/test_collect_metrics_orchestration.py b/tests/test_collect_metrics_orchestration.py index e543bdb..9422527 100644 --- a/tests/test_collect_metrics_orchestration.py +++ b/tests/test_collect_metrics_orchestration.py @@ -1,5 +1,6 @@ import json from datetime import datetime, timedelta, timezone +import sqlite3 from abstractions import PostMetrics from main import collect_metrics @@ -40,21 +41,24 @@ def snapshot(likes): ) +def _get_metrics(metrics_db_path, post_id): + with sqlite3.connect(metrics_db_path) as conn: + return conn.execute( + "SELECT likes, reposts, comments, collected_at FROM post_metrics WHERE post_id = ? ORDER BY id", + (post_id,), + ).fetchall() + + def test_collects_only_recent_posts_with_registered_collectors(tmp_path): database_path = tmp_path / "database.json" + metrics_db_path = tmp_path / "metrics.sqlite" now = datetime.now(timezone.utc) - previous = { - "collected_at": (now - timedelta(hours=1)).isoformat(), - "likes": 1, - "reposts": 0, - "comments": 0, - } database_path.write_text( json.dumps( { "posted_pets": [], "posts": [ - post("recent", (now - timedelta(days=1)).isoformat(), metrics=[previous]), + post("recent", (now - timedelta(days=1)).isoformat()), post("none", (now - timedelta(days=2)).isoformat()), post("old", (now - timedelta(days=15)).isoformat()), post( @@ -68,43 +72,46 @@ def test_collects_only_recent_posts_with_registered_collectors(tmp_path): ) collector = FakeCollector({"recent": snapshot(9), "none": None}) - collect_metrics([collector], database_path=database_path, window_days=14) + collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path, window_days=14) - data = json.loads(database_path.read_text()) assert collector.calls == [("recent", "at://recent"), ("none", "at://none")] - assert data["posts"][0]["metrics"][0] == previous - assert data["posts"][0]["metrics"][1]["likes"] == 9 - assert data["posts"][0]["metrics"][1]["reposts"] == 2 - assert data["posts"][0]["metrics"][1]["comments"] == 3 - collected_at = datetime.fromisoformat( - data["posts"][0]["metrics"][1]["collected_at"] - ) + + recent_rows = _get_metrics(metrics_db_path, "recent") + assert len(recent_rows) == 1 + assert recent_rows[0][0] == 9 # likes + assert recent_rows[0][1] == 2 # reposts + assert recent_rows[0][2] == 3 # comments + collected_at = datetime.fromisoformat(recent_rows[0][3]) assert collected_at.tzinfo == timezone.utc - assert data["posts"][1]["metrics"] == [] - assert data["posts"][2]["metrics"] == [] - assert data["posts"][3]["metrics"] == [] + + assert _get_metrics(metrics_db_path, "none") == [] + assert _get_metrics(metrics_db_path, "old") == [] + assert _get_metrics(metrics_db_path, "unknown") == [] def test_missing_database_is_a_noop(tmp_path): database_path = tmp_path / "database.json" + metrics_db_path = tmp_path / "metrics.sqlite" - collect_metrics([], database_path=database_path) + collect_metrics([], database_path=database_path, metrics_db_path=metrics_db_path) assert not database_path.exists() def test_missing_posts_key_is_a_noop(tmp_path): database_path = tmp_path / "database.json" + metrics_db_path = tmp_path / "metrics.sqlite" original = {"posted_pets": []} database_path.write_text(json.dumps(original)) - collect_metrics([], database_path=database_path) + collect_metrics([], database_path=database_path, metrics_db_path=metrics_db_path) assert json.loads(database_path.read_text()) == original def test_collector_error_does_not_stop_other_posts(tmp_path): database_path = tmp_path / "database.json" + metrics_db_path = tmp_path / "metrics.sqlite" recent = datetime.now(timezone.utc).isoformat() database_path.write_text( json.dumps({"posted_pets": [], "posts": [post("bad", recent), post("good", recent)]}) @@ -113,25 +120,29 @@ def test_collector_error_does_not_stop_other_posts(tmp_path): {"bad": RuntimeError("unreachable"), "good": snapshot(4)} ) - collect_metrics([collector], database_path=database_path) + collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path) - data = json.loads(database_path.read_text()) - assert data["posts"][0]["metrics"] == [] - assert data["posts"][1]["metrics"][0]["likes"] == 4 + assert _get_metrics(metrics_db_path, "bad") == [] + assert _get_metrics(metrics_db_path, "good")[0][0] == 4 def test_repeated_collection_appends_snapshots_without_adding_posts(tmp_path): database_path = tmp_path / "database.json" + metrics_db_path = tmp_path / "metrics.sqlite" recent = datetime.now(timezone.utc).isoformat() database_path.write_text( json.dumps({"posted_pets": [], "posts": [post("recent", recent)]}) ) collector = FakeCollector({"recent": snapshot(5)}) - collect_metrics([collector], database_path=database_path) + collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path) collector.responses["recent"] = snapshot(7) - collect_metrics([collector], database_path=database_path) + collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path) + + rows = _get_metrics(metrics_db_path, "recent") + assert len(rows) == 2 + assert [r[0] for r in rows] == [5, 7] - data = json.loads(database_path.read_text()) - assert len(data["posts"]) == 1 - assert [item["likes"] for item in data["posts"][0]["metrics"]] == [5, 7] + with sqlite3.connect(metrics_db_path) as conn: + post_count = conn.execute("SELECT COUNT(*) FROM posts").fetchone()[0] + assert post_count == 1 diff --git a/tests/test_main.py b/tests/test_main.py index 00ddc8d..6bfe00e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ import json from pathlib import Path +import sqlite3 from tempfile import TemporaryDirectory import unittest @@ -75,13 +76,17 @@ def test_run_calls_source_posters_and_collectors(self): with TemporaryDirectory() as temporary_directory: database_path = Path(temporary_directory) / "database.json" + metrics_db_path = Path(temporary_directory) / "metrics.sqlite" results = run( [source], [poster_one, poster_two], collectors=[collector], database_path=database_path, + metrics_db_path=metrics_db_path, ) data = json.loads(database_path.read_text()) + with sqlite3.connect(metrics_db_path) as conn: + rows = conn.execute("SELECT likes FROM post_metrics").fetchall() self.assertTrue(source.fetch_called) self.assertTrue(poster_one.format_called) @@ -92,7 +97,7 @@ def test_run_calls_source_posters_and_collectors(self): self.assertEqual(len(data["posted_pets"]), 1) self.assertEqual(len(data["posts"]), 2) self.assertEqual(len(collector.calls), 2) - self.assertEqual(data["posts"][0]["metrics"][0]["likes"], 3) + self.assertEqual(rows[0][0], 3) def test_run_with_mixed_species_pool(self): dog = AdoptablePet( From ffb53dc1046b5ffbd78c0615e955c1db3b6e28fd Mon Sep 17 00:00:00 2001 From: binamkayastha Date: Tue, 4 Aug 2026 20:32:12 -0400 Subject: [PATCH 2/2] Write metrics to both database.json and SQLite for easier migration collect_metrics now appends snapshots to database.json (original behavior) and also inserts rows into post_metrics in SQLite, so existing consumers of database.json are unaffected during the transition. Co-Authored-By: Claude Sonnet 4.6 --- main.py | 7 +++++ tests/test_collect_metrics_orchestration.py | 30 ++++++++++++++++++--- tests/test_main.py | 1 + 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 088e542..1d9cd02 100644 --- a/main.py +++ b/main.py @@ -200,6 +200,7 @@ def collect_metrics(collectors, database_path="database.json", metrics_db_path=" collector.platform_name: collector for collector in collectors } cutoff = datetime.now(timezone.utc) - timedelta(days=window_days) + updated = False with sqlite3.connect(metrics_db_path) as conn: for entry in posts: @@ -219,6 +220,9 @@ def collect_metrics(collectors, database_path="database.json", metrics_db_path=" snapshot = asdict(metrics) collected_at = datetime.now(timezone.utc).isoformat() + snapshot["collected_at"] = collected_at + entry.setdefault("metrics", []).append(snapshot) + updated = True conn.execute( """ INSERT INTO post_metrics (post_id, collected_at, likes, reposts, comments) @@ -241,6 +245,9 @@ def collect_metrics(collectors, database_path="database.json", metrics_db_path=" post_id, exc, ) + + if updated: + _write_database(database_path, data) except Exception as exc: logger.error("Metric collection failed: %s", exc) diff --git a/tests/test_collect_metrics_orchestration.py b/tests/test_collect_metrics_orchestration.py index 9422527..9ffdca5 100644 --- a/tests/test_collect_metrics_orchestration.py +++ b/tests/test_collect_metrics_orchestration.py @@ -53,12 +53,18 @@ def test_collects_only_recent_posts_with_registered_collectors(tmp_path): database_path = tmp_path / "database.json" metrics_db_path = tmp_path / "metrics.sqlite" now = datetime.now(timezone.utc) + previous = { + "collected_at": (now - timedelta(hours=1)).isoformat(), + "likes": 1, + "reposts": 0, + "comments": 0, + } database_path.write_text( json.dumps( { "posted_pets": [], "posts": [ - post("recent", (now - timedelta(days=1)).isoformat()), + post("recent", (now - timedelta(days=1)).isoformat(), metrics=[previous]), post("none", (now - timedelta(days=2)).isoformat()), post("old", (now - timedelta(days=15)).isoformat()), post( @@ -74,16 +80,25 @@ def test_collects_only_recent_posts_with_registered_collectors(tmp_path): collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path, window_days=14) + data = json.loads(database_path.read_text()) assert collector.calls == [("recent", "at://recent"), ("none", "at://none")] + assert data["posts"][0]["metrics"][0] == previous + assert data["posts"][0]["metrics"][1]["likes"] == 9 + assert data["posts"][0]["metrics"][1]["reposts"] == 2 + assert data["posts"][0]["metrics"][1]["comments"] == 3 + collected_at = datetime.fromisoformat( + data["posts"][0]["metrics"][1]["collected_at"] + ) + assert collected_at.tzinfo == timezone.utc + assert data["posts"][1]["metrics"] == [] + assert data["posts"][2]["metrics"] == [] + assert data["posts"][3]["metrics"] == [] recent_rows = _get_metrics(metrics_db_path, "recent") assert len(recent_rows) == 1 assert recent_rows[0][0] == 9 # likes assert recent_rows[0][1] == 2 # reposts assert recent_rows[0][2] == 3 # comments - collected_at = datetime.fromisoformat(recent_rows[0][3]) - assert collected_at.tzinfo == timezone.utc - assert _get_metrics(metrics_db_path, "none") == [] assert _get_metrics(metrics_db_path, "old") == [] assert _get_metrics(metrics_db_path, "unknown") == [] @@ -122,6 +137,9 @@ def test_collector_error_does_not_stop_other_posts(tmp_path): collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path) + data = json.loads(database_path.read_text()) + assert data["posts"][0]["metrics"] == [] + assert data["posts"][1]["metrics"][0]["likes"] == 4 assert _get_metrics(metrics_db_path, "bad") == [] assert _get_metrics(metrics_db_path, "good")[0][0] == 4 @@ -139,6 +157,10 @@ def test_repeated_collection_appends_snapshots_without_adding_posts(tmp_path): collector.responses["recent"] = snapshot(7) collect_metrics([collector], database_path=database_path, metrics_db_path=metrics_db_path) + data = json.loads(database_path.read_text()) + assert len(data["posts"]) == 1 + assert [item["likes"] for item in data["posts"][0]["metrics"]] == [5, 7] + rows = _get_metrics(metrics_db_path, "recent") assert len(rows) == 2 assert [r[0] for r in rows] == [5, 7] diff --git a/tests/test_main.py b/tests/test_main.py index 6bfe00e..0aaccf4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -97,6 +97,7 @@ def test_run_calls_source_posters_and_collectors(self): self.assertEqual(len(data["posted_pets"]), 1) self.assertEqual(len(data["posts"]), 2) self.assertEqual(len(collector.calls), 2) + self.assertEqual(data["posts"][0]["metrics"][0]["likes"], 3) self.assertEqual(rows[0][0], 3) def test_run_with_mixed_species_pool(self):