feat: implement backend ingestion, analysis, and worker pipeline

scripts/seed_reference_data.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.append(str(ROOT / "services" / "api"))
+
+from sqlalchemy import delete, select  # noqa: E402
+
+from app.core.config import get_settings  # noqa: E402
+from app.db.base import Base  # noqa: E402
+from app.db.session import engine, SessionLocal  # noqa: E402
+from app.models.entities import (  # noqa: E402
+    EvaluationMetric,
+    ModelRoute,
+    PromptVersion,
+    SourceConfig,
+    ThemeMap,
+    KoreanSecurity,
+)
+
+
+SOURCE_CONFIGS = [
+    {
+        "name": "Federal Reserve Press Releases",
+        "kind": "official_macro_policy",
+        "base_url": "https://www.federalreserve.gov/feeds/press_all.xml",
+        "schedule": "*/30 * * * 1-5",
+        "reliability_score": 0.98,
+    },
+    {
+        "name": "U.S. Treasury Press Releases",
+        "kind": "official_macro_policy",
+        "base_url": "https://home.treasury.gov/news/press-releases",
+        "schedule": "*/30 * * * 1-5",
+        "reliability_score": 0.95,
+    },
+    {
+        "name": "SEC Press Releases",
+        "kind": "corporate_filing",
+        "base_url": "https://www.sec.gov/news/pressreleases.rss",
+        "schedule": "*/30 * * * 1-5",
+        "reliability_score": 0.94,
+    },
+    {
+        "name": "CNBC Top News",
+        "kind": "trusted_market_news",
+        "base_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html",
+        "schedule": "*/30 * * * 1-5",
+        "reliability_score": 0.82,
+    },
+    {
+        "name": "MarketWatch Top Stories",
+        "kind": "trusted_market_news",
+        "base_url": "https://feeds.marketwatch.com/marketwatch/topstories/",
+        "schedule": "*/30 * * * 1-5",
+        "reliability_score": 0.8,
+    },
+    {
+        "name": "BEA News",
+        "kind": "macro_release",
+        "base_url": "https://www.bea.gov/rss/news.xml",
+        "schedule": "0 * * * 1-5",
+        "reliability_score": 0.9,
+    },
+]
+
+MODEL_ROUTES = [
+    ("macro_analyst", "gpt-5.4-mini"),
+    ("sector_analyst", "gpt-5.4-mini"),
+    ("microstructure_analyst", "gpt-5.4-mini"),
+    ("skeptic_analyst", "gpt-5.4-mini"),
+    ("korea_translator", "gpt-5.4"),
+    ("final_judge", "gpt-5.4"),
+]
+
+THEME_MAPS = [
+    ("ai_semiconductor", "HBM/반도체 장비", 0.92, ["hbm", "semiconductor", "packaging"]),
+    ("power_infrastructure", "전력기기/변압기", 0.88, ["grid", "transformer", "power"]),
+    ("defense", "방산/항공우주", 0.83, ["defense", "aerospace", "missile"]),
+    ("energy_raw_materials", "원전/우라늄", 0.8, ["uranium", "nuclear", "energy"]),
+    ("logistics_supply_chain", "해운/물류 자동화", 0.74, ["shipping", "logistics", "port"]),
+]
+
+KOREAN_SECURITIES = [
+    ("000660", "SK하이닉스", "KOSPI", "반도체", ["hbm", "semiconductor", "ai"]),
+    ("042700", "한미반도체", "KOSPI", "반도체장비", ["packaging", "hbm", "semiconductor"]),
+    ("089890", "코세스", "KOSDAQ", "반도체장비", ["packaging", "ai", "semiconductor"]),
+    ("017370", "우신시스템", "KOSPI", "전력설비", ["grid", "power", "automation"]),
+    ("010120", "LS ELECTRIC", "KOSPI", "전력기기", ["transformer", "grid", "power"]),
+    ("267260", "HD현대일렉트릭", "KOSPI", "전력기기", ["transformer", "grid", "power"]),
+    ("012450", "한화에어로스페이스", "KOSPI", "방산", ["defense", "aerospace", "missile"]),
+    ("079550", "LIG넥스원", "KOSPI", "방산", ["defense", "aerospace", "missile"]),
+    ("034020", "두산에너빌리티", "KOSPI", "원전", ["nuclear", "energy", "uranium"]),
+    ("105840", "우진", "KOSPI", "원전", ["nuclear", "uranium", "energy"]),
+    ("028670", "팬오션", "KOSPI", "해운", ["shipping", "logistics", "bulk"]),
+    ("124560", "태웅로직스", "KOSDAQ", "물류", ["logistics", "shipping", "port"]),
+]
+
+
+async def main() -> None:
+    settings = get_settings()
+    async with engine.begin() as connection:
+        await connection.run_sync(Base.metadata.create_all)
+    async with SessionLocal() as session:
+        for model in (SourceConfig, ModelRoute, PromptVersion, ThemeMap, KoreanSecurity, EvaluationMetric):
+            await session.execute(delete(model))
+
+        session.add_all(SourceConfig(**source) for source in SOURCE_CONFIGS)
+        session.add_all(
+            ModelRoute(role=role, model=model_name, temperature=0.2, metadata_json={})
+            for role, model_name in MODEL_ROUTES
+        )
+        session.add(
+            PromptVersion(
+                version=settings.prompt_version,
+                system_prompt=(
+                    "공식 원문과 시장 반응만 사용해 한국장 다음 영업일 테마를 추론한다. "
+                    "선반영과 갭페이드 리스크를 반드시 함께 제시한다."
+                ),
+                developer_prompt=(
+                    "모든 변경은 롤링 OOS 평가 후 수동 승인 전까지 운영 라우팅에 반영하지 않는다."
+                ),
+            )
+        )
+        session.add_all(
+            ThemeMap(
+                us_category=us_category,
+                korea_theme=korea_theme,
+                mapping_weight=weight,
+                beneficiary_tags=tags,
+                lag_profile="next_day_open",
+                notes="초기 수동 규칙 매핑",
+            )
+            for us_category, korea_theme, weight, tags in THEME_MAPS
+        )
+        session.add_all(
+            KoreanSecurity(
+                ticker=ticker,
+                name=name,
+                market=market,
+                sector=sector,
+                theme_tags=theme_tags,
+                metadata_json={},
+            )
+            for ticker, name, market, sector, theme_tags in KOREAN_SECURITIES
+        )
+        session.add_all(
+            [
+                EvaluationMetric(metric_name="theme_hit_rate", metric_value=0.0, split="validation", evaluation_run_id=None),  # type: ignore[arg-type]
+                EvaluationMetric(metric_name="leader_hit_rate", metric_value=0.0, split="validation", evaluation_run_id=None),  # type: ignore[arg-type]
+                EvaluationMetric(metric_name="false_positive_rate", metric_value=0.0, split="validation", evaluation_run_id=None),  # type: ignore[arg-type]
+                EvaluationMetric(metric_name="gap_fade_rate", metric_value=0.0, split="validation", evaluation_run_id=None),  # type: ignore[arg-type]
+            ]
+        )
+        await session.commit()
+
+        source_count = len((await session.execute(select(SourceConfig))).scalars().all())
+        print(f"Seed complete: {source_count} sources configured")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

services/api/app/__init__.py

@@ -0,0 +1 @@
+

services/api/app/api/routes/dashboard.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_db_session
+from app.repositories.dashboard import (
+    fetch_admin_settings,
+    fetch_dashboard,
+    fetch_evaluation_summary,
+    fetch_event_explorer,
+    fetch_replay,
+    fetch_theme_board,
+    fetch_weekly_replay,
+)
+from app.core.config import get_settings
+
+router = APIRouter()
+
+
+@router.get("/dashboard")
+async def dashboard(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_dashboard(session)
+
+
+@router.get("/events")
+async def event_explorer(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_event_explorer(session)
+
+
+@router.get("/themes")
+async def theme_board(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_theme_board(session)
+
+
+@router.get("/replay/weekly")
+async def weekly_replay(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_weekly_replay(session)
+
+
+@router.get("/replay/{date_label}")
+async def replay(date_label: str, session: AsyncSession = Depends(get_db_session)):
+    return await fetch_replay(session, date_label)
+
+
+@router.get("/evaluations/summary")
+async def evaluation_summary(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_evaluation_summary(session)
+
+
+@router.get("/admin/settings")
+async def admin_settings(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_admin_settings(session, get_settings())

services/api/app/api/routes/health.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.core.config import get_settings
+from app.schemas.api import HealthResponse
+
+router = APIRouter()
+
+
+@router.get("/health", response_model=HealthResponse)
+async def healthcheck() -> HealthResponse:
+    settings = get_settings()
+    return HealthResponse(status="ok", environment=settings.env, version="0.1.0")
+

services/api/app/api/routes/jobs.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import uuid
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import get_settings
+from app.db.session import get_db_session
+from app.models.entities import JobRun
+from app.repositories.dashboard import fetch_recent_jobs
+from app.schemas.api import JobRequest, JobResponse
+from app.services.jobs.queue import enqueue_job
+
+router = APIRouter()
+
+
+@router.post("/jobs/ingest", response_model=JobResponse)
+async def trigger_ingestion(
+    payload: JobRequest,
+    session: AsyncSession = Depends(get_db_session),
+) -> JobResponse:
+    settings = get_settings()
+    run_id = str(uuid.uuid4())
+    job_id = await enqueue_job(settings, "run_ingestion_job", run_id)
+    session.add(
+        JobRun(
+            id=uuid.UUID(run_id),
+            job_id=job_id,
+            job_name="run_ingestion_job",
+            trigger_kind="ingest",
+            status="queued",
+        )
+    )
+    await session.commit()
+    return JobResponse(status="queued", jobName="run_ingestion_job", jobId=job_id)
+
+
+@router.post("/jobs/analyze", response_model=JobResponse)
+async def trigger_analysis(
+    payload: JobRequest,
+    session: AsyncSession = Depends(get_db_session),
+) -> JobResponse:
+    settings = get_settings()
+    run_id = str(uuid.uuid4())
+    job_id = await enqueue_job(
+        settings,
+        "run_analysis_job",
+        run_id,
+        payload.as_of.isoformat() if payload.as_of else None,
+    )
+    session.add(
+        JobRun(
+            id=uuid.UUID(run_id),
+            job_id=job_id,
+            job_name="run_analysis_job",
+            trigger_kind="analyze",
+            status="queued",
+        )
+    )
+    await session.commit()
+    return JobResponse(status="queued", jobName="run_analysis_job", jobId=job_id)
+
+
+@router.post("/jobs/refresh", response_model=JobResponse)
+async def trigger_refresh(
+    payload: JobRequest,
+    session: AsyncSession = Depends(get_db_session),
+) -> JobResponse:
+    settings = get_settings()
+    run_id = str(uuid.uuid4())
+    job_id = await enqueue_job(
+        settings,
+        "run_full_pipeline_job",
+        run_id,
+        payload.as_of.isoformat() if payload.as_of else None,
+    )
+    session.add(
+        JobRun(
+            id=uuid.UUID(run_id),
+            job_id=job_id,
+            job_name="run_full_pipeline_job",
+            trigger_kind="refresh",
+            status="queued",
+        )
+    )
+    await session.commit()
+    return JobResponse(status="queued", jobName="run_full_pipeline_job", jobId=job_id)
+
+
+@router.get("/jobs/recent")
+async def recent_jobs(session: AsyncSession = Depends(get_db_session)):
+    return await fetch_recent_jobs(session)

services/api/app/core/config.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_name: str = "Finance Helper API"
+    env: Literal["local", "staging", "production", "test"] = "local"
+    api_prefix: str = "/api/v1"
+    log_level: str = "INFO"
+
+    database_url: str = Field(
+        default="postgresql+asyncpg://finance:finance@localhost:5432/finance_helper"
+    )
+    redis_url: str = "redis://localhost:6379/0"
+    openai_api_key: str | None = None
+    openai_base_url: str | None = None
+
+    prompt_version: str = "baseline-v1"
+    evidence_cache_ttl_seconds: int = 60 * 60 * 12
+    daily_cost_budget_usd: float = 25.0
+    per_job_cost_budget_usd: float = 5.0
+
+    premarket_cron: str = "0 8 * * 1-5"
+    postmarket_cron: str = "45 15 * * 1-5"
+
+    model_macro_analyst: str = "gpt-5.4-mini"
+    model_sector_analyst: str = "gpt-5.4-mini"
+    model_microstructure_analyst: str = "gpt-5.4-mini"
+    model_skeptic_analyst: str = "gpt-5.4-mini"
+    model_korea_translator: str = "gpt-5.4"
+    model_final_judge: str = "gpt-5.4"
+
+    alpha_vantage_api_key: str | None = None
+    polygon_api_key: str | None = None
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()
+