MVP: Scraper backend & Docker environment.

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI Pipeline
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        # Instalujemy pytest, jeśli nie ma go w requirements (ale u Ciebie jest)
+        pip install pytest httpx
+    - name: Run Tests
+      run: |
+        pytest tests/

.gitignore

@@ -0,0 +1,49 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# Database
+*.db
+*.sqlite
+*.sqlite3
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Docker
+.dockerignore
+
+# Logs
+*.log
+
+# OS
+.DS_Store
+Thumbs.db

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN apt-get update && apt-get install -y \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+
+COPY app/ ./app/
+
+RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
+USER appuser
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD python -c "import requests; requests.get('http://localhost:8000/health')" || exit 1
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1 +1,152 @@
-# Auto-Briefing-Agent
+# Auto-Briefing-Agent
+
+A robust, Dockerized API service using FastAPI that scrapes news titles from Hacker News (news.ycombinator.com), deduplicates them using a database, and prepares them for an LLM pipeline.
+
+## Features
+
+- **FastAPI** web framework for high-performance API endpoints
+- **SQLModel** with SQLite for efficient data storage and deduplication
+- **BeautifulSoup** for web scraping with ethical considerations
+- **Docker** containerization for easy deployment
+- **Structured logging** for monitoring and debugging
+- **Robots.txt compliance** with request delays and custom User-Agent
+
+## Project Structure
+
+```
+.
+├── app/
+│   ├── __init__.py
+│   ├── main.py          # FastAPI application and endpoints
+│   ├── scraper.py       # Hacker News scraping logic
+│   ├── models.py        # Database models (Article)
+│   └── database.py      # Database connection and session management
+├── tests/
+│   ├── __init__.py
+│   ├── test_main.py     # API endpoint tests
+│   └── test_scraper.py  # Scraper tests
+├── Dockerfile           # Production-ready container definition
+├── docker-compose.yml   # Docker Compose configuration
+├── requirements.txt    # Python dependencies
+└── README.md
+
+```
+
+## Quick Start
+
+### Using Docker Compose (Recommended)
+
+1. **Build and run the service:**
+   ```bash
+   docker-compose up --build
+   ```
+
+2. **The API will be available at:**
+   - API: http://localhost:8000
+   - API Docs: http://localhost:8000/docs
+   - Health Check: http://localhost:8000/health
+
+### Using Docker directly
+
+1. **Build the image:**
+   ```bash
+   docker build -t auto-briefing-agent .
+   ```
+
+2. **Run the container:**
+   ```bash
+   docker run -p 8000:8000 -v $(pwd)/articles.db:/app/articles.db auto-briefing-agent
+   ```
+
+### Local Development
+
+1. **Install dependencies:**
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+2. **Run the application:**
+   ```bash
+   uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
+   ```
+
+3. **Run tests:**
+   ```bash
+   pytest tests/
+   ```
+
+## API Endpoints
+
+### POST `/scrape`
+
+Scrapes the top 5 articles from Hacker News and returns only new articles (deduplicated by URL).
+
+**Request:**
+```bash
+curl -X POST http://localhost:8000/scrape
+```
+
+**Response:**
+```json
+[
+  {
+    "id": 1,
+    "title": "Article Title",
+    "url": "https://example.com/article",
+    "created_at": "2024-01-01T12:00:00",
+    "is_processed": false
+  }
+]
+```
+
+### GET `/health`
+
+Health check endpoint.
+
+**Response:**
+```json
+{
+  "status": "healthy"
+}
+```
+
+### GET `/`
+
+Root endpoint with API information.
+
+## Database Schema
+
+The `Article` table contains:
+- `id`: Primary key (auto-increment)
+- `title`: Article title (indexed)
+- `url`: Article URL (unique, indexed)
+- `created_at`: Timestamp of when the article was scraped
+- `is_processed`: Boolean flag for LLM pipeline processing status
+
+## Ethical Scraping
+
+The scraper implements several ethical practices:
+
+1. **Custom User-Agent**: Identifies the bot with contact information
+2. **Request Delays**: 2-second delay between requests to respect robots.txt
+3. **Error Handling**: Graceful handling of network errors and parsing issues
+4. **Logging**: Comprehensive logging of all scraping activities
+
+## Development
+
+### Running Tests
+
+```bash
+pytest tests/ -v
+```
+
+### Code Structure
+
+- **main.py**: FastAPI application with endpoint definitions
+- **scraper.py**: HackerNewsScraper class with scraping logic
+- **models.py**: SQLModel Article model definition
+- **database.py**: Database initialization and session management
+
+## License
+
+This project is for educational/research purposes.

app/__init__.py

@@ -0,0 +1 @@
+# Auto-Briefing-Agent Application Package

app/database.py

@@ -0,0 +1,21 @@
+from sqlmodel import SQLModel, create_engine, Session
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+DATABASE_URL = "sqlite:///./articles.db"
+
+
+engine = create_engine(DATABASE_URL, echo=False, connect_args={"check_same_thread": False})
+
+
+def init_db():
+    logger.info("Initializing database...")
+    SQLModel.metadata.create_all(engine)
+    logger.info("Database initialized successfully.")
+
+
+def get_session():
+    with Session(engine) as session:
+        yield session

app/main.py

@@ -0,0 +1,102 @@
+import logging
+from typing import List
+from fastapi import FastAPI, Depends, HTTPException
+from sqlmodel import Session, select
+from contextlib import asynccontextmanager
+
+from app.database import init_db, get_session
+from app.models import Article
+from app.scraper import HackerNewsScraper
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logger.info("Starting Auto-Briefing-Agent API...")
+    init_db()
+    logger.info("API startup complete.")
+    yield
+
+    logger.info("Shutting down Auto-Briefing-Agent API...")
+
+
+app = FastAPI(
+    title="Auto-Briefing-Agent API",
+    description="API service for scraping and deduplicating Hacker News articles",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+
+@app.get("/")
+async def root():
+    return {
+        "message": "Auto-Briefing-Agent API",
+        "version": "1.0.0",
+        "endpoints": {
+            "scrape": "POST /scrape - Scrape Hacker News and return new articles",
+            "health": "GET /health - Health check endpoint"
+        }
+    }
+
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy"}
+
+
+@app.post("/scrape", response_model=List[dict])
+async def scrape_articles(session: Session = Depends(get_session)):
+    logger.info("Scrape endpoint called")
+
+    try:
+        scraper = HackerNewsScraper()
+
+        scraped_articles = scraper.scrape_top_articles(limit=5)
+        logger.info(f"Scraped {len(scraped_articles)} articles from Hacker News")
+
+        new_articles = []
+
+        for article_data in scraped_articles:
+            url = article_data['url']
+
+            statement = select(Article).where(Article.url == url)
+            existing = session.exec(statement).first()
+
+            if existing:
+                logger.info(f"Article with URL '{url[:50]}...' already exists, skipping")
+                continue
+
+            article = Article(
+                title=article_data['title'],
+                url=url,
+                is_processed=False
+            )
+
+            session.add(article)
+            session.commit()
+            session.refresh(article)
+
+            logger.info(f"Saved new article: {article.title[:50]}... (ID: {article.id})")
+
+            new_articles.append({
+                "id": article.id,
+                "title": article.title,
+                "url": article.url,
+                "created_at": article.created_at.isoformat(),
+                "is_processed": article.is_processed
+            })
+
+        logger.info(f"Returning {len(new_articles)} new articles")
+        return new_articles
+
+    except Exception as e:
+        logger.error(f"Error in scrape endpoint: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")