diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..64bdbe8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI Pipeline + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + # Instalujemy pytest, jeśli nie ma go w requirements (ale u Ciebie jest) + pip install pytest httpx + - name: Run Tests + run: | + pytest tests/ \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a363b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +env/ +ENV/ +.venv + +# Database +*.db +*.sqlite +*.sqlite3 + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Docker +.dockerignore + +# Logs +*.log + +# OS +.DS_Store +Thumbs.db diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4355fc4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.10-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN apt-get update && apt-get install -y \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --upgrade pip && \ + pip install -r requirements.txt + +COPY app/ ./app/ + +RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import requests; requests.get('http://localhost:8000/health')" || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index cf7ae5d..5d01397 100644 --- a/README.md +++ b/README.md @@ -1 +1,152 @@ -# Auto-Briefing-Agent \ No newline at end of file +# Auto-Briefing-Agent + +A robust, Dockerized API service using FastAPI that scrapes news titles from Hacker News (news.ycombinator.com), deduplicates them using a database, and prepares them for an LLM pipeline. + +## Features + +- **FastAPI** web framework for high-performance API endpoints +- **SQLModel** with SQLite for efficient data storage and deduplication +- **BeautifulSoup** for web scraping with ethical considerations +- **Docker** containerization for easy deployment +- **Structured logging** for monitoring and debugging +- **Robots.txt compliance** with request delays and custom User-Agent + +## Project Structure + +``` +. +├── app/ +│ ├── __init__.py +│ ├── main.py # FastAPI application and endpoints +│ ├── scraper.py # Hacker News scraping logic +│ ├── models.py # Database models (Article) +│ └── database.py # Database connection and session management +├── tests/ +│ ├── __init__.py +│ ├── test_main.py # API endpoint tests +│ └── test_scraper.py # Scraper tests +├── Dockerfile # Production-ready container definition +├── docker-compose.yml # Docker Compose configuration +├── requirements.txt # Python dependencies +└── README.md + +``` + +## Quick Start + +### Using Docker Compose (Recommended) + +1. **Build and run the service:** + ```bash + docker-compose up --build + ``` + +2. **The API will be available at:** + - API: http://localhost:8000 + - API Docs: http://localhost:8000/docs + - Health Check: http://localhost:8000/health + +### Using Docker directly + +1. **Build the image:** + ```bash + docker build -t auto-briefing-agent . + ``` + +2. **Run the container:** + ```bash + docker run -p 8000:8000 -v $(pwd)/articles.db:/app/articles.db auto-briefing-agent + ``` + +### Local Development + +1. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` + +2. **Run the application:** + ```bash + uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 + ``` + +3. **Run tests:** + ```bash + pytest tests/ + ``` + +## API Endpoints + +### POST `/scrape` + +Scrapes the top 5 articles from Hacker News and returns only new articles (deduplicated by URL). + +**Request:** +```bash +curl -X POST http://localhost:8000/scrape +``` + +**Response:** +```json +[ + { + "id": 1, + "title": "Article Title", + "url": "https://example.com/article", + "created_at": "2024-01-01T12:00:00", + "is_processed": false + } +] +``` + +### GET `/health` + +Health check endpoint. + +**Response:** +```json +{ + "status": "healthy" +} +``` + +### GET `/` + +Root endpoint with API information. + +## Database Schema + +The `Article` table contains: +- `id`: Primary key (auto-increment) +- `title`: Article title (indexed) +- `url`: Article URL (unique, indexed) +- `created_at`: Timestamp of when the article was scraped +- `is_processed`: Boolean flag for LLM pipeline processing status + +## Ethical Scraping + +The scraper implements several ethical practices: + +1. **Custom User-Agent**: Identifies the bot with contact information +2. **Request Delays**: 2-second delay between requests to respect robots.txt +3. **Error Handling**: Graceful handling of network errors and parsing issues +4. **Logging**: Comprehensive logging of all scraping activities + +## Development + +### Running Tests + +```bash +pytest tests/ -v +``` + +### Code Structure + +- **main.py**: FastAPI application with endpoint definitions +- **scraper.py**: HackerNewsScraper class with scraping logic +- **models.py**: SQLModel Article model definition +- **database.py**: Database initialization and session management + +## License + +This project is for educational/research purposes. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..23ff76f --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +# Auto-Briefing-Agent Application Package diff --git a/app/database.py b/app/database.py new file mode 100644 index 0000000..4bacdfe --- /dev/null +++ b/app/database.py @@ -0,0 +1,21 @@ +from sqlmodel import SQLModel, create_engine, Session +import logging + +logger = logging.getLogger(__name__) + + +DATABASE_URL = "sqlite:///./articles.db" + + +engine = create_engine(DATABASE_URL, echo=False, connect_args={"check_same_thread": False}) + + +def init_db(): + logger.info("Initializing database...") + SQLModel.metadata.create_all(engine) + logger.info("Database initialized successfully.") + + +def get_session(): + with Session(engine) as session: + yield session diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..8cbd6f8 --- /dev/null +++ b/app/main.py @@ -0,0 +1,102 @@ +import logging +from typing import List +from fastapi import FastAPI, Depends, HTTPException +from sqlmodel import Session, select +from contextlib import asynccontextmanager + +from app.database import init_db, get_session +from app.models import Article +from app.scraper import HackerNewsScraper + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("Starting Auto-Briefing-Agent API...") + init_db() + logger.info("API startup complete.") + yield + + logger.info("Shutting down Auto-Briefing-Agent API...") + + +app = FastAPI( + title="Auto-Briefing-Agent API", + description="API service for scraping and deduplicating Hacker News articles", + version="1.0.0", + lifespan=lifespan +) + + +@app.get("/") +async def root(): + return { + "message": "Auto-Briefing-Agent API", + "version": "1.0.0", + "endpoints": { + "scrape": "POST /scrape - Scrape Hacker News and return new articles", + "health": "GET /health - Health check endpoint" + } + } + + +@app.get("/health") +async def health(): + return {"status": "healthy"} + + +@app.post("/scrape", response_model=List[dict]) +async def scrape_articles(session: Session = Depends(get_session)): + logger.info("Scrape endpoint called") + + try: + scraper = HackerNewsScraper() + + scraped_articles = scraper.scrape_top_articles(limit=5) + logger.info(f"Scraped {len(scraped_articles)} articles from Hacker News") + + new_articles = [] + + for article_data in scraped_articles: + url = article_data['url'] + + statement = select(Article).where(Article.url == url) + existing = session.exec(statement).first() + + if existing: + logger.info(f"Article with URL '{url[:50]}...' already exists, skipping") + continue + + article = Article( + title=article_data['title'], + url=url, + is_processed=False + ) + + session.add(article) + session.commit() + session.refresh(article) + + logger.info(f"Saved new article: {article.title[:50]}... (ID: {article.id})") + + new_articles.append({ + "id": article.id, + "title": article.title, + "url": article.url, + "created_at": article.created_at.isoformat(), + "is_processed": article.is_processed + }) + + logger.info(f"Returning {len(new_articles)} new articles") + return new_articles + + except Exception as e: + logger.error(f"Error in scrape endpoint: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}") diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..962ae3b --- /dev/null +++ b/app/models.py @@ -0,0 +1,12 @@ +from datetime import datetime +from typing import Optional +from sqlmodel import SQLModel, Field + + +class Article(SQLModel, table=True): + + id: Optional[int] = Field(default=None, primary_key=True) + title: str = Field(index=True) + url: str = Field(unique=True, index=True) + created_at: datetime = Field(default_factory=datetime.utcnow) + is_processed: bool = Field(default=False) diff --git a/app/scraper.py b/app/scraper.py new file mode 100644 index 0000000..919c9bc --- /dev/null +++ b/app/scraper.py @@ -0,0 +1,73 @@ +import time +import logging +import requests +from bs4 import BeautifulSoup +from typing import List, Dict + +logger = logging.getLogger(__name__) + +USER_AGENT = "Auto-Briefing-Agent/1.0 (Educational/Research Purpose)" +REQUEST_DELAY = 30 + + +class HackerNewsScraper: + + def __init__(self, delay: float = REQUEST_DELAY): + self.delay = delay + self.session = requests.Session() + self.session.headers.update({ + 'User-Agent': USER_AGENT + }) + + def scrape_top_articles(self, limit: int = 5) -> List[Dict[str, str]]: + logger.info(f"Starting scrape of top {limit} articles from Hacker News") + + try: + time.sleep(self.delay) + + response = self.session.get('https://news.ycombinator.com', timeout=10) + response.raise_for_status() + + logger.info(f"Successfully fetched Hacker News page (Status: {response.status_code})") + + soup = BeautifulSoup(response.content, 'html.parser') + + articles = [] + article_rows = soup.find_all('tr', class_='athing') + + for idx, row in enumerate(article_rows[:limit]): + try: + title_container = row.find(class_='titleline') + if not title_container: + continue + + title_link = title_container.find('a') + + if title_link: + title = title_link.get_text(strip=True) + url = title_link.get('href', '') + + if url.startswith('item?'): + url = f"https://news.ycombinator.com/{url}" + elif not url.startswith('http'): + url = f"https://news.ycombinator.com/{url}" + + articles.append({ + 'title': title, + 'url': url + }) + logger.info(f"Scraped article {idx + 1}: {title[:50]}...") + + except Exception as e: + logger.warning(f"Error parsing article {idx + 1}: {e}") + continue + + logger.info(f"Successfully scraped {len(articles)} articles") + return articles + + except requests.RequestException as e: + logger.error(f"Error fetching Hacker News: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error during scraping: {e}") + raise \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8ecf223 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +services: + api: + build: . + container_name: auto-briefing-agent + ports: + - "8000:8000" + volumes: + - .:/app + - ./articles.db:/app/articles.db + environment: + - PYTHONUNBUFFERED=1 + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..89bb594 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +sqlmodel==0.0.14 +requests==2.31.0 +beautifulsoup4==4.12.2 +lxml==4.9.3 +pytest==7.4.3 +pytest-asyncio==0.21.1 +httpx==0.25.2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..d4839a6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Tests package diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..d5a853c --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,30 @@ +import pytest +from fastapi.testclient import TestClient +from app.main import app +from app.database import init_db, engine +from sqlmodel import SQLModel + + +@pytest.fixture +def client(): + """Create a test client.""" + SQLModel.metadata.create_all(engine) + yield TestClient(app) + SQLModel.metadata.drop_all(engine) + + +def test_root_endpoint(client): + """Test root endpoint.""" + response = client.get("/") + assert response.status_code == 200 + data = response.json() + assert "message" in data + assert "version" in data + + +def test_health_endpoint(client): + """Test health check endpoint.""" + response = client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..5aecbbf --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,53 @@ +import pytest +from unittest.mock import Mock, patch +from app.scraper import HackerNewsScraper, USER_AGENT, REQUEST_DELAY + + +def test_scraper_initialization(): + scraper = HackerNewsScraper() + assert scraper.delay == REQUEST_DELAY + assert scraper.session.headers['User-Agent'] == USER_AGENT + + +def test_scraper_custom_delay(): + scraper = HackerNewsScraper(delay=5.0) + assert scraper.delay == 5.0 + + +@patch('app.scraper.requests.Session.get') +def test_scrape_top_articles_logic(mock_get): + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = b""" + + + + + + + + + +
+ + Super AI Project + +
+ + Show HN: Local Link + +
+ + + """ + mock_get.return_value = mock_response + + scraper = HackerNewsScraper(delay=0) + articles = scraper.scrape_top_articles(limit=2) + + assert len(articles) == 2 + assert articles[0]['title'] == "Super AI Project" + assert articles[0]['url'] == "https://example.com/ai-news" + + assert articles[1]['url'] == "https://news.ycombinator.com/item?id=12345" \ No newline at end of file