diff --git a/.agents/skills/x/SKILL.md b/.agents/skills/x/SKILL.md index 48fdf01..3468b9c 100644 --- a/.agents/skills/x/SKILL.md +++ b/.agents/skills/x/SKILL.md @@ -1,6 +1,6 @@ --- name: X -description: Use when building applications that interact with X (formerly Twitter) data and functionality. Reach for this skill when agents need to search posts, manage user interactions, stream real-time data, handle authentication, work with API endpoints, manage rate limits, or integrate X data into applications. +description: Use when building applications that interact with X (formerly Twitter) data. Reach for this skill when you need to search posts, manage user relationships, send direct messages, create content, access real-time streams, or analyze trends. Use it for data collection, content publishing, social listening, automation, and integration with X's platform. metadata: mintlify-proj: x version: "1.0" @@ -10,162 +10,232 @@ metadata: ## Product summary -The X API provides programmatic access to X's public conversation through REST endpoints. Agents use it to read posts, publish content, manage users, search archives, stream real-time data, and analyze trends. The API uses pay-per-usage pricing with flexible authentication methods (OAuth 1.0a, OAuth 2.0, Bearer tokens). Key files: Developer Console at console.x.com for app credentials; API base URL is `https://api.x.com/2/`. Official SDKs available for Python (`xdk`) and TypeScript (`@xdevplatform/xdk`). Primary docs: https://docs.x.com/x-api/introduction +The X API provides programmatic access to X's public conversation through modern REST endpoints. Build applications that search posts, publish content, manage users, access direct messages, create lists, and analyze trends. The API uses pay-per-usage pricing with no commitments. Key endpoints live at `https://api.x.com/2/`. Authenticate with Bearer tokens (app-only) or OAuth 1.0a/2.0 (user context). Official SDKs exist for Python (`xdk`) and TypeScript (`@xdevplatform/xdk`). Primary documentation: https://docs.x.com/x-api/introduction ## When to use -Reach for this skill when: -- Building applications that read, search, or publish posts -- Integrating user authentication (OAuth flows) -- Streaming real-time posts or managing filtered rules -- Looking up user profiles, followers, or relationships -- Managing direct messages, lists, bookmarks, or spaces -- Handling API authentication, rate limits, or error responses -- Paginating through large result sets -- Requesting specific data fields or related objects via expansions -- Debugging API errors (401, 403, 429, etc.) -- Choosing between recent search (7 days) vs. full-archive search +Use this skill when: +- **Searching posts**: Find posts by keyword, user, date range, or operators (hashtags, media, links, etc.) +- **Publishing content**: Create posts, manage edits, delete posts +- **User operations**: Look up users, manage follows/blocks/mutes, get user timelines +- **Direct messages**: Send/receive DMs, manage conversations +- **Lists**: Create, manage, and retrieve curated lists +- **Real-time data**: Stream posts matching rules, get filtered stream updates +- **Analytics**: Access engagement metrics, post counts, trends +- **Spaces**: Find and retrieve live audio conversations +- **Media**: Upload images/videos with posts, manage media metadata ## Quick reference ### Authentication methods | Method | Use case | Scope | -|:-------|:---------|:------| -| **Bearer Token** | App-only, public data | Read-only, no user context | -| **OAuth 1.0a** | User-context requests, actions on behalf of user | Read, write, DMs (3 levels) | -| **OAuth 2.0** | Modern user-context, fine-grained scopes | Recommended for new projects | -| **Basic Auth** | Enterprise APIs only | Server-to-server | +|--------|----------|-------| +| Bearer Token (app-only) | Read-only public data | App-only, no user context | +| OAuth 1.0a | User-context operations | Act on behalf of a user | +| OAuth 2.0 PKCE | User-context with scopes | Modern user authorization | -### Common endpoints - -| Resource | Endpoint | Method | Use | -|:---------|:---------|:-------|:----| -| User lookup | `/2/users/by/username/:username` | GET | Get user by username | -| Post lookup | `/2/tweets/:id` | GET | Get post by ID | -| Recent search | `/2/tweets/search/recent` | GET | Search last 7 days | -| Full-archive search | `/2/tweets/search/all` | GET | Search all posts (paid) | -| Filtered stream | `/2/tweets/search/stream` | GET | Real-time posts matching rules | -| Create post | `/2/tweets` | POST | Publish a post | -| User timeline | `/2/users/:id/tweets` | GET | Get user's posts | -| Followers | `/2/users/:id/followers` | GET | Get user's followers | +Get credentials from Developer Console at https://console.x.com -### Field parameters +### Common endpoints -Request additional data with field parameters: +| Task | Endpoint | Method | +|------|----------|--------| +| Look up user by username | `GET /2/users/by/username/{username}` | GET | +| Get user's posts | `GET /2/users/{id}/tweets` | GET | +| Search recent posts | `GET /2/tweets/search/recent` | GET | +| Search all posts (archive) | `GET /2/tweets/search/all` | GET | +| Create post | `POST /2/tweets` | POST | +| Delete post | `DELETE /2/tweets/{id}` | DELETE | +| Get post by ID | `GET /2/tweets/{id}` | GET | +| Send direct message | `POST /2/dm_conversations/with/{participant_id}/messages` | POST | +| Get DM events | `GET /2/dm_events` | GET | +| Follow user | `POST /2/users/{id}/following` | POST | +| Get followers | `GET /2/users/{id}/followers` | GET | +| Create list | `POST /2/lists` | POST | +| Get list posts | `GET /2/lists/{id}/tweets` | GET | +| Add filtered stream rule | `POST /2/tweets/search/stream/rules` | POST | +| Connect to filtered stream | `GET /2/tweets/search/stream` | GET | + +### Field and expansion parameters + +Request additional data with `fields` and `expansions`: ```bash -# Post fields -?tweet.fields=created_at,public_metrics,lang,author_id - -# User fields -?user.fields=created_at,description,public_metrics,verified +# Request specific fields +?tweet.fields=created_at,public_metrics,author_id +?user.fields=description,public_metrics,verified -# Media fields +# Include related objects +?expansions=author_id,attachments.media_keys ?media.fields=url,preview_image_url,alt_text - -# Expansions (include related objects) -?expansions=author_id,referenced_tweets.id ``` -### Rate limit headers +Common field combinations: +- **Post analytics**: `tweet.fields=created_at,public_metrics,possibly_sensitive` +- **User profiles**: `user.fields=created_at,description,location,public_metrics,verified` +- **Full context**: `expansions=author_id,referenced_tweets.id` + `user.fields=username,name` + +### Pagination -Every response includes: -- `x-rate-limit-limit` — max requests in window -- `x-rate-limit-remaining` — requests left -- `x-rate-limit-reset` — Unix timestamp when window resets +Use cursor-based pagination for large result sets: -### Response structure +```bash +# Initial request +?max_results=100 +# Next page (from response meta.next_token) +?max_results=100&pagination_token=abc123xyz +``` + +Response structure: ```json { - "data": { /* primary result */ }, - "includes": { /* related objects from expansions */ }, - "meta": { /* pagination, result count */ } + "data": [...], + "meta": { + "result_count": 100, + "next_token": "7140w9gefhslx3" + } } ``` +### Rate limits + +Check response headers for rate limit info: +- `x-rate-limit-limit`: Max requests in window +- `x-rate-limit-remaining`: Requests left +- `x-rate-limit-reset`: Unix timestamp when window resets + +Implement exponential backoff for 429 (Too Many Requests) responses. + +### Search operators + +Build queries with operators: + +| Operator | Example | Matches | +|----------|---------|---------| +| `from:` | `from:xdevelopers` | Posts by user | +| `to:` | `to:xdevelopers` | Replies to user | +| `#` | `#python` | Hashtag | +| `has:images` | `has:images` | Posts with images | +| `has:videos` | `has:videos` | Posts with videos | +| `has:links` | `has:links` | Posts with URLs | +| `lang:` | `lang:en` | Language | +| `is:retweet` | `-is:retweet` | Exclude retweets | +| `conversation_id:` | `conversation_id:123` | Thread ID | + ## Decision guidance -| Scenario | Choose | Why | -|:---------|:-------|:----| -| **Search recent vs. full-archive** | Recent if <7 days old | Recent is free; full-archive requires paid access | -| **Bearer token vs. OAuth** | Bearer for public data only | OAuth needed for user-context actions (post, like, DM) | -| **Filtered stream vs. polling** | Filtered stream for real-time | Stream is efficient; polling wastes rate limits | -| **Pagination vs. since_id** | Pagination for backfill | since_id for incremental updates (fewer API calls) | -| **Fields vs. expansions** | Fields for object's own data | Expansions for related objects (author, media, etc.) | +### When to use Bearer Token vs OAuth + +| Scenario | Use | +|----------|-----| +| Read-only public data, no user context needed | Bearer Token (app-only) | +| Need to post, like, follow on behalf of user | OAuth 1.0a or 2.0 | +| Building web app with user sign-in | OAuth 2.0 PKCE | +| Accessing private metrics or user data | OAuth (user context) | + +### When to use search vs filtered stream + +| Use case | Choose | +|----------|--------| +| Historical data, specific date range, one-time query | Search (recent or all) | +| Real-time monitoring, continuous updates | Filtered stream | +| Large volume, high throughput | Filtered stream | +| Specific past event, archive research | Full-archive search | + +### When to use fields vs expansions + +| Need | Use | +|------|-----| +| Additional data on primary object (post, user) | `fields` parameter | +| Related objects (post author, media, replies) | `expansions` parameter | +| Both primary and related data | Combine both | ## Workflow -1. **Set up credentials** - - Create app at console.x.com - - Generate Bearer Token (app-only) or OAuth credentials (user-context) - - Store securely; never commit to code - -2. **Choose authentication method** - - Public data only? Use Bearer Token - - Need user actions? Use OAuth 1.0a or 2.0 - - Check endpoint docs for required auth type - -3. **Build the request** - - Identify endpoint (search, lookup, manage, stream) - - Add required parameters (query, user_id, etc.) - - Request fields and expansions for needed data - - Set max_results and pagination_token if needed - -4. **Handle the response** - - Check HTTP status code (200 = success, 4xx/5xx = error) - - Parse `data` field for primary results - - Check `includes` for related objects - - Use `meta.next_token` for pagination - -5. **Manage rate limits** - - Monitor `x-rate-limit-remaining` header - - Implement exponential backoff for 429 errors - - Cache responses when possible - - Use streaming instead of polling for real-time data - -6. **Verify and test** - - Test with cURL or Postman first - - Check error responses match expected format - - Verify pagination works for large result sets - - Confirm fields/expansions return expected data +### Typical task: Search and analyze posts + +1. **Authenticate**: Get Bearer Token from Developer Console or generate OAuth tokens +2. **Build query**: Use search operators to construct query string (e.g., `from:xdevelopers lang:en`) +3. **Request fields**: Decide what data you need (`tweet.fields=created_at,public_metrics`) +4. **Make request**: Call `GET /2/tweets/search/recent` or `/2/tweets/search/all` +5. **Handle pagination**: Check `meta.next_token` in response, loop if needed +6. **Parse response**: Extract `data` array, handle `includes` for related objects +7. **Check errors**: Verify no errors in response, handle 429 rate limits with backoff + +### Typical task: Create and manage posts + +1. **Authenticate**: Use OAuth 1.0a or 2.0 (requires user context) +2. **Prepare content**: Validate text length (280 chars), prepare media if needed +3. **Upload media** (if needed): `POST /2/media/upload` with chunked upload +4. **Create post**: `POST /2/tweets` with text and media IDs +5. **Capture ID**: Save returned post ID for future reference +6. **Edit or delete**: Use `POST /2/tweets/{id}` to edit or `DELETE /2/tweets/{id}` to remove +7. **Verify**: Check response for success, handle validation errors + +### Typical task: Stream real-time posts + +1. **Authenticate**: Get Bearer Token +2. **Define rules**: Create filtering rules with operators (e.g., `from:xdevelopers`) +3. **Add rules**: `POST /2/tweets/search/stream/rules` with rule objects +4. **Connect stream**: `GET /2/tweets/search/stream` (keep connection open) +5. **Process events**: Parse JSON objects as they arrive +6. **Handle disconnections**: Implement reconnect logic with exponential backoff +7. **Manage rules**: Update or delete rules as needed with `POST` or `DELETE` ## Common gotchas -- **Missing Bearer Token**: Ensure token is in `Authorization: Bearer $TOKEN` header, not as query parameter -- **Forgetting expansions**: Requesting `author_id` field without `expansions=author_id` returns only the ID, not author details -- **Rate limit confusion**: Rate limits and billing are separate; you can hit rate limits without incurring costs -- **Protected accounts**: Posts from protected accounts only visible with user authorization; returns 403 otherwise +- **Bearer Token vs OAuth**: Bearer tokens are app-only; use OAuth for user-context operations (posting, following, etc.) +- **Field limits**: You cannot request subfields (e.g., `public_metrics.like_count`); request the parent field - **Pagination tokens expire**: Don't store tokens long-term; regenerate if needed -- **Query length limits**: Recent search = 512 chars, full-archive = 1,024 chars (4,096 for Enterprise) -- **Stream rules limit**: Filtered stream allows max 1,000 rules; hitting cap returns error -- **Deleted posts return 404**: Don't assume post exists; handle 404 gracefully -- **Partial errors in batch requests**: 200 response may include both `data` and `errors` array; check both -- **OAuth 2.0 callback URLs**: Must match exactly in Developer Console (including trailing slashes); use `http://127.0.0.1` for local dev, not `localhost` +- **Rate limits are per endpoint**: Different endpoints have different limits; check headers +- **Search recent is 7 days only**: Use full-archive search for older posts (requires enterprise access) +- **Streaming rules are global**: Rules apply to all connections for your app; manage carefully +- **Character counting**: Use `twitter-text` library for accurate 280-character validation +- **Media upload is chunked**: Large files require multiple append requests before finalization +- **Expansions return defaults only**: Combine with `fields` to get additional data on expanded objects +- **Response includes are optional**: Not all endpoints return `includes`; check API reference +- **429 errors require backoff**: Don't retry immediately; wait and use exponential backoff +- **Deleted posts return 404**: Attempting to fetch deleted posts fails; handle gracefully +- **DM endpoints require OAuth**: Direct messages always need user context, never app-only ## Verification checklist -Before submitting work: - -- [ ] Authentication credentials are correct and not hardcoded -- [ ] Endpoint URL matches API docs (base: `https://api.x.com/2/`) -- [ ] Required parameters are included (query, user_id, etc.) -- [ ] Field parameters use correct syntax (`tweet.fields=`, `user.fields=`, etc.) -- [ ] Expansions are paired with field parameters for related objects -- [ ] Error handling checks HTTP status code and `errors` array -- [ ] Rate limit headers are monitored (implement backoff for 429) -- [ ] Pagination logic handles `next_token` correctly +Before submitting work with X API: + +- [ ] Authentication method matches use case (Bearer vs OAuth) +- [ ] All required fields are requested (not relying on defaults) +- [ ] Pagination is handled (loop until no `next_token`) +- [ ] Error responses are checked (look for `errors` array) +- [ ] Rate limit headers are monitored (`x-rate-limit-remaining`) +- [ ] 429 responses trigger exponential backoff (not immediate retry) +- [ ] Search queries use correct operators for the data needed +- [ ] Expansions are paired with corresponding `fields` parameters +- [ ] Media uploads use chunked endpoint for large files +- [ ] Streaming rules are validated before adding +- [ ] User-context operations use OAuth, not Bearer Token - [ ] Response parsing handles both `data` and `includes` objects -- [ ] OAuth tokens are stored securely (env vars, not code) -- [ ] Tested with actual API (not just mock data) +- [ ] Timestamps are in ISO 8601 format (e.g., `2024-01-15T12:00:00.000Z`) +- [ ] Post text is validated for length and special characters +- [ ] DM recipient IDs are correct (not usernames) ## Resources -- **Comprehensive navigation**: https://docs.x.com/llms.txt — Full page-by-page listing for agent reference -- **Getting started**: https://docs.x.com/x-api/introduction — Overview and quick start -- **Authentication guide**: https://docs.x.com/fundamentals/authentication/overview — All auth methods explained -- **Rate limits reference**: https://docs.x.com/x-api/fundamentals/rate-limits — Per-endpoint limits and handling +**Comprehensive navigation**: https://docs.x.com/llms.txt — Full page-by-page listing for agent navigation + +**Critical documentation**: +1. [X API Introduction](https://docs.x.com/x-api/introduction) — Overview, pricing, key features +2. [Make Your First Request](https://docs.x.com/make-your-first-request) — Quick start with cURL examples +3. [Authentication Overview](https://docs.x.com/fundamentals/authentication/overview) — Auth methods and setup +4. [Fields & Expansions](https://docs.x.com/x-api/fundamentals/fields) — Customize response data +5. [Pagination](https://docs.x.com/x-api/fundamentals/pagination) — Handle large result sets +6. [Rate Limits](https://docs.x.com/x-api/fundamentals/rate-limits) — Understand request limits +7. [Search Operators](https://docs.x.com/x-api/posts/search/introduction) — Build search queries +8. [Filtered Stream](https://docs.x.com/x-api/posts/filtered-stream/introduction) — Real-time streaming +9. [Error Handling](https://docs.x.com/x-api/fundamentals/response-codes-and-errors) — Status codes and errors +10. [Python SDK](https://docs.x.com/xdks/python/overview) — Official Python library +11. [TypeScript SDK](https://docs.x.com/xdks/typescript/overview) — Official TypeScript library --- diff --git a/README.md b/README.md index 84b6bb4..c3a2596 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A Python CLI tool to sync your X (Twitter) bookmarks and liked posts to Raindrop ## Features - Sync X bookmarks and/or liked posts to Raindrop.io collections +- Optional: map X bookmark folders to Raindrop subcollections under the parent collection - Per-source configuration: separate collections, tags, link modes, and remove-from-X settings - Configurable link handling: - Use X post permalink @@ -284,6 +285,7 @@ remove_from_x = false skip_existing_links = true link_mode = "permalink" # permalink, first_external_url, or both both_behavior = "one_external_plus_note" # one_external_plus_note or two_raindrops +map_folders_to_subcollections = false # Map X bookmark folders to Raindrop child collections [sync.likes] enabled = false @@ -304,6 +306,13 @@ supported and are treated as bookmark settings for backward compatibility. Each source can be configured independently: different Raindrop collections, tags, link modes, and remove-from-X behavior. +When `map_folders_to_subcollections = true` on `[sync.bookmarks]`, X bookmark +folders are mirrored as Raindrop subcollections under the configured parent +collection. For example, bookmarks in an X folder named `Test` are stored in a +child collection also named `Test`. The child collection is created if it does +not already exist. Unfiled bookmarks stay in the parent collection. This option +requires a regular Raindrop collection (not All / Unsorted / Trash). + CLI flags override config when passed. Boolean flags are tri-state, so `--no-remove-from-x` and `--no-dry-run` clear values that are enabled in config. `log_level` controls stdlib/structlog verbosity for CLI commands. @@ -351,6 +360,8 @@ The tool stores data in the current working directory: **API Request Breakdown:** - Fetching bookmarks: 1 request per 100 bookmarks (paginated) - Fetching liked posts: 1 request per 100 likes (paginated) +- Listing bookmark folders: 1 request per 100 folders (when mapping folders) +- Listing bookmarks in a folder: 1 request per folder (when mapping folders) - Deleting a bookmark / unliking a post: 1 request per item **Rate Limit Behavior:** diff --git a/config.example.toml b/config.example.toml index f753f47..88dd647 100644 --- a/config.example.toml +++ b/config.example.toml @@ -37,6 +37,7 @@ remove_from_x = false skip_existing_links = true link_mode = "permalink" both_behavior = "one_external_plus_note" +map_folders_to_subcollections = false [sync.likes] enabled = false diff --git a/src/x2raindrop_cli/cli.py b/src/x2raindrop_cli/cli.py index 764e44f..6652a72 100644 --- a/src/x2raindrop_cli/cli.py +++ b/src/x2raindrop_cli/cli.py @@ -244,6 +244,13 @@ def sync( help="Skip links already present in the target Raindrop collection", ), ] = None, + map_folders_to_subcollections: Annotated[ + bool | None, + typer.Option( + "--map-folders-to-subcollections/--no-map-folders-to-subcollections", + help="Map X bookmark folders to Raindrop subcollections", + ), + ] = None, link_mode: Annotated[ LinkMode | None, typer.Option( @@ -295,6 +302,8 @@ def sync( settings.sync.bookmarks.skip_existing_links = skip_existing_links if settings.sync.likes.enabled: settings.sync.likes.skip_existing_links = skip_existing_links + if map_folders_to_subcollections is not None: + settings.sync.bookmarks.map_folders_to_subcollections = map_folders_to_subcollections if link_mode is not None: if settings.sync.bookmarks.enabled: settings.sync.bookmarks.link_mode = link_mode @@ -366,6 +375,10 @@ def sync( "Bookmarks Skip Existing Links", "Yes" if settings.sync.bookmarks.skip_existing_links else "No", ) + table.add_row( + "Map Folders to Subcollections", + "Yes" if settings.sync.bookmarks.map_folders_to_subcollections else "No", + ) table.add_row("Likes Enabled", "Yes" if settings.sync.likes.enabled else "No") if settings.sync.likes.enabled: table.add_row("Likes Collection ID", str(settings.sync.likes.collection_id)) @@ -816,6 +829,10 @@ def config_show( ) table.add_row("Bookmarks Link Mode", settings.sync.bookmarks.link_mode.value) table.add_row("Bookmarks Both Behavior", settings.sync.bookmarks.both_behavior.value) + table.add_row( + "Map Folders to Subcollections", + str(settings.sync.bookmarks.map_folders_to_subcollections), + ) table.add_row("Likes Enabled", str(settings.sync.likes.enabled)) table.add_row( "Likes Collection ID", diff --git a/src/x2raindrop_cli/config.py b/src/x2raindrop_cli/config.py index e18aa2b..2891b1a 100644 --- a/src/x2raindrop_cli/config.py +++ b/src/x2raindrop_cli/config.py @@ -197,6 +197,8 @@ class SourceSyncSettings(BaseModel): skip_existing_links: Whether to skip links that already exist in Raindrop. link_mode: How to determine the Raindrop link. both_behavior: Behavior when link_mode is 'both'. + map_folders_to_subcollections: For bookmarks, map X folders to Raindrop + child collections under the configured parent collection. """ enabled: bool = Field(True, description="Whether this source is synced") @@ -213,6 +215,12 @@ class SourceSyncSettings(BaseModel): BothBehavior.ONE_EXTERNAL_PLUS_NOTE, description="Behavior when link_mode is 'both'", ) + map_folders_to_subcollections: bool = Field( + False, + description=( + "Map X bookmark folders to Raindrop subcollections under the parent collection" + ), + ) @field_validator("tags", mode="before") @classmethod @@ -309,6 +317,16 @@ def validate_enabled_sources(self) -> None: ) if not self.any_enabled(): raise ValueError("At least one sync source (bookmarks or likes) must be enabled") + if ( + self.bookmarks.enabled + and self.bookmarks.map_folders_to_subcollections + and self.bookmarks.collection_id is not None + and self.bookmarks.collection_id <= 0 + ): + raise ValueError( + "map_folders_to_subcollections requires a regular Raindrop collection " + "(not 0/All, -1/Unsorted, or -99/Trash)" + ) class Settings(BaseSettings): @@ -448,6 +466,7 @@ def create_default_config(path: Path | None = None) -> Path: "skip_existing_links": True, "link_mode": "permalink", "both_behavior": "one_external_plus_note", + "map_folders_to_subcollections": False, }, "likes": { "enabled": False, diff --git a/src/x2raindrop_cli/models.py b/src/x2raindrop_cli/models.py index d5ff18c..6b0aa7f 100644 --- a/src/x2raindrop_cli/models.py +++ b/src/x2raindrop_cli/models.py @@ -51,6 +51,20 @@ class BothBehavior(StrEnum): TWO_RAINDROPS = "two_raindrops" +class BookmarkFolder(BaseModel): + """Represents an X bookmark folder. + + Attributes: + id: Unique folder identifier from the X API. + name: Display name of the folder. + """ + + model_config = ConfigDict(frozen=True) + + id: str = Field(..., description="Unique bookmark folder identifier") + name: str = Field(..., description="Bookmark folder display name") + + class BookmarkItem(BaseModel): """Represents a bookmark from X (Twitter). @@ -62,6 +76,7 @@ class BookmarkItem(BaseModel): created_at: When the tweet was created. permalink: The direct URL to the tweet. external_urls: List of external URLs found in the tweet. + folder_name: X bookmark folder name, if the post is filed in a folder. """ model_config = ConfigDict(frozen=True) @@ -73,6 +88,10 @@ class BookmarkItem(BaseModel): created_at: datetime | None = Field(None, description="Tweet creation timestamp") permalink: str = Field(..., description="Direct URL to the tweet") external_urls: list[str] = Field(default_factory=list, description="External URLs in tweet") + folder_name: str | None = Field( + None, + description="X bookmark folder name when the post is filed in a folder", + ) def get_title(self) -> str: """Generate a title for the bookmark. diff --git a/src/x2raindrop_cli/raindrop/client.py b/src/x2raindrop_cli/raindrop/client.py index 63c6b13..421c3f2 100644 --- a/src/x2raindrop_cli/raindrop/client.py +++ b/src/x2raindrop_cli/raindrop/client.py @@ -125,6 +125,18 @@ def list_collection_links(self, collection_id: int) -> set[str]: """ ... + def create_collection(self, title: str, parent_id: int) -> RaindropCollection: + """Create a child collection under a parent collection. + + Args: + title: Collection title. + parent_id: Parent collection ID. + + Returns: + Created collection details. + """ + ... + def check_link_exists(self, link: str, collection_id: int | None = None) -> bool: """Check if a link already exists in Raindrop. @@ -225,6 +237,52 @@ def list_collections(self) -> list[RaindropCollection]: logger.debug("Listed collections", count=len(collections)) return collections + def create_collection(self, title: str, parent_id: int) -> RaindropCollection: + """Create a child collection under a parent collection. + + Uses the REST create endpoint so ``parent.$id`` is sent in the + documented shape (the python-raindropio helper sends a bare integer). + + Args: + title: Collection title. + parent_id: Parent collection ID. + + Returns: + Created collection details. + + Raises: + ValueError: If the API response is missing a collection ID. + """ + create_url = "https://api.raindrop.io/rest/v1/collection" + payload = {"title": title, "parent": {"$id": parent_id}} + logger.debug("Creating collection", title=title, parent_id=parent_id) + response = self.api.post(create_url, json=payload) + response.raise_for_status() + response_data = response.json() + item = response_data.get("item", {}) + if not isinstance(item, dict): + raise ValueError("Unexpected collection create response: item is not an object") + + item_id = item.get("_id") + if item_id is None: + item_id = item.get("id") + if item_id is None: + raise ValueError("Unexpected collection create response: missing collection id") + + created = RaindropCollection( + id=int(item_id), + title=str(item.get("title") or title), + count=int(item.get("count") or 0), + parent_id=parent_id, + ) + logger.info( + "Created collection", + collection_id=created.id, + title=created.title, + parent_id=parent_id, + ) + return created + def get_collection_by_title(self, title: str) -> RaindropCollection | None: """Find a collection by its title. diff --git a/src/x2raindrop_cli/sync/service.py b/src/x2raindrop_cli/sync/service.py index f0b2168..ed3f4dd 100644 --- a/src/x2raindrop_cli/sync/service.py +++ b/src/x2raindrop_cli/sync/service.py @@ -80,17 +80,20 @@ def resolve_links( def create_raindrop_requests( bookmark: BookmarkItem, settings: SourceSyncSettings, + collection_id: int | None = None, ) -> list[RaindropCreateRequest]: """Create Raindrop request(s) from a bookmark or liked post. Args: bookmark: X post to convert. settings: Source-specific sync settings. + collection_id: Optional collection override (e.g. a folder subcollection). Returns: List of RaindropCreateRequest objects. """ - if settings.collection_id is None: + target_collection_id = collection_id if collection_id is not None else settings.collection_id + if target_collection_id is None: raise ValueError("collection_id must be set in sync settings") links = resolve_links(bookmark, settings.link_mode, settings.both_behavior) @@ -106,7 +109,7 @@ def create_raindrop_requests( title=title, excerpt=excerpt, tags=list(settings.tags), - collection_id=settings.collection_id, + collection_id=target_collection_id, note=note, source_tweet_id=bookmark.tweet_id, ) @@ -182,6 +185,24 @@ def _resolve_collection_ids(self) -> None: ) source_settings.collection_id = collection.id + self._validate_folder_mapping() + + def _validate_folder_mapping(self) -> None: + """Require a regular parent collection when mapping X folders. + + Raises: + ValueError: If folder mapping is enabled without a usable parent. + """ + bookmarks = self.settings.bookmarks + if not bookmarks.enabled or not bookmarks.map_folders_to_subcollections: + return + collection_id = bookmarks.collection_id + if collection_id is None or collection_id <= 0: + raise ValueError( + "map_folders_to_subcollections requires a regular Raindrop collection " + "(not 0/All, -1/Unsorted, or -99/Trash)" + ) + def sync( self, progress_callback: ProgressCallback | None = None, @@ -207,6 +228,7 @@ def sync( fetch_items=self.x_client.get_bookmarks, remove_from_x=self.x_client.delete_bookmark, progress_callback=progress_callback, + map_folders=self.settings.bookmarks.map_folders_to_subcollections, ) if self.settings.likes.enabled: @@ -256,6 +278,7 @@ def _sync_source( fetch_items: Callable[[], Iterator[BookmarkItem]], remove_from_x: Callable[[str], bool], progress_callback: ProgressCallback | None, + map_folders: bool = False, ) -> SourceSyncResult: """Sync one X post source to Raindrop.""" source_result = SourceSyncResult() @@ -263,6 +286,10 @@ def _sync_source( logger.info(f"Fetching {source_label} from X...") posts = list(fetch_items()) + if map_folders: + if progress_callback: + progress_callback(0, len(posts), "Mapping X bookmark folders...") + posts = self._apply_bookmark_folders(posts) source_result.total = len(posts) if progress_callback: @@ -270,22 +297,17 @@ def _sync_source( logger.info(f"Found {source_label}", count=source_result.total) - existing_links: set[str] | None = None - if source_settings.skip_existing_links: - if source_settings.collection_id is None: - raise ValueError("collection_id must be set when skip_existing_links is enabled") - if progress_callback: - progress_callback( - 0, - source_result.total, - f"Loading existing Raindrop links for {source_label}...", - ) - existing_links = self._existing_links_for(source_settings.collection_id) - logger.info( - "Using Raindrop link inventory", - source=source.value, - collection_id=source_settings.collection_id, - existing_count=len(existing_links), + folder_collection_ids = self._resolve_folder_subcollections( + posts=posts, + source_settings=source_settings, + map_folders=map_folders, + ) + + if source_settings.skip_existing_links and progress_callback: + progress_callback( + 0, + source_result.total, + f"Loading existing Raindrop links for {source_label}...", ) pending_requests: list[tuple[int, BookmarkItem, list[RaindropCreateRequest]]] = [] @@ -294,6 +316,7 @@ def _sync_source( log = logger.bind( tweet_id=post.tweet_id, source=source.value, + folder=post.folder_name, progress=f"{idx + 1}/{source_result.total}", ) @@ -308,15 +331,39 @@ def _sync_source( ) continue + target_collection_id = self._target_collection_id( + post=post, + source_settings=source_settings, + folder_collection_ids=folder_collection_ids, + ) + if target_collection_id is None: + log.info( + "Dry run - would create subcollection and raindrop(s)", + folder=post.folder_name, + ) + source_result.newly_synced += 1 + if progress_callback: + progress_callback( + idx + 1, + source_result.total, + f"Dry run (would create folder): {post.folder_name}", + ) + continue + try: - requests = create_raindrop_requests(post, source_settings) + requests = create_raindrop_requests( + post, + source_settings, + collection_id=target_collection_id, + ) except ValueError as error: log.error("Failed to create request", error=str(error)) source_result.failed += 1 source_result.add_error(f"[{post.tweet_id}] {error}") continue - if existing_links is not None: + if source_settings.skip_existing_links: + existing_links = self._existing_links_for(target_collection_id) requests = self._filter_existing_requests( requests=requests, existing_links=existing_links, @@ -342,11 +389,14 @@ def _sync_source( log.info( "Dry run - would create raindrop(s)", links=[request.link for request in requests], + collection_id=target_collection_id, + folder=post.folder_name, ) - # Keep dry-run inventory consistent so later items aren't double-counted. - if existing_links is not None: - for request in requests: - existing_links.add(normalize_link(request.link)) + if source_settings.skip_existing_links: + self._remember_created_links( + target_collection_id, + [request.link for request in requests], + ) source_result.newly_synced += 1 if progress_callback: progress_callback( @@ -369,6 +419,127 @@ def _sync_source( return source_result + def _apply_bookmark_folders(self, posts: list[BookmarkItem]) -> list[BookmarkItem]: + """Annotate bookmarks with their X folder names. + + Posts that appear only in a folder (not in the main bookmarks list) + are added as permalink-only items so folder contents are not dropped. + """ + folders = self.x_client.get_bookmark_folders() + tweet_to_folder: dict[str, str] = {} + for folder in folders: + for tweet_id in self.x_client.get_bookmark_ids_in_folder(folder.id): + existing_folder = tweet_to_folder.get(tweet_id) + if existing_folder and existing_folder != folder.name: + logger.warning( + "Bookmark is in multiple X folders; using the first folder", + tweet_id=tweet_id, + first_folder=existing_folder, + ignored_folder=folder.name, + ) + continue + tweet_to_folder[tweet_id] = folder.name + + annotated = [ + post.model_copy(update={"folder_name": tweet_to_folder.get(post.tweet_id)}) + for post in posts + ] + known_ids = {post.tweet_id for post in annotated} + for tweet_id, folder_name in tweet_to_folder.items(): + if tweet_id in known_ids: + continue + annotated.append( + BookmarkItem( + tweet_id=tweet_id, + text="", + permalink=f"https://x.com/i/status/{tweet_id}", + folder_name=folder_name, + ) + ) + + logger.info( + "Mapped X bookmark folders", + folder_count=len(folders), + filed_count=sum(1 for post in annotated if post.folder_name), + unfiled_count=sum(1 for post in annotated if not post.folder_name), + ) + return annotated + + def _resolve_folder_subcollections( + self, + posts: list[BookmarkItem], + source_settings: SourceSyncSettings, + map_folders: bool, + ) -> dict[str, int]: + """Find or create Raindrop subcollections for X bookmark folders. + + Returns: + Mapping of folder display name to Raindrop collection ID. + """ + if not map_folders: + return {} + if source_settings.collection_id is None: + raise ValueError("collection_id must be set when mapping folders") + + parent_id = source_settings.collection_id + folder_names = sorted({post.folder_name for post in posts if post.folder_name}) + if not folder_names: + return {} + + collections = self.raindrop_client.list_collections() + children_by_title = { + collection.title.lower(): collection + for collection in collections + if collection.parent_id == parent_id + } + + mapping: dict[str, int] = {} + for folder_name in folder_names: + existing = children_by_title.get(folder_name.lower()) + if existing is not None: + mapping[folder_name] = existing.id + logger.info( + "Using existing Raindrop subcollection for X folder", + folder=folder_name, + collection_id=existing.id, + parent_id=parent_id, + ) + continue + if self.settings.dry_run: + logger.info( + "Dry run - would create Raindrop subcollection", + folder=folder_name, + parent_id=parent_id, + ) + continue + created = self.raindrop_client.create_collection(folder_name, parent_id) + children_by_title[folder_name.lower()] = created + mapping[folder_name] = created.id + + return mapping + + def _target_collection_id( + self, + post: BookmarkItem, + source_settings: SourceSyncSettings, + folder_collection_ids: dict[str, int], + ) -> int | None: + """Return the Raindrop collection ID for a post. + + Returns: + Collection ID, or None when a dry-run would still need to create + the destination subcollection. + """ + if post.folder_name: + mapped_id = folder_collection_ids.get(post.folder_name) + if mapped_id is not None: + return mapped_id + if self.settings.dry_run: + return None + if source_settings.collection_id is None: + raise ValueError("collection_id must be set in sync settings") + return source_settings.collection_id + def _filter_existing_requests( self, requests: list[RaindropCreateRequest], @@ -445,6 +616,7 @@ def _finalize_batched_sync( source_settings=source_settings, idx=idx, post=post, + post_requests=post_requests, created_links=post_links, source_result=source_result, remove_from_x=remove_from_x, @@ -491,6 +663,7 @@ def _sync_pending_posts_individually( source_settings=source_settings, idx=idx, post=post, + post_requests=post_requests, created_links=created_links, source_result=source_result, remove_from_x=remove_from_x, @@ -503,6 +676,7 @@ def _mark_post_synced( source_settings: SourceSyncSettings, idx: int, post: BookmarkItem, + post_requests: list[RaindropCreateRequest], created_links: list[str], source_result: SourceSyncResult, remove_from_x: Callable[[str], bool], @@ -537,8 +711,8 @@ def _mark_post_synced( deleted_from_x, source=source, ) - if source_settings.collection_id is not None: - self._remember_created_links(source_settings.collection_id, created_links) + for request, link in zip(post_requests, created_links, strict=True): + self._remember_created_links(request.collection_id, [link]) source_result.newly_synced += 1 if progress_callback: progress_callback( diff --git a/src/x2raindrop_cli/x/client.py b/src/x2raindrop_cli/x/client.py index e4e68aa..812831b 100644 --- a/src/x2raindrop_cli/x/client.py +++ b/src/x2raindrop_cli/x/client.py @@ -16,7 +16,7 @@ import structlog from xdk import Client as XdkClient -from x2raindrop_cli.models import BookmarkItem +from x2raindrop_cli.models import BookmarkFolder, BookmarkItem from x2raindrop_cli.x.auth_pkce import OAuth2Token, refresh_access_token logger = structlog.get_logger(__name__) @@ -53,6 +53,25 @@ def get_liked_posts(self, max_results: int | None = None) -> Iterator[BookmarkIt """ ... + def get_bookmark_folders(self) -> list[BookmarkFolder]: + """List X bookmark folders for the authenticated user. + + Returns: + Bookmark folders with IDs and display names. + """ + ... + + def get_bookmark_ids_in_folder(self, folder_id: str) -> list[str]: + """List tweet IDs stored in an X bookmark folder. + + Args: + folder_id: Bookmark folder ID. + + Returns: + Tweet IDs in the folder. + """ + ... + def delete_bookmark(self, tweet_id: str) -> bool: """Remove a bookmark from X. @@ -221,6 +240,100 @@ def get_liked_posts(self, max_results: int | None = None) -> Iterator[BookmarkIt max_results=max_results, ) + def get_bookmark_folders(self) -> list[BookmarkFolder]: + """List X bookmark folders for the authenticated user. + + Returns: + Bookmark folders with IDs and display names. + """ + self._ensure_fresh_token() + user_id = self.get_authenticated_user_id() + folders: list[BookmarkFolder] = [] + page_count = 0 + + for page in self._x_client.users.get_bookmark_folders( + id=user_id, + max_results=MAX_RESULTS_PER_PAGE, + ): + page_count += 1 + self._request_count += 1 + data = self._model_to_dict(page) + items = data.get("data") or [] + if not isinstance(items, list) or not items: + break + for item in items: + folder = self._parse_folder(item) + if folder is not None: + folders.append(folder) + + logger.info( + "Fetched bookmark folders", + total=len(folders), + pages_fetched=page_count, + api_requests=self._request_count, + ) + return folders + + def get_bookmark_ids_in_folder(self, folder_id: str) -> list[str]: + """List tweet IDs stored in an X bookmark folder. + + Args: + folder_id: Bookmark folder ID. + + Returns: + Tweet IDs in the folder. + """ + self._ensure_fresh_token() + user_id = self.get_authenticated_user_id() + tweet_ids: list[str] = [] + page_count = 0 + + for page in self._x_client.users.get_bookmarks_by_folder_id( + id=user_id, + folder_id=folder_id, + max_results=MAX_RESULTS_PER_PAGE, + ): + page_count += 1 + self._request_count += 1 + data = self._model_to_dict(page) + items = data.get("data") or [] + if not isinstance(items, list) or not items: + break + for item in items: + tweet_id = self._extract_item_id(item) + if tweet_id: + tweet_ids.append(tweet_id) + + logger.info( + "Fetched bookmarks in folder", + folder_id=folder_id, + total=len(tweet_ids), + pages_fetched=page_count, + api_requests=self._request_count, + ) + return tweet_ids + + def _parse_folder(self, item: Any) -> BookmarkFolder | None: + """Parse a bookmark folder object from an XDK response item.""" + data = self._model_to_dict(item) if not isinstance(item, dict) else item + folder_id = data.get("id") + name = data.get("name") + if folder_id is None or not isinstance(name, str) or not name.strip(): + return None + return BookmarkFolder(id=str(folder_id), name=name.strip()) + + def _extract_item_id(self, item: Any) -> str | None: + """Extract an ID string from a folder-post response item.""" + if isinstance(item, dict): + raw_id = item.get("id") + else: + raw_id = getattr(item, "id", None) + if raw_id is None: + raw_id = self._model_to_dict(item).get("id") + if raw_id is None: + return None + return str(raw_id) + def _fetch_posts( self, fetch_pages: Any, diff --git a/tests/fakes.py b/tests/fakes.py index e9784d6..c1830b5 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -7,7 +7,7 @@ from collections.abc import Iterator -from x2raindrop_cli.models import BookmarkItem, RaindropCreateRequest +from x2raindrop_cli.models import BookmarkFolder, BookmarkItem, RaindropCreateRequest from x2raindrop_cli.raindrop.client import CreatedRaindrop, RaindropCollection, normalize_link @@ -23,6 +23,8 @@ def __init__( bookmarks: list[BookmarkItem] | None = None, liked_posts: list[BookmarkItem] | None = None, user_id: str = "123456789", + bookmark_folders: list[BookmarkFolder] | None = None, + folder_bookmark_ids: dict[str, list[str]] | None = None, ) -> None: """Initialize mock client. @@ -30,12 +32,16 @@ def __init__( bookmarks: List of bookmarks to return. liked_posts: List of liked posts to return. user_id: Fake user ID to return. + bookmark_folders: X bookmark folders to return. + folder_bookmark_ids: Mapping of folder ID to tweet IDs in that folder. """ self.bookmarks = bookmarks or [] self.liked_posts = liked_posts or [] self.user_id = user_id self.deleted_tweet_ids: list[str] = [] self.unliked_tweet_ids: list[str] = [] + self.bookmark_folders = bookmark_folders or [] + self.folder_bookmark_ids = folder_bookmark_ids or {} def get_authenticated_user_id(self) -> str: """Get the mock user ID.""" @@ -49,6 +55,14 @@ def get_liked_posts(self, max_results: int | None = None) -> Iterator[BookmarkIt """Yield pre-configured liked posts.""" yield from self._yield_items(self.liked_posts, max_results) + def get_bookmark_folders(self) -> list[BookmarkFolder]: + """Return pre-configured bookmark folders.""" + return list(self.bookmark_folders) + + def get_bookmark_ids_in_folder(self, folder_id: str) -> list[str]: + """Return pre-configured tweet IDs for a bookmark folder.""" + return list(self.folder_bookmark_ids.get(folder_id, [])) + def _yield_items( self, items: list[BookmarkItem], @@ -95,7 +109,9 @@ def __init__( ] self.created_raindrops: list[RaindropCreateRequest] = [] self.batch_create_calls: list[list[RaindropCreateRequest]] = [] + self.created_collections: list[RaindropCollection] = [] self._next_id = 1 + self._next_collection_id = 90000 self.existing_links = {normalize_link(link) for link in existing_links or []} self.list_collection_links_calls: list[int] = [] @@ -128,12 +144,26 @@ def create_raindrops(self, requests: list[RaindropCreateRequest]) -> list[Create self.batch_create_calls.append(list(requests)) return [self.create_raindrop(request) for request in requests] + def create_collection(self, title: str, parent_id: int) -> RaindropCollection: + """Create and track a child collection.""" + collection = RaindropCollection( + id=self._next_collection_id, + title=title, + count=0, + parent_id=parent_id, + ) + self._next_collection_id += 1 + self.collections.append(collection) + self.created_collections.append(collection) + return collection + def list_collection_links(self, collection_id: int) -> set[str]: """Return configured existing links plus links created in this session.""" self.list_collection_links_calls.append(collection_id) links = set(self.existing_links) for request in self.created_raindrops: - links.add(normalize_link(request.link)) + if request.collection_id == collection_id: + links.add(normalize_link(request.link)) return links def check_link_exists(self, link: str, collection_id: int | None = None) -> bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8842fa5..a3e339b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -155,6 +155,44 @@ def fake_sync(self: Any, progress_callback: Any = None) -> SyncResult: assert result.exit_code == 0, result.stdout assert captured["remove_from_x"] is True + def test_map_folders_to_subcollections_sets_true(self, temp_dir: Path) -> None: + """Test --map-folders-to-subcollections overrides config false.""" + config_path = temp_dir / "config.toml" + create_default_config(config_path) + text = config_path.read_text() + text = text.replace('token = "YOUR_RAINDROP_TOKEN"', 'token = "test-raindrop-token"') + text = text.replace( + 'collection_title = ""\ntags = [\n "x-bookmark"', + 'collection_id = 12345\ncollection_title = ""\ntags = [\n "x-bookmark"', + 1, + ) + config_path.write_text(text) + captured: dict[str, Any] = {} + + def fake_sync(self: Any, progress_callback: Any = None) -> SyncResult: + del progress_callback + captured["map_folders"] = self.settings.bookmarks.map_folders_to_subcollections + return SyncResult() + + with ( + patch("x2raindrop_cli.cli._get_x_token", return_value=MagicMock()), + patch("x2raindrop_cli.cli.XClient") as mock_x_cls, + patch("x2raindrop_cli.cli.RaindropClient") as mock_rd_cls, + patch("x2raindrop_cli.cli.SyncService.sync", fake_sync), + ): + mock_x = MagicMock() + mock_x.request_count = 0 + mock_x_cls.return_value = mock_x + mock_rd_cls.return_value = MagicMock() + + result = runner.invoke( + app, + ["sync", "--config", str(config_path), "--map-folders-to-subcollections"], + ) + + assert result.exit_code == 0, result.stdout + assert captured["map_folders"] is True + def test_sync_requires_raindrop_token(self, temp_dir: Path) -> None: """Test sync fails when Raindrop token is missing.""" config_path = temp_dir / "config.toml" diff --git a/tests/test_config.py b/tests/test_config.py index cb60daa..76c17d3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -161,6 +161,7 @@ def test_default_values(self) -> None: assert settings.skip_existing_links is True assert settings.link_mode == LinkMode.PERMALINK assert settings.both_behavior == BothBehavior.ONE_EXTERNAL_PLUS_NOTE + assert settings.map_folders_to_subcollections is False def test_parse_tags_from_string(self) -> None: """Test parsing tags from comma-separated string.""" @@ -261,6 +262,20 @@ def test_validate_enabled_sources_allows_collection_title(self) -> None: settings.validate_enabled_sources() + def test_validate_folder_mapping_rejects_unsorted(self) -> None: + """Test folder mapping rejects Unsorted as the parent collection.""" + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=-1, + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + + with pytest.raises(ValueError, match="map_folders_to_subcollections"): + settings.validate_enabled_sources() + class TestRaindropSettings: """Tests for Raindrop settings.""" diff --git a/tests/test_models.py b/tests/test_models.py index b828a46..5436396 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -10,6 +10,7 @@ import pytest from x2raindrop_cli.models import ( + BookmarkFolder, BookmarkItem, BothBehavior, LinkMode, @@ -74,6 +75,7 @@ def test_create_with_all_fields(self) -> None: assert bookmark.author_name == "Test User" assert bookmark.permalink == "https://x.com/testuser/status/12345" assert bookmark.external_urls == ["https://example.com"] + assert bookmark.folder_name is None def test_create_minimal(self) -> None: """Test creating BookmarkItem with minimal fields.""" @@ -123,6 +125,28 @@ def test_immutable(self, sample_bookmark: BookmarkItem) -> None: with pytest.raises(ValidationError): sample_bookmark.tweet_id = "new_id" # type: ignore + def test_folder_name_optional(self) -> None: + """Test folder_name can be set on a bookmark.""" + bookmark = BookmarkItem( + tweet_id="12345", + text="Filed tweet", + permalink="https://x.com/i/status/12345", + folder_name="Test", + ) + + assert bookmark.folder_name == "Test" + + +class TestBookmarkFolder: + """Tests for BookmarkFolder model.""" + + def test_create(self) -> None: + """Test creating a bookmark folder.""" + folder = BookmarkFolder(id="folder-1", name="Test") + + assert folder.id == "folder-1" + assert folder.name == "Test" + class TestRaindropCreateRequest: """Tests for RaindropCreateRequest model.""" diff --git a/tests/test_raindrop_client.py b/tests/test_raindrop_client.py index 392cd20..96e8ec4 100644 --- a/tests/test_raindrop_client.py +++ b/tests/test_raindrop_client.py @@ -206,6 +206,21 @@ def test_list_collection_links_includes_created(self) -> None: assert "https://created.example.com" in links +class TestMockRaindropCreateCollection: + """Tests for mock collection creation.""" + + def test_create_collection_appends_child(self) -> None: + """Test create_collection tracks a nested collection.""" + client = MockRaindropClient(collections=[]) + + created = client.create_collection("Test", 65746356) + + assert created.title == "Test" + assert created.parent_id == 65746356 + assert created in client.collections + assert created in client.created_collections + + class TestRaindropCreateRequest: """Tests for RaindropCreateRequest model.""" diff --git a/tests/test_sync_service.py b/tests/test_sync_service.py index 76b72c5..b8433f4 100644 --- a/tests/test_sync_service.py +++ b/tests/test_sync_service.py @@ -163,6 +163,18 @@ def test_raises_without_collection_id(self, sample_bookmark: BookmarkItem) -> No with pytest.raises(ValueError, match="collection_id must be set"): create_raindrop_requests(sample_bookmark, settings) + def test_collection_id_override( + self, sample_bookmark: BookmarkItem, sync_settings: SyncSettings + ) -> None: + """Test an explicit collection ID overrides source settings.""" + requests = create_raindrop_requests( + sample_bookmark, + sync_settings.bookmarks, + collection_id=999, + ) + + assert requests[0].collection_id == 999 + class TestSyncService: """Tests for SyncService orchestration.""" @@ -793,3 +805,242 @@ def test_missing_collection_title_raises( with pytest.raises(ValueError, match="Does Not Exist"): service.sync() + + +class TestMapFoldersToSubcollections: + """Tests for mapping X bookmark folders to Raindrop subcollections.""" + + def test_uses_existing_subcollection( + self, + sample_bookmarks: list[BookmarkItem], + in_memory_state: InMemoryState, + ) -> None: + """Test bookmarks in an X folder go to a matching Raindrop child collection.""" + from x2raindrop_cli.models import BookmarkFolder + from x2raindrop_cli.raindrop.client import RaindropCollection + + filed = sample_bookmarks[0] + unfiled = sample_bookmarks[1] + x_client = MockXClient( + bookmarks=[filed, unfiled], + bookmark_folders=[BookmarkFolder(id="folder-test", name="Test")], + folder_bookmark_ids={"folder-test": [filed.tweet_id]}, + ) + raindrop_client = MockRaindropClient( + collections=[ + RaindropCollection(id=65746356, title="x.com", count=0), + RaindropCollection(id=111, title="Test", count=0, parent_id=65746356), + ] + ) + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=65746356, + tags=["x-bookmark"], + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + service = SyncService( + x_client=x_client, + raindrop_client=raindrop_client, + state=in_memory_state, + settings=settings, + ) + + result = service.sync() + + assert result.bookmarks.newly_synced == 2 + assert raindrop_client.created_collections == [] + by_tweet = { + request.source_tweet_id: request for request in raindrop_client.created_raindrops + } + assert by_tweet[filed.tweet_id].collection_id == 111 + assert by_tweet[unfiled.tweet_id].collection_id == 65746356 + + def test_ignores_same_named_collection_outside_parent( + self, + sample_bookmarks: list[BookmarkItem], + in_memory_state: InMemoryState, + ) -> None: + """Test a same-named collection is only reused when it is a child.""" + from x2raindrop_cli.models import BookmarkFolder + from x2raindrop_cli.raindrop.client import RaindropCollection + + filed = sample_bookmarks[0] + x_client = MockXClient( + bookmarks=[filed], + bookmark_folders=[BookmarkFolder(id="folder-test", name="Test")], + folder_bookmark_ids={"folder-test": [filed.tweet_id]}, + ) + raindrop_client = MockRaindropClient( + collections=[ + RaindropCollection(id=65746356, title="x.com", count=0), + RaindropCollection(id=222, title="Test", count=0, parent_id=None), + ] + ) + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=65746356, + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + service = SyncService( + x_client=x_client, + raindrop_client=raindrop_client, + state=in_memory_state, + settings=settings, + ) + + result = service.sync() + + assert result.bookmarks.newly_synced == 1 + assert len(raindrop_client.created_collections) == 1 + assert raindrop_client.created_collections[0].parent_id == 65746356 + assert raindrop_client.created_raindrops[0].collection_id != 222 + + def test_creates_missing_subcollection( + self, + sample_bookmarks: list[BookmarkItem], + in_memory_state: InMemoryState, + ) -> None: + """Test a missing Raindrop child collection is created for the X folder.""" + from x2raindrop_cli.models import BookmarkFolder + from x2raindrop_cli.raindrop.client import RaindropCollection + + filed = sample_bookmarks[0] + x_client = MockXClient( + bookmarks=[filed], + bookmark_folders=[BookmarkFolder(id="folder-test", name="Test")], + folder_bookmark_ids={"folder-test": [filed.tweet_id]}, + ) + raindrop_client = MockRaindropClient( + collections=[RaindropCollection(id=65746356, title="x.com", count=0)] + ) + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=65746356, + tags=["x-bookmark"], + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + service = SyncService( + x_client=x_client, + raindrop_client=raindrop_client, + state=in_memory_state, + settings=settings, + ) + + result = service.sync() + + assert result.bookmarks.newly_synced == 1 + assert len(raindrop_client.created_collections) == 1 + created = raindrop_client.created_collections[0] + assert created.title == "Test" + assert created.parent_id == 65746356 + assert raindrop_client.created_raindrops[0].collection_id == created.id + + def test_dry_run_does_not_create_subcollection( + self, + sample_bookmarks: list[BookmarkItem], + in_memory_state: InMemoryState, + ) -> None: + """Test dry-run does not create missing Raindrop subcollections.""" + from x2raindrop_cli.models import BookmarkFolder + from x2raindrop_cli.raindrop.client import RaindropCollection + + filed = sample_bookmarks[0] + x_client = MockXClient( + bookmarks=[filed], + bookmark_folders=[BookmarkFolder(id="folder-test", name="Test")], + folder_bookmark_ids={"folder-test": [filed.tweet_id]}, + ) + raindrop_client = MockRaindropClient( + collections=[RaindropCollection(id=65746356, title="x.com", count=0)] + ) + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=65746356, + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + dry_run=True, + ) + service = SyncService( + x_client=x_client, + raindrop_client=raindrop_client, + state=in_memory_state, + settings=settings, + ) + + result = service.sync() + + assert result.bookmarks.newly_synced == 1 + assert raindrop_client.created_collections == [] + assert raindrop_client.created_raindrops == [] + + def test_includes_folder_only_bookmarks( + self, + in_memory_state: InMemoryState, + ) -> None: + """Test tweets present only in a folder are still synced.""" + from x2raindrop_cli.models import BookmarkFolder + from x2raindrop_cli.raindrop.client import RaindropCollection + + x_client = MockXClient( + bookmarks=[], + bookmark_folders=[BookmarkFolder(id="folder-test", name="Test")], + folder_bookmark_ids={"folder-test": ["9999999999"]}, + ) + raindrop_client = MockRaindropClient( + collections=[ + RaindropCollection(id=65746356, title="x.com", count=0), + RaindropCollection(id=111, title="Test", count=0, parent_id=65746356), + ] + ) + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=65746356, + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + service = SyncService( + x_client=x_client, + raindrop_client=raindrop_client, + state=in_memory_state, + settings=settings, + ) + + result = service.sync() + + assert result.bookmarks.total == 1 + assert result.bookmarks.newly_synced == 1 + assert raindrop_client.created_raindrops[0].collection_id == 111 + assert raindrop_client.created_raindrops[0].source_tweet_id == "9999999999" + + def test_rejects_unsorted_parent_collection(self, in_memory_state: InMemoryState) -> None: + """Test folder mapping requires a regular Raindrop collection.""" + settings = SyncSettings( + bookmarks=SourceSyncSettings( + enabled=True, + collection_id=-1, + map_folders_to_subcollections=True, + ), + likes=SourceSyncSettings(enabled=False), + ) + service = SyncService( + x_client=MockXClient(), + raindrop_client=MockRaindropClient(), + state=in_memory_state, + settings=settings, + ) + + with pytest.raises(ValueError, match="map_folders_to_subcollections"): + service.sync() diff --git a/tests/test_x_client.py b/tests/test_x_client.py index 5e40246..bf7e55d 100644 --- a/tests/test_x_client.py +++ b/tests/test_x_client.py @@ -89,6 +89,25 @@ def test_get_liked_posts_returns_configured(self, sample_bookmarks: list[Bookmar assert len(liked) == 3 assert liked[0].tweet_id == "1111111111" + def test_get_bookmark_folders_returns_configured(self) -> None: + """Test get_bookmark_folders returns configured folders.""" + from x2raindrop_cli.models import BookmarkFolder + + folders = [BookmarkFolder(id="folder-1", name="Test")] + client = MockXClient(bookmark_folders=folders) + + result = client.get_bookmark_folders() + + assert result == folders + + def test_get_bookmark_ids_in_folder(self) -> None: + """Test get_bookmark_ids_in_folder returns configured tweet IDs.""" + client = MockXClient(folder_bookmark_ids={"folder-1": ["111", "222"]}) + + result = client.get_bookmark_ids_in_folder("folder-1") + + assert result == ["111", "222"] + class TestXClientParsing: """Tests for XClient tweet parsing helpers.""" @@ -170,6 +189,34 @@ def test_extract_external_urls_handles_null_entities(self) -> None: finally: client.close() + def test_parse_folder_requires_name(self) -> None: + """Test folder parsing skips items without a name.""" + from datetime import datetime, timedelta + + from x2raindrop_cli.x.auth_pkce import OAuth2Token + from x2raindrop_cli.x.client import XClient + + token = OAuth2Token( + access_token="test", + refresh_token=None, + token_type="bearer", + expires_at=datetime.now() + timedelta(hours=1), + scope="tweet.read", + ) + client = XClient(token) + try: + parsed = client._parse_folder({"id": "folder-1", "name": "Test"}) + skipped = client._parse_folder({"id": "folder-2", "name": " "}) + tweet_id = client._extract_item_id({"id": 12345}) + finally: + client.close() + + assert parsed is not None + assert parsed.id == "folder-1" + assert parsed.name == "Test" + assert skipped is None + assert tweet_id == "12345" + class TestBookmarkItemParsing: """Tests for BookmarkItem creation and parsing logic."""