From 26e7b72a7a6f905352e999fb47b973a27239b48e Mon Sep 17 00:00:00 2001 From: Uchechukwu Orji Date: Wed, 27 May 2026 23:39:16 +0100 Subject: [PATCH 1/2] add flavours to title --- backend/src/cms_backend/api/main.py | 8 +++ backend/src/cms_backend/api/routes/titles.py | 3 + backend/src/cms_backend/db/book.py | 9 +++ backend/src/cms_backend/db/books.py | 16 ++--- backend/src/cms_backend/db/models.py | 3 + backend/src/cms_backend/db/title.py | 12 ++++ .../9dc25bcae26f_add_flavours_to_title.py | 37 ++++++++++++ .../src/cms_backend/mill/processors/title.py | 18 ++++-- backend/src/cms_backend/schemas/orms.py | 2 + backend/tests/api/routes/test_titles.py | 3 + backend/tests/conftest.py | 2 + backend/tests/db/test_books.py | 37 +++++++++++- .../processors/test_zimfarm_notification.py | 48 +++++++++++++++ frontend/src/components/BookTable.vue | 14 ++++- frontend/src/components/EditBookForm.vue | 7 ++- frontend/src/components/TitleForm.vue | 60 +++++++++++++++++++ frontend/src/components/TitleFormDialog.vue | 2 +- frontend/src/types/book.ts | 1 + frontend/src/types/title.ts | 3 + frontend/src/views/BookView.vue | 41 ++++++++++++- frontend/src/views/TitleView.vue | 24 ++++++++ 21 files changed, 328 insertions(+), 22 deletions(-) create mode 100644 backend/src/cms_backend/migrations/versions/9dc25bcae26f_add_flavours_to_title.py diff --git a/backend/src/cms_backend/api/main.py b/backend/src/cms_backend/api/main.py index 6ee91c10..48c59839 100644 --- a/backend/src/cms_backend/api/main.py +++ b/backend/src/cms_backend/api/main.py @@ -105,6 +105,14 @@ async def request_validation_error_handler(_, exc: RequestValidationError): ) +@app.exception_handler(ValueError) +async def value_error_handler(_, exc: ValueError): + return JSONResponse( + status_code=HTTPStatus.BAD_REQUEST, + content={"success": False, "message": exc.args[0]}, + ) + + @app.exception_handler(ValidationError) async def validation_error_handler(_, exc: ValidationError): # transform the pydantic validation errors to a dictionary mapping diff --git a/backend/src/cms_backend/api/routes/titles.py b/backend/src/cms_backend/api/routes/titles.py index 4ce18f49..8761918c 100644 --- a/backend/src/cms_backend/api/routes/titles.py +++ b/backend/src/cms_backend/api/routes/titles.py @@ -66,6 +66,7 @@ class BaseTitleCreateUpdateSchema(BaseModel): publisher: NotEmptyString | None = None language: NotEmptyString | None = None illustration_48x48_at_1: Base64Str | None = None + flavours: list[str] | None = None @model_validator(mode="after") def validate_unique_collection_titles(self) -> Self: @@ -158,6 +159,7 @@ def create_title( source=title_data.source, long_description=title_data.long_description, description=title_data.description, + flavours=title_data.flavours, ) return create_title_light_schema(title) @@ -188,6 +190,7 @@ def update_title( license_=title_data.license, relation=title_data.relation, source=title_data.source, + flavours=title_data.flavours, ) return create_title_light_schema(title) diff --git a/backend/src/cms_backend/db/book.py b/backend/src/cms_backend/db/book.py index 4d169bef..90086016 100644 --- a/backend/src/cms_backend/db/book.py +++ b/backend/src/cms_backend/db/book.py @@ -100,6 +100,9 @@ def create_book_full_schema(book: Book) -> BookFullSchema: current_locations=current_locations, target_locations=target_locations, title_archived=book.title.archived if book.title else False, + has_flavour_mismatch=book.flavour not in book.title.flavours + if book.title + else False, ) @@ -251,6 +254,12 @@ def move_book( if not book.title: raise ValueError(f"Book {book_id} has no associated title.") + if destination == "prod" and book.flavour not in book.title.flavours: + raise ValueError( + f"Book flavour '{book.flavour}' is not in title flavours " + f"{book.title.flavours}" + ) + existing_filename = current_location.filename goes_to_staging = destination == "staging" diff --git a/backend/src/cms_backend/db/books.py b/backend/src/cms_backend/db/books.py index c2e766e5..0e21e9b8 100644 --- a/backend/src/cms_backend/db/books.py +++ b/backend/src/cms_backend/db/books.py @@ -46,13 +46,8 @@ def get_books( Book.date, Book.flavour, Book.issues, - ).order_by( - Book.has_error.desc(), - Book.location_kind, - Book.needs_file_operation.desc(), - Book.created_at.desc(), - Book.id, - ) + Title.flavours, + ).join(Title, Book.title_id == Title.id, isouter=True) if book_id is not None: stmt = stmt.where(Book.id.cast(String).ilike(f"%{book_id}%")) @@ -124,6 +119,9 @@ def get_books( date=date, flavour=flavour, issues=book_issues, + has_flavour_mismatch=flavour not in title_flavours + if title_flavours is not None + else False, ) for ( book_id_result, @@ -138,6 +136,7 @@ def get_books( date, flavour, book_issues, + title_flavours, ) in session.execute( stmt.offset(skip) .limit(limit) @@ -146,6 +145,7 @@ def get_books( Book.location_kind, Book.needs_file_operation, Book.created_at.desc(), + Book.id, ) ).all() ], @@ -327,7 +327,7 @@ def get_book_languages(session: OrmSession) -> BookLanguagesSchema: def get_book_flavours(session: OrmSession) -> ListResult[str]: - """Get a list of book falavours""" + """Get a list of book flavours""" stmt = ( select(Book.flavour) .distinct() diff --git a/backend/src/cms_backend/db/models.py b/backend/src/cms_backend/db/models.py index 2f24f85b..f3c1f21f 100644 --- a/backend/src/cms_backend/db/models.py +++ b/backend/src/cms_backend/db/models.py @@ -213,6 +213,9 @@ class Title(Base): maturity: Mapped[str] = mapped_column(init=False, index=True, default="unstable") events: Mapped[list[str]] = mapped_column(init=False, default_factory=list) archived: Mapped[bool] = mapped_column(default=False, server_default=false()) + flavours: Mapped[list[str]] = mapped_column( + default_factory=list, server_default="{}" + ) books: Mapped[list["Book"]] = relationship( back_populates="title", diff --git a/backend/src/cms_backend/db/title.py b/backend/src/cms_backend/db/title.py index c9cf0603..4161800d 100644 --- a/backend/src/cms_backend/db/title.py +++ b/backend/src/cms_backend/db/title.py @@ -50,6 +50,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: license=title.license, relation=title.relation, source=title.source, + flavours=title.flavours, books=[ BookLightSchema( id=book.id, @@ -64,6 +65,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: date=book.date, flavour=book.flavour, issues=book.issues, + has_flavour_mismatch=book.flavour not in title.flavours, ) for book in sorted( title.books, @@ -107,6 +109,7 @@ def create_title_light_schema(title: Title) -> TitleLightSchema: license=title.license, relation=title.relation, source=title.source, + flavours=title.flavours, ) @@ -183,6 +186,7 @@ def get_titles( Title.license.label("title_license"), Title.relation.label("title_relation"), Title.source.label("title_source"), + Title.flavours.label("title_flavours"), ) .join(CollectionTitle, CollectionTitle.title_id == Title.id, isouter=True) .join(Collection, CollectionTitle.collection_id == Collection.id, isouter=True) @@ -226,6 +230,7 @@ def get_titles( license=title_license, relation=title_relation, source=title_source, + flavours=title_flavours, ) for ( title_id, @@ -242,6 +247,7 @@ def get_titles( title_license, title_relation, title_source, + title_flavours, ) in session.execute(stmt.offset(skip).limit(limit)).all() ], ) @@ -263,6 +269,7 @@ def create_title( license_: str | None = None, relation: str | None = None, source: str | None = None, + flavours: list[str] | None = None, ) -> Title: """Create a new title""" @@ -281,6 +288,7 @@ def create_title( title.source = source title.description = description title.long_description = long_description + title.flavours = [] if flavours is None else flavours title.events.append(f"{getnow()}: title created") session.add(title) @@ -332,6 +340,7 @@ def update_title( license_: str | None = None, relation: str | None = None, source: str | None = None, + flavours: list[str] | None = None, ) -> Title: """Update a title's details @@ -428,6 +437,9 @@ def update_title( title.source = source title.events.append(f"{getnow()}: source updated from {old_source} to {source}") + if flavours is not None: + title.flavours = flavours + name_changed: bool = False # Update name if provided if name and name != title.name: diff --git a/backend/src/cms_backend/migrations/versions/9dc25bcae26f_add_flavours_to_title.py b/backend/src/cms_backend/migrations/versions/9dc25bcae26f_add_flavours_to_title.py new file mode 100644 index 00000000..9c1f9462 --- /dev/null +++ b/backend/src/cms_backend/migrations/versions/9dc25bcae26f_add_flavours_to_title.py @@ -0,0 +1,37 @@ +"""add flavours to title + +Revision ID: 9dc25bcae26f +Revises: a8f64135b053 +Create Date: 2026-05-26 12:24:41.086893 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "9dc25bcae26f" +down_revision = "e70e0c595eb9" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "title", + sa.Column( + "flavours", + postgresql.ARRAY(sa.String()), + server_default="{}", + nullable=False, + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("title", "flavours") + # ### end Alembic commands ### diff --git a/backend/src/cms_backend/mill/processors/title.py b/backend/src/cms_backend/mill/processors/title.py index f207e0dd..3a91782b 100644 --- a/backend/src/cms_backend/mill/processors/title.py +++ b/backend/src/cms_backend/mill/processors/title.py @@ -55,21 +55,29 @@ def add_book_to_title(session: OrmSession, book: Book, title: Title): title.relation = book.zim_metadata.get("Relation") title.source = book.zim_metadata.get("Source") + issues: list[str] = [] + different_metadata_keys = get_differing_metadata_keys(book) if different_metadata_keys: - book.issues = ["metadata mismatch"] + issues.append("metadata mismatch") book.events.append( f"{getnow()}: book metadata is different from title metadata: " f"{','.join(different_metadata_keys)}" ) + if title.flavours and book.flavour not in title.flavours: + issues.append("flavour mismatch") + book.events.append( + f"{getnow()}: book flavour is not in list of title flavours" + ) + + book.issues = issues + # Determine if this book goes to staging or prod based on # - title maturity: For now, only 'stable' maturity move straight to prod, # other maturity moves through staging first - # - if book has different metadata from title - goes_to_staging = ( - title.maturity != "stable" or len(different_metadata_keys) != 0 - ) + # - issues: books with any issues move to staging regardless of maturity + goes_to_staging = title.maturity != "stable" or len(issues) != 0 target_locations = ( [ diff --git a/backend/src/cms_backend/schemas/orms.py b/backend/src/cms_backend/schemas/orms.py index 5b45a4e4..b41577f8 100644 --- a/backend/src/cms_backend/schemas/orms.py +++ b/backend/src/cms_backend/schemas/orms.py @@ -33,6 +33,7 @@ class TitleLightSchema(BaseModel): license: str | None relation: str | None source: str | None + flavours: list[str] class BaseTitleCollectionSchema(BaseModel): @@ -116,6 +117,7 @@ class BookLightSchema(BaseModel): date: str | None flavour: str | None issues: list[str] + has_flavour_mismatch: bool class BookFullSchema(BookLightSchema): diff --git a/backend/tests/api/routes/test_titles.py b/backend/tests/api/routes/test_titles.py index ce34f6bf..2f6a1489 100644 --- a/backend/tests/api/routes/test_titles.py +++ b/backend/tests/api/routes/test_titles.py @@ -58,6 +58,7 @@ def test_get_titles( "relation", "source", "license", + "flavours", } assert data["items"][0]["name"] == "wikipedia_fr_all" @@ -326,6 +327,7 @@ def test_get_title_by_id( "relation", "source", "license", + "flavours", } # Verify field values @@ -379,6 +381,7 @@ def test_get_title_by_id_with_books( "flavour", "deletion_date", "issues", + "has_flavour_mismatch", } assert data["books"][0]["title_id"] == str(title.id) assert data["books"][1]["title_id"] == str(title.id) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 094704b4..e7060698 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -167,6 +167,7 @@ def _create_title( license: str | None = None, # noqa: A002 relation: str | None = None, source: str | None = None, + flavours: list[str] | None = None, ) -> Title: db_title = Title( name=name, @@ -181,6 +182,7 @@ def _create_title( license=license, relation=relation, source=source, + flavours=flavours if flavours is not None else [], ) dbsession.add(db_title) dbsession.flush() diff --git a/backend/tests/db/test_books.py b/backend/tests/db/test_books.py index ba2dc521..570765da 100644 --- a/backend/tests/db/test_books.py +++ b/backend/tests/db/test_books.py @@ -633,11 +633,11 @@ def test_move_book_staging_to_prod( monkeypatch: pytest.MonkeyPatch, ): """Test moving a book from staging to prod""" - title = create_title(name="test_en_all") + title = create_title(name="test_en_all", flavours=["mini", "maxi"]) collection = create_collection(warehouse=warehouse) create_collection_title(title=title, collection=collection, path=Path("zim")) - book = create_book(name="test_en_all", date="2024-01") + book = create_book(name="test_en_all", date="2024-01", flavour="maxi") book.title = title book.location_kind = "staging" create_book_location( @@ -666,6 +666,39 @@ def test_move_book_staging_to_prod( assert target_locations[0].filename == "test_en_all_2024-01.zim" +def test_move_book_with_different_flavor_from_title_to_prod( + dbsession: OrmSession, + warehouse: Warehouse, + create_book: Callable[..., Book], + create_title: Callable[..., Title], + create_collection: Callable[..., Collection], + create_collection_title: Callable[..., CollectionTitle], + create_book_location: Callable[..., BookLocation], + monkeypatch: pytest.MonkeyPatch, +): + """Test moving a book with different flavor from it's title from staging to prod""" + title = create_title(name="test_en_all", flavours=["mini", "maxi"]) + collection = create_collection(warehouse=warehouse) + create_collection_title(title=title, collection=collection, path=Path("zim")) + + book = create_book(name="test_en_all", date="2024-01", flavour="nopic") + book.title = title + book.location_kind = "staging" + create_book_location( + book=book, + warehouse_id=Context.staging_warehouse_id, + path=Context.staging_base_path, + filename="test_en_all_2024-01.zim", + status="current", + ) + dbsession.flush() + + now = getnow() + monkeypatch.setattr("cms_backend.db.book.getnow", lambda: now) + with pytest.raises(ValueError): + move_book(dbsession, book_id=book.id, destination="prod") + + def test_move_book_same_destination_raises_error( dbsession: OrmSession, create_book: Callable[..., Book], diff --git a/backend/tests/mill/processors/test_zimfarm_notification.py b/backend/tests/mill/processors/test_zimfarm_notification.py index ed684022..da3dcb08 100644 --- a/backend/tests/mill/processors/test_zimfarm_notification.py +++ b/backend/tests/mill/processors/test_zimfarm_notification.py @@ -582,6 +582,54 @@ def test_moves_book_to_staging_due_to_diffrent_metadata_from_title( assert book.needs_file_operation is True assert book.needs_processing is False + def test_moves_book_to_staging_due_to_diffrent_flavour_from_title( + self, + dbsession: OrmSession, + warehouse: Warehouse, # noqa: ARG002 + create_zimfarm_notification: Callable[..., ZimfarmNotification], + create_title: Callable[..., Title], + create_collection: Callable[..., Collection], + create_warehouse: Callable[..., Warehouse], + ): + """ + Test that book goes to staging because there is a flavour mismatch between + it and it's title + """ + + title = create_title(name="test_en_all", flavours=["maxi", "mini"]) + title.maturity = "stable" + + prod = create_warehouse( + name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") + ) + collection = create_collection(warehouse=prod) + + ct = CollectionTitle(path=Path("wikipedia")) + ct.title = title + ct.collection = collection + dbsession.add(ct) + dbsession.flush() + + content = VALID_NOTIFICATION_CONTENT.copy() + content["folder_name"] = "" + + notification = create_zimfarm_notification(content=content) + dbsession.flush() + + process_notification(dbsession, notification) + + assert notification.status == "processed" + + book = dbsession.query(Book).filter_by(id=notification.id).first() + assert book is not None + assert book.title_id == title.id + assert book.location_kind == "staging" + assert len(book.issues) == 1 + assert set(book.issues) == {"flavour mismatch"} + assert book.has_error is False + assert book.needs_file_operation is True + assert book.needs_processing is False + class TestValidNotificationOnArchivedTitle: """Test valid notifications that are associated to an archived title.""" diff --git a/frontend/src/components/BookTable.vue b/frontend/src/components/BookTable.vue index 7539e936..ec0b373c 100644 --- a/frontend/src/components/BookTable.vue +++ b/frontend/src/components/BookTable.vue @@ -41,8 +41,18 @@