diff --git a/backend/maint-scripts/update_recipes_and_title_flavours.py b/backend/maint-scripts/update_recipes_and_title_flavours.py new file mode 100755 index 0000000..d715a7d --- /dev/null +++ b/backend/maint-scripts/update_recipes_and_title_flavours.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Maintenance script to populate CMS database with recipes and title flavours from +zimfarm API. + +This script is idempotent and can be run multiple times without creating duplicates. +It +- Finds all recipes on Zimfarm API +- Creates missing recipes on CMS database +- Fetches all the successful tasks for the recipe +- For each task + - Find a title with the same name from the tasks' file information + - Creates a title flavour for the title and associates it with the recipe + - If existing flavour differs from current recipe, warning messages are logged + +Environment variables required: +- ZIMFARM_API_URL: URL of Zimfarm API to fetch recipes and tasks from +- USERNAME: the username of the account that will create history entries for recipes. + Defaults to 'maint-scrpts' +""" + +import os +from dataclasses import dataclass +from json import JSONDecodeError +from typing import Any +from uuid import UUID + +import requests # pyright: ignore[reportMissingModuleSource] +from sqlalchemy import select +from sqlalchemy.orm import Session as OrmSession +from sqlalchemy.orm.attributes import flag_modified + +from cms_backend import logger +from cms_backend.db import Session +from cms_backend.db.account import get_account_by_username +from cms_backend.db.flavour import get_title_flavour_or_none +from cms_backend.db.models import Book, TitleFlavour, ZimfarmRecipe +from cms_backend.db.title import get_title_by_name_or_none +from cms_backend.db.zimfarm_recipe import ( + create_zimfarm_recipe, + create_zimfarm_recipe_history_entry, + get_zimfarm_recipe_by_id_or_none, +) +from cms_backend.utils.zim import get_missing_keys + + +@dataclass +class Response: + """A response from the webapi""" + + status_code: int + success: bool + json: dict[str, Any] + + +def query_api( + url: str, + method: str = "get", + *, + headers: dict[str, Any] | None = None, + payload: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + timeout: int = 30, +) -> Response: + req_headers: dict[str, Any] = {} + + req_headers.update( # pyright: ignore[reportUnknownMemberType] + headers if headers else {} + ) + func = { + "GET": requests.get, + "POST": requests.post, + "PATCH": requests.patch, + "DELETE": requests.delete, + "PUT": requests.put, + }.get(method.upper(), requests.get) + + resp = None + try: + resp = func( + url, headers=req_headers, json=payload, params=params, timeout=timeout + ) + return Response( + status_code=resp.status_code, + success=resp.ok, + json=resp.json() if resp.text and resp.text.strip() else {}, + ) + except (JSONDecodeError, Exception) as exc: + logger.exception( + f"unexpected error while making request to {url} : " + f"{resp.text if resp else exc}" + ) + return Response( + status_code=resp.status_code if resp else -1, + success=resp.ok if resp else False, + json={}, + ) + + +def process_task( + session: OrmSession, + *, + task: dict[str, Any], + recipe: ZimfarmRecipe, + zimfarm_api_url: str, + author_id: UUID, +): + response = query_api(f"{zimfarm_api_url}/tasks/{task['id']}") + if not response.success: + logger.error( + f"Unable to fetch task {task['id']} from {zimfarm_api_url}: {response.json}" + ) + return + + for filename in response.json.get("files", {}): + metadata = response.json["files"][filename].get("info", {}).get("metadata", {}) + missing_keys = get_missing_keys(metadata, "Name") + if missing_keys: + logger.warning( + f"Task {task['id']} metadata is missing keys: {','.join(missing_keys)}" + ) + continue + + title = get_title_by_name_or_none(session, name=metadata["Name"]) + if title is None: + logger.debug( + f"Title with name '{metadata['Name']}' from task {task['id']} " + "does not yet exist on CMS" + ) + continue + + recipe.title = title + + flavour = metadata.get("Flavour") + if flavour is None: + logger.debug(f"Task {task['id']} has no flavour") + continue + + flavour = flavour[1:] if flavour.startswith("_") else flavour + + tf = get_title_flavour_or_none(session, title.id, flavour) + if tf: + logger.debug( + f"Title flavour '{tf.flavour}' already exists for title '{title.name}'" + ) + if tf.recipe_id != recipe.id: + logger.warning( + f"Title flavour '{tf.flavour}' for title '{title.name}' is " + f"attached to a different recipe from zimfarm '{recipe.name}'" + ) + else: + tf = TitleFlavour(flavour=flavour) + tf.title = title + tf.recipe = recipe + session.add(tf) + session.flush() + logger.info(f"Created title flavour '{flavour}' for title '{title.name}'") + create_zimfarm_recipe_history_entry( + session, + recipe, + author_id=author_id, + comment=f"Added '{flavour}' for title '{title.name}'", + ) + + # update books notifications whose flavour matches + books = session.scalars( + select(Book).where(Book.flavour == flavour, Book.title_id == title.id) + ).all() + for book in books: + if book.zimfarm_notification and get_missing_keys( + book.zimfarm_notification.content, "recipe_id", "recipe_name" + ): + zimfarm_notification = book.zimfarm_notification + zimfarm_notification.content["recipe_id"] = str(recipe.id) + zimfarm_notification.content["recipe_name"] = recipe.name + flag_modified(zimfarm_notification, "content") + + +def process_recipe( + session: OrmSession, zf_recipe: ZimfarmRecipe, zimfarm_api_url: str, author_id: UUID +): + skip = 0 + limit = 50 + while True: + response = query_api( + f"{zimfarm_api_url}/tasks", + params={ + "skip": skip, + "limit": limit, + "status": ["succeeded"], + "recipe_id": zf_recipe.id, + "sort_criteria": "done", + }, + ) + if not response.success: + logger.error( + f"Unable to process tasks for recipe {zf_recipe.name}: {response.json}" + ) + break + tasks = response.json["items"] + if len(tasks) == 0: + logger.info(f"No more tasks to process for recipe {zf_recipe.name}") + break + + for task in tasks: + with session.begin_nested(): + process_task( + session, + task=task, + recipe=zf_recipe, + zimfarm_api_url=zimfarm_api_url, + author_id=author_id, + ) + skip += limit + + +def populate_recipes_from_zimfarm( + session: OrmSession, zimfarm_api_url: str, author_id: UUID +): + """Fetch recipes from zimfarm and attach CMS titles/title flavours to recipes.""" + skip = 0 + limit = 50 + + while True: + response = query_api( + f"{zimfarm_api_url}/recipes", params={"skip": skip, "limit": limit} + ) + if not response.success: + logger.error( + f"Unable to fetch recipes from {zimfarm_api_url}: {response.json}. " + "Exiting..." + ) + break + recipes = response.json["items"] + if len(recipes) == 0: + logger.info(f"No more recipes returned from {zimfarm_api_url}") + break + for recipe in recipes: + zf_recipe = get_zimfarm_recipe_by_id_or_none(session, UUID(recipe["id"])) + if zf_recipe is None: + zf_recipe = create_zimfarm_recipe( + session, recipe_id=recipe["id"], recipe_name=recipe["name"] + ) + logger.info(f"Created zimfarm recipe '{zf_recipe.name}'") + process_recipe(session, zf_recipe, zimfarm_api_url, author_id=author_id) + skip += limit + + +def main(): + + zimfarm_api_url = os.getenv("ZIMFARM_API_URL", "https://api.farm.openzim.org/v2") + with Session.begin() as session: + author = get_account_by_username( + session, username=os.getenv("USERNAME", default="maint-scripts") + ) + populate_recipes_from_zimfarm(session, zimfarm_api_url, author_id=author.id) + + +if __name__ == "__main__": + main() diff --git a/backend/maint-scripts/update_title_flavours_from_books.py b/backend/maint-scripts/update_title_flavours_from_books.py deleted file mode 100755 index 64b2e43..0000000 --- a/backend/maint-scripts/update_title_flavours_from_books.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# ruff: noqa: T201 -"""This script sets the title flavours from the set of all books belonging to the title. - -It also fixes books whose flavours start with an underscore. -""" - -from typing import Any - -from sqlalchemy import select -from sqlalchemy.orm import Session as OrmSession - -from cms_backend import logger -from cms_backend.db import Session -from cms_backend.db.models import Book, Title -from cms_backend.db.title import get_title_by_id - - -def update_title_flavour(session: OrmSession, title: Title) -> tuple[bool, str]: - if title.archived: - logger.info(f"Skipping archived title {title.id} ({title.name})") - return (False, "Title is archived") - - books = session.scalars( - select(Book) - .join(Title, Book.title_id == Title.id) - .order_by(Book.flavour) - .where(Book.flavour.isnot(None), Book.title_id == title.id) - ).all() - for book in books: - if book.flavour and book.flavour.startswith("_"): - new_flavour = book.flavour[1:] - logger.info( - f"Updated book {book.name} from {book.flavour} to {new_flavour}" - ) - book.flavour = new_flavour - session.add(book) - - flavours = {book.flavour for book in books if book.flavour is not None} - title.flavours = list(flavours) - session.add(title) - session.flush() - - if not flavours: - logger.info( - f"No flavours found in books belonging to title {title.id} ({title.name})" - ) - return ( - False, - f"No flavours found in books belonging to title {title.id} ({title.name})", - ) - - logger.info(f"✓ Updated title {title.id} ({title.name}) flavours to {flavours}") - return (True, "") - - -def main(): - - with Session.begin() as session: - title_ids = session.scalars(select(Title.id)).all() - logger.info(f"Found {len(title_ids)} titles to process") - nb_titles_updated = 0 - nb_titles_skipped = 0 - reasons: list[dict[str, Any]] = [] - - for title_id in title_ids: - title = get_title_by_id(session, title_id=title_id) - processed, reason = update_title_flavour(session, title) - if processed: - nb_titles_updated += 1 - else: - nb_titles_skipped += 1 - reasons.append({title.name: reason}) - - logger.info( - f"Updated {nb_titles_updated} title(s) metadata, skipped " - f"{nb_titles_skipped} titles(s)" - ) - - if reasons: - print("\nSkipped titles summary:") - print("| Title Name | Reason |") - print("|------------|--------|") - for entry in reasons: - for title_name, reason in entry.items(): - print(f"| {title_name} | {reason} |") - - -if __name__ == "__main__": - main() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8c80521..69af8d5 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "Werkzeug == 3.1.5", "xxhash == 3.7.0", "pycountry == 26.2.16", + "requests == 2.34.2", ] dynamic = ["version"] diff --git a/backend/src/cms_backend/api/main.py b/backend/src/cms_backend/api/main.py index 835aa7c..13e1c36 100644 --- a/backend/src/cms_backend/api/main.py +++ b/backend/src/cms_backend/api/main.py @@ -24,6 +24,9 @@ from cms_backend.api.routes.zimfarm_notifications import ( router as zimfarm_notification_router, ) +from cms_backend.api.routes.zimfarm_recipes import ( + router as zimfarm_recipe_router, +) from cms_backend.context import Context from cms_backend.db.exceptions import ( RecordAlreadyExistsError, @@ -69,6 +72,7 @@ def create_app(*, debug: bool = True): main_router = APIRouter(prefix="/v1") main_router.include_router(router=config_router) main_router.include_router(router=zimfarm_notification_router) + main_router.include_router(router=zimfarm_recipe_router) main_router.include_router(router=healthcheck_router) main_router.include_router(router=titles_router) main_router.include_router(router=books_router) diff --git a/backend/src/cms_backend/api/routes/account.py b/backend/src/cms_backend/api/routes/account.py index 41cead0..a9a6d70 100644 --- a/backend/src/cms_backend/api/routes/account.py +++ b/backend/src/cms_backend/api/routes/account.py @@ -8,7 +8,6 @@ from werkzeug.security import check_password_hash, generate_password_hash from cms_backend.api.routes.dependencies import get_current_account, require_permission -from cms_backend.api.routes.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.api.routes.http_errors import BadRequestError, ForbiddenError from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db import gen_dbsession @@ -25,6 +24,7 @@ from cms_backend.db.models import Account from cms_backend.roles import RoleEnum from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.schemas.models import AccountUpdateSchema from cms_backend.schemas.orms import AccountSchema from cms_backend.utils import is_valid_uuid diff --git a/backend/src/cms_backend/api/routes/books.py b/backend/src/cms_backend/api/routes/books.py index 17f30d0..dbb7437 100644 --- a/backend/src/cms_backend/api/routes/books.py +++ b/backend/src/cms_backend/api/routes/books.py @@ -8,7 +8,6 @@ from sqlalchemy.orm import Session as OrmSession from cms_backend.api.routes.dependencies import get_current_account, require_permission -from cms_backend.api.routes.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db import gen_dbsession from cms_backend.db.book import backup_book as db_backup_book @@ -28,6 +27,7 @@ from cms_backend.db.books import get_zim_urls as db_get_zim_urls from cms_backend.db.models import Account from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.schemas.models import ( BookLanguagesSchema, BookUpdateSchema, diff --git a/backend/src/cms_backend/api/routes/collection.py b/backend/src/cms_backend/api/routes/collection.py index 788e6b6..8a6409f 100644 --- a/backend/src/cms_backend/api/routes/collection.py +++ b/backend/src/cms_backend/api/routes/collection.py @@ -9,7 +9,6 @@ from sqlalchemy.orm import Session as OrmSession from cms_backend.api.routes.dependencies import get_current_account, require_permission -from cms_backend.api.routes.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.api.routes.utils import build_library_xml from cms_backend.db import gen_dbsession @@ -35,6 +34,7 @@ from cms_backend.db.exceptions import RecordDoesNotExistError from cms_backend.db.models import Account from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.schemas.models import CollectionUpdateSchema from cms_backend.schemas.orms import ( CollectionFullSchema, diff --git a/backend/src/cms_backend/api/routes/events.py b/backend/src/cms_backend/api/routes/events.py index 3d72639..ee60aee 100644 --- a/backend/src/cms_backend/api/routes/events.py +++ b/backend/src/cms_backend/api/routes/events.py @@ -3,11 +3,11 @@ from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session as OrmSession -from cms_backend.api.routes.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db import gen_dbsession from cms_backend.db.event import get_events as db_get_events from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.schemas.orms import EventLightSchema router = APIRouter(prefix="/events", tags=["events"]) diff --git a/backend/src/cms_backend/api/routes/titles.py b/backend/src/cms_backend/api/routes/titles.py index 3b7da59..d782785 100644 --- a/backend/src/cms_backend/api/routes/titles.py +++ b/backend/src/cms_backend/api/routes/titles.py @@ -12,11 +12,6 @@ get_current_account_or_none, require_permission, ) -from cms_backend.api.routes.fields import ( - LimitFieldMax200, - NotEmptyString, - SkipField, -) from cms_backend.api.routes.http_errors import ForbiddenError from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db import gen_dbsession @@ -41,6 +36,11 @@ from cms_backend.db.title import revert_title as db_revert_title from cms_backend.db.title import update_title as db_update_title from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import ( + LimitFieldMax200, + NotEmptyString, + SkipField, +) from cms_backend.schemas.models import TitleCreateSchema, TitleUpdateSchema from cms_backend.schemas.orms import ( TitleFullSchema, diff --git a/backend/src/cms_backend/api/routes/warehouse.py b/backend/src/cms_backend/api/routes/warehouse.py index fd2d0cf..63c0046 100644 --- a/backend/src/cms_backend/api/routes/warehouse.py +++ b/backend/src/cms_backend/api/routes/warehouse.py @@ -4,9 +4,9 @@ from sqlalchemy.orm import Session as OrmSession from cms_backend.api.routes.dependencies import gen_dbsession -from cms_backend.api.routes.fields import LimitFieldMax200, SkipField from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db.warehouse import get_warehouses as db_get_warehouses +from cms_backend.schemas.fields import LimitFieldMax200, SkipField router = APIRouter(prefix="/warehouses", tags=["warehouses"]) diff --git a/backend/src/cms_backend/api/routes/zimfarm_notifications.py b/backend/src/cms_backend/api/routes/zimfarm_notifications.py index cf7f0c7..6128f91 100644 --- a/backend/src/cms_backend/api/routes/zimfarm_notifications.py +++ b/backend/src/cms_backend/api/routes/zimfarm_notifications.py @@ -8,7 +8,6 @@ from cms_backend import logger from cms_backend.api.routes.dependencies import require_permission -from cms_backend.api.routes.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata from cms_backend.db import gen_dbsession from cms_backend.db.zimfarm_notification import ( @@ -24,6 +23,7 @@ get_zimfarm_notifications as db_get_zimfarm_notifications, ) from cms_backend.schemas import BaseModel, WithExtraModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField from cms_backend.schemas.orms import ( ZimfarmNotificationFullSchema, ZimfarmNotificationLightSchema, diff --git a/backend/src/cms_backend/api/routes/zimfarm_recipes.py b/backend/src/cms_backend/api/routes/zimfarm_recipes.py new file mode 100644 index 0000000..db4ceea --- /dev/null +++ b/backend/src/cms_backend/api/routes/zimfarm_recipes.py @@ -0,0 +1,155 @@ +from http import HTTPStatus +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Path, Query +from fastapi.responses import JSONResponse +from sqlalchemy.orm import Session as OrmSession + +from cms_backend.api.routes.dependencies import get_current_account, require_permission +from cms_backend.api.routes.models import ListResponse, calculate_pagination_metadata +from cms_backend.db import gen_dbsession +from cms_backend.db.models import Account +from cms_backend.db.title import get_title +from cms_backend.db.zimfarm_recipe import ( + create_zimfarm_recipe_history_schema, + create_zimfarm_recipe_schema, +) +from cms_backend.db.zimfarm_recipe import ( + get_zimfarm_recipe as db_get_zimfarm_recipe, +) +from cms_backend.db.zimfarm_recipe import ( + get_zimfarm_recipe_history as db_get_zimfarm_recipe_history, +) +from cms_backend.db.zimfarm_recipe import ( + get_zimfarm_recipe_history_entry as db_get_zimfarm_recipe_history_entry, +) +from cms_backend.db.zimfarm_recipe import get_zimfarm_recipes as db_get_zimfarm_recipes +from cms_backend.db.zimfarm_recipe import ( + update_zimfarm_recipe as db_update_zimfarm_recipe, +) +from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import LimitFieldMax200, NotEmptyString, SkipField +from cms_backend.schemas.orms import ( + ZimfarmRecipeFullSchema, + ZimfarmRecipeHistorySchema, + ZimfarmRecipeLightSchema, +) + +router = APIRouter(prefix="/recipes", tags=["zimfarm-recipes"]) + + +class ZimfarmRecipesGetSchema(BaseModel): + skip: SkipField = 0 + limit: LimitFieldMax200 = 20 + name: NotEmptyString | None = None + + +class RecipeUpdateSchema(BaseModel): + title_name: NotEmptyString + flavours: list[str] + old_recipes: set[UUID] + + +@router.get("") +def get_zimfarm_recipes( + params: Annotated[ZimfarmRecipesGetSchema, Query()], + session: Annotated[OrmSession, Depends(gen_dbsession)], +) -> ListResponse[ZimfarmRecipeLightSchema]: + """Get a list of zimfarm recipes""" + + results = db_get_zimfarm_recipes( + session, + skip=params.skip, + limit=params.limit, + name=params.name, + ) + + return ListResponse[ZimfarmRecipeLightSchema]( + meta=calculate_pagination_metadata( + nb_records=results.nb_records, + skip=params.skip, + limit=params.limit, + page_size=len(results.records), + ), + items=results.records, + ) + + +@router.get("/{recipe_identifier}") +def get_zimfarm_recipe( + recipe_identifier: Annotated[NotEmptyString, Path()], + session: Annotated[OrmSession, Depends(gen_dbsession)], +) -> ZimfarmRecipeFullSchema: + return create_zimfarm_recipe_schema( + db_get_zimfarm_recipe(session, recipe_identifier) + ) + + +@router.put( + "/{recipe_identifier}", + dependencies=[ + Depends(require_permission(namespace="title", name="update")), + Depends(require_permission(namespace="recipe", name="update")), + ], +) +def update_zimfarm_recipe( + recipe_identifier: Annotated[NotEmptyString, Path()], + request: RecipeUpdateSchema, + session: OrmSession = Depends(gen_dbsession), + current_account: Account = Depends(get_current_account), +): + title = get_title(session, request.title_name) + recipe = db_get_zimfarm_recipe(session, recipe_identifier) + db_update_zimfarm_recipe( + session, + recipe=recipe, + flavours=request.flavours, + title=title, + old_recipes=request.old_recipes, + author=current_account, + ) + + return JSONResponse( + content={"message": f"recipe '{recipe_identifier}' has been updated"}, + status_code=HTTPStatus.OK, + ) + + +@router.get( + "/{recipe_identifier}/history", + dependencies=[Depends(require_permission(namespace="recipe", name="update"))], +) +def get_title_history( + recipe_identifier: Annotated[NotEmptyString, Path()], + session: OrmSession = Depends(gen_dbsession), + skip: Annotated[SkipField, Query()] = 0, + limit: Annotated[LimitFieldMax200, Query()] = 200, +) -> ListResponse[ZimfarmRecipeHistorySchema]: + results = db_get_zimfarm_recipe_history( + session, recipe_identifier=recipe_identifier, skip=skip, limit=limit + ) + return ListResponse( + items=results.records, + meta=calculate_pagination_metadata( + nb_records=results.nb_records, + skip=skip, + limit=limit, + page_size=len(results.records), + ), + ) + + +@router.get( + "/{recipe_identifier}/history/{history_id}", + dependencies=[Depends(require_permission(namespace="title", name="update"))], +) +def get_zimfarm_recipe_history_entry( + recipe_identifier: Annotated[NotEmptyString, Path()], + history_id: Annotated[UUID, Path()], + session: OrmSession = Depends(gen_dbsession), +) -> ZimfarmRecipeHistorySchema: + history_entry = db_get_zimfarm_recipe_history_entry( + session, recipe_identifier=recipe_identifier, history_id=history_id + ) + return create_zimfarm_recipe_history_schema(history_entry) diff --git a/backend/src/cms_backend/context.py b/backend/src/cms_backend/context.py index 55ffdd4..5820073 100644 --- a/backend/src/cms_backend/context.py +++ b/backend/src/cms_backend/context.py @@ -107,3 +107,6 @@ class Context: article_count_change_threshold: float = field( default=float(os.getenv("ARTICLE_COUNT_CHANGE_THRESHOLD", "0.1")) ) + zimfarm_api_url: str = field( + default=os.getenv("ZIMFARM_API_URL", "https://api.farm.openzim.org/v2") + ) diff --git a/backend/src/cms_backend/db/book.py b/backend/src/cms_backend/db/book.py index f703c71..cb5bc37 100644 --- a/backend/src/cms_backend/db/book.py +++ b/backend/src/cms_backend/db/book.py @@ -12,7 +12,14 @@ from cms_backend.db import count_from_stmt from cms_backend.db.book_location import create_book_target_locations from cms_backend.db.exceptions import RecordDoesNotExistError -from cms_backend.db.models import Book, BookHistory, ZimfarmNotification +from cms_backend.db.flavour import get_title_flavours +from cms_backend.db.models import ( + Book, + BookHistory, + Title, + ZimfarmNotification, + ZimfarmRecipe, +) from cms_backend.db.rules import ( apply_retention_rules, has_flavour_mismatch, @@ -94,6 +101,23 @@ def create_book_full_schema(book: Book) -> BookFullSchema: if location.status == "target" ] + if book.title: + recipe_id = next( + ( + title_flavour.recipe_id + for title_flavour in book.title.flavours + if title_flavour.flavour == book.flavour + ), + None, + ) + else: + recipe_id = ( + UUID(book.zimfarm_notification.content["recipe_id"]) + if book.zimfarm_notification + and book.zimfarm_notification.content.get("recipe_id") + else None + ) + return BookFullSchema( id=book.id, title_id=book.title_id, @@ -116,12 +140,15 @@ def create_book_full_schema(book: Book) -> BookFullSchema: current_locations=current_locations, target_locations=target_locations, title_archived=book.title.archived if book.title else False, - has_flavour_mismatch=has_flavour_mismatch(book.flavour, book.title.flavours) + has_flavour_mismatch=has_flavour_mismatch( + book.flavour, get_title_flavours(book.title) + ) if book.title else False, has_backup=any( current_location.is_backup for current_location in current_locations ), + recipe_id=recipe_id, ) @@ -282,13 +309,13 @@ def move_book( if not book.title: raise ValueError(f"Book {book_id} has no associated title.") - if destination == "prod" and has_flavour_mismatch( - book.flavour, book.title.flavours - ): - raise ValueError( - f"Book flavour '{book.flavour}' is not in title expected flavours " - f"{book.title.flavours}" - ) + if destination == "prod": + title_flavours = get_title_flavours(book.title) + if has_flavour_mismatch(book.flavour, title_flavours): + raise ValueError( + f"Book flavour '{book.flavour}' is not in title expected flavours " + f"{title_flavours}" + ) existing_filename = current_location.filename @@ -589,7 +616,7 @@ def update_book_issues(session: OrmSession, book: Book, *, update_events: bool = f"{','.join(different_metadata_keys)}" ) - if has_flavour_mismatch(book.flavour, book.title.flavours): + if has_flavour_mismatch(book.flavour, get_title_flavours(book.title)): issues.append("flavour mismatch") if update_events: book.events.append( @@ -789,13 +816,13 @@ def remove_book_backup( def book_goes_to_staging(book: Book) -> bool: """Determine if a book goes to staging. - Assumes book already has an associated title. A book goes to `prod` if: + A book goes to `prod` if: + - it has a title - book title maturity is 'stable', and - book has no issues """ if not book.title: - raise ValueError("Book must have a title.") - + return True return book.title.maturity != "stable" or len(book.issues) != 0 @@ -930,3 +957,18 @@ def _recover_deleted_book(session: OrmSession, book: Book) -> Book: session.flush() return book + + +def book_has_recipe_issue( + book_flavour: str | None, book_title: Title, recipe: ZimfarmRecipe +) -> bool: + """Check if book has recipe issues.""" + if recipe.title is None: + return True + if recipe.title.id != book_title.id and book_title.id not in [ + tf.recipe_id for tf in book_title.flavours + ]: + return True + if has_flavour_mismatch(book_flavour, [tf.flavour for tf in recipe.flavours]): + return True + return False diff --git a/backend/src/cms_backend/db/books.py b/backend/src/cms_backend/db/books.py index ebfec1f..ba8b5ed 100644 --- a/backend/src/cms_backend/db/books.py +++ b/backend/src/cms_backend/db/books.py @@ -3,12 +3,19 @@ from uuid import UUID from pydantic import AnyUrl -from sqlalchemy import String, and_, case, or_, select +from sqlalchemy import String, and_, case, func, or_, select from sqlalchemy.orm import Session as OrmSession from cms_backend.context import Context from cms_backend.db import count_from_stmt -from cms_backend.db.models import Book, BookLocation, Collection, CollectionTitle, Title +from cms_backend.db.models import ( + Book, + BookLocation, + Collection, + CollectionTitle, + Title, + TitleFlavour, +) from cms_backend.db.rules import has_flavour_mismatch from cms_backend.schemas.models import BookLanguagesSchema, ZimUrlSchema, ZimUrlsSchema from cms_backend.schemas.orms import BookLightSchema, ListResult @@ -36,6 +43,12 @@ def get_books( ) -> ListResult[BookLightSchema]: """Get a list of books""" + flavours_subquery = ( + select(func.array_agg(TitleFlavour.flavour)) + .where(TitleFlavour.title_id == Book.title_id) + .scalar_subquery() + ) + stmt = select( Book.id, Book.title_id, @@ -49,7 +62,7 @@ def get_books( Book.date, Book.flavour, Book.issues, - Title.flavours, + flavours_subquery.label("title_flavours"), ).join(Title, Book.title_id == Title.id, isouter=True) if book_id is not None: diff --git a/backend/src/cms_backend/db/flavour.py b/backend/src/cms_backend/db/flavour.py new file mode 100644 index 0000000..c7889c6 --- /dev/null +++ b/backend/src/cms_backend/db/flavour.py @@ -0,0 +1,26 @@ +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session as OrmSession +from sqlalchemy.orm import selectinload + +from cms_backend.db.models import Title, TitleFlavour +from cms_backend.schemas.orms import TitleFlavourSchema + + +def get_title_flavours(title: Title) -> list[str]: + return [title_flavour.flavour for title_flavour in title.flavours] + + +def create_title_flavour_schema(tf: TitleFlavour) -> TitleFlavourSchema: + return TitleFlavourSchema(flavour=tf.flavour, recipe_id=tf.recipe_id) + + +def get_title_flavour_or_none( + session: OrmSession, title_id: UUID, flavour: str +) -> TitleFlavour | None: + return session.scalars( + select(TitleFlavour) + .where(TitleFlavour.title_id == title_id, TitleFlavour.flavour == flavour) + .options(selectinload(TitleFlavour.recipe)) + ).one_or_none() diff --git a/backend/src/cms_backend/db/models.py b/backend/src/cms_backend/db/models.py index ac6b67b..9af5063 100644 --- a/backend/src/cms_backend/db/models.py +++ b/backend/src/cms_backend/db/models.py @@ -242,9 +242,6 @@ class Title(Base): maturity: Mapped[str] = mapped_column(init=False, index=True, default="unstable") events: Mapped[list[str]] = mapped_column(init=False, default_factory=list) archived: Mapped[bool] = mapped_column(default=False, server_default=false()) - flavours: Mapped[list[str]] = mapped_column( - default_factory=list, server_default="{}" - ) books: Mapped[list["Book"]] = relationship( back_populates="title", @@ -269,6 +266,17 @@ class Title(Base): order_by="TitleHistory.created_at.desc()", ) + zimfarm_recipes: Mapped[list["ZimfarmRecipe"]] = relationship( + back_populates="title", init=False, default_factory=list + ) + flavours: Mapped[list["TitleFlavour"]] = relationship( + back_populates="title", + cascade="all, delete", + passive_deletes=True, + init=False, + default_factory=list, + ) + class TitleHistory(Base): __tablename__ = "title_history" @@ -296,9 +304,6 @@ class TitleHistory(Base): source: Mapped[str | None] = mapped_column(default=None) maturity: Mapped[str] = mapped_column(default="unstable") archived: Mapped[bool] = mapped_column(default=False, server_default=false()) - flavours: Mapped[list[str]] = mapped_column( - default_factory=list, server_default="{}" - ) collection_titles: Mapped[list[dict[str, Any]]] = mapped_column( default_factory=list, server_default="{}" ) @@ -465,3 +470,68 @@ class Event(Base): created_at: Mapped[datetime] topic: Mapped[str] payload: Mapped[dict[str, Any]] + + +class TitleFlavour(Base): + __tablename__ = "title_flavour" + title_id: Mapped[UUID] = mapped_column( + ForeignKey("title.id", ondelete="CASCADE"), init=False, primary_key=True + ) + flavour: Mapped[str] = mapped_column(primary_key=True) + recipe_id: Mapped[UUID] = mapped_column( + ForeignKey("zimfarm_recipe.id", ondelete="CASCADE"), init=False + ) + recipe: Mapped["ZimfarmRecipe"] = relationship( + init=False, back_populates="flavours" + ) + title: Mapped["Title"] = relationship(back_populates="flavours", init=False) + + +class ZimfarmRecipe(Base): + __tablename__ = "zimfarm_recipe" + id: Mapped[UUID] = mapped_column(primary_key=True) + name: Mapped[str] + + title_id: Mapped[UUID | None] = mapped_column( + ForeignKey("title.id", ondelete="SET NULL"), init=False + ) + + title: Mapped[Optional["Title"]] = relationship( + init=False, back_populates="zimfarm_recipes" + ) + flavours: Mapped[list["TitleFlavour"]] = relationship( + back_populates="recipe", init=False + ) + history_entries: Mapped[list["ZimfarmRecipeHistory"]] = relationship( + back_populates="zimfarm_recipe", + cascade="all, delete", + passive_deletes=True, + init=False, + default_factory=list, + # return the history entries in descending order of created_at + order_by="ZimfarmRecipeHistory.created_at.desc()", + ) + + +class ZimfarmRecipeHistory(Base): + __tablename__ = "zimfarm_recipe_history" + id: Mapped[UUID] = mapped_column( + init=False, primary_key=True, server_default=text("uuid_generate_v4()") + ) + title_name: Mapped[str | None] + title_id: Mapped[UUID | None] + comment: Mapped[str | None] + created_at: Mapped[datetime] = mapped_column( + default_factory=getnow, server_default=func.now() + ) + zimfarm_recipe_id: Mapped[UUID] = mapped_column( + ForeignKey("zimfarm_recipe.id", ondelete="CASCADE"), init=False + ) + author_id: Mapped[UUID] = mapped_column(ForeignKey("account.id"), init=False) + flavours: Mapped[list[str]] = mapped_column( + server_default="{}", default_factory=list + ) + author: Mapped["Account"] = relationship(init=False) + zimfarm_recipe: Mapped["ZimfarmRecipe"] = relationship( + back_populates="history_entries", init=False + ) diff --git a/backend/src/cms_backend/db/title.py b/backend/src/cms_backend/db/title.py index 55ee0a7..3a5b915 100644 --- a/backend/src/cms_backend/db/title.py +++ b/backend/src/cms_backend/db/title.py @@ -21,6 +21,7 @@ from cms_backend.db.collection import get_collection_by_name from cms_backend.db.event import create_title_modified_event from cms_backend.db.exceptions import RecordAlreadyExistsError, RecordDoesNotExistError +from cms_backend.db.flavour import create_title_flavour_schema, get_title_flavours from cms_backend.db.models import ( Collection, CollectionTitle, @@ -48,6 +49,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: """Create a full schema of a title.""" + title_flavours = get_title_flavours(title) return TitleFullSchema( id=title.id, name=title.name, @@ -63,7 +65,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: license=title.license, relation=title.relation, source=title.source, - flavours=title.flavours, + flavours=[create_title_flavour_schema(tf) for tf in title.flavours], books=[ BookLightSchema( id=book.id, @@ -78,7 +80,7 @@ def create_title_full_schema(title: Title) -> TitleFullSchema: date=book.date, flavour=book.flavour, issues=book.issues, - has_flavour_mismatch=has_flavour_mismatch(book.flavour, title.flavours), + has_flavour_mismatch=has_flavour_mismatch(book.flavour, title_flavours), ) for book in sorted( title.books, @@ -122,7 +124,6 @@ def create_title_light_schema(title: Title) -> TitleLightSchema: license=title.license, relation=title.relation, source=title.source, - flavours=title.flavours, ) @@ -130,7 +131,12 @@ def get_title_by_id_or_none(session: OrmSession, *, title_id: UUID) -> Title | N """Get a title by ID""" return session.scalars( select(Title) - .options(selectinload(Title.books), selectinload(Title.collections)) + .options( + selectinload(Title.books), + selectinload(Title.collections), + selectinload(Title.zimfarm_recipes), + selectinload(Title.flavours), + ) .where(Title.id == title_id) ).one_or_none() @@ -149,7 +155,12 @@ def get_title_by_name_or_none(session: OrmSession, *, name: str) -> Title | None return session.scalars( select(Title) - .options(selectinload(Title.books), selectinload(Title.collections)) + .options( + selectinload(Title.books), + selectinload(Title.collections), + selectinload(Title.zimfarm_recipes), + selectinload(Title.flavours), + ) .where(Title.name == name) ).one_or_none() @@ -203,7 +214,6 @@ def get_titles( Title.license.label("title_license"), Title.relation.label("title_relation"), Title.source.label("title_source"), - Title.flavours.label("title_flavours"), ) .join(CollectionTitle, CollectionTitle.title_id == Title.id, isouter=True) .join(Collection, CollectionTitle.collection_id == Collection.id, isouter=True) @@ -247,7 +257,6 @@ def get_titles( license=title_license, relation=title_relation, source=title_source, - flavours=title_flavours, ) for ( title_id, @@ -264,14 +273,17 @@ def get_titles( title_license, title_relation, title_source, - title_flavours, ) in session.execute(stmt.offset(skip).limit(limit)).all() ], ) def create_title( - session: OrmSession, *, author_id: UUID, payload: TitleCreateSchema + session: OrmSession, + *, + author_id: UUID, + payload: TitleCreateSchema, + create_event: bool = True, ) -> Title: """Create a new title""" @@ -290,7 +302,6 @@ def create_title( title.source = payload.source title.description = payload.description title.long_description = payload.long_description - title.flavours = [] if payload.flavours is None else payload.flavours title.events.append(f"{getnow()}: title created") if payload.collection_titles: @@ -322,9 +333,10 @@ def create_title( logger.exception("Unknown exception encountered while creating title") raise - create_title_modified_event( - session, action="created", title_name=title.name, title_id=title.id - ) + if create_event: + create_title_modified_event( + session, action="created", title_name=title.name, title_id=title.id + ) return title @@ -347,7 +359,6 @@ def create_title_history_entry( source=title.source, maturity=title.maturity, archived=title.archived, - flavours=title.flavours, collection_titles=[ { "collection_name": ct.collection.name, @@ -368,6 +379,7 @@ def update_title( title_identifier: str, author_id: UUID, payload: TitleUpdateSchema, + create_event: bool = True, ) -> Title: """Update a title's details @@ -393,7 +405,9 @@ def update_title( raise RecordDoesNotExistError("Title is not archived.") update_data = payload.model_dump( - exclude_unset=True, exclude={"collection_titles", "comment"}, mode="json" + exclude_unset=True, + exclude={"collection_titles", "comment", "flavours"}, + mode="json", ) name_changed = payload.name is not None and payload.name != title.name @@ -434,6 +448,7 @@ def update_title( session.delete(tc) title.collections.clear() + session.flush() for entry in payload.collection_titles: collection = get_collection_by_name( @@ -487,7 +502,7 @@ def update_title( f"{getnow()}: locations updated due to title collection change" ) - if name_changed: + if name_changed and create_event: create_title_modified_event( session, action="updated", title_name=title.name, title_id=title.id ) @@ -614,7 +629,6 @@ def create_title_history_schema(entry: TitleHistory) -> TitleHistorySchema: license=entry.license, relation=entry.relation, source=entry.source, - flavours=entry.flavours, comment=entry.comment, collections=[ BaseTitleCollectionSchema( @@ -707,7 +721,6 @@ def revert_title( publisher=entry.publisher, language=entry.language, illustration_48x48_at_1=entry.illustration_48x48_at_1, - flavours=entry.flavours, ), ) return title diff --git a/backend/src/cms_backend/db/zimfarm_recipe.py b/backend/src/cms_backend/db/zimfarm_recipe.py new file mode 100644 index 0000000..82bf610 --- /dev/null +++ b/backend/src/cms_backend/db/zimfarm_recipe.py @@ -0,0 +1,311 @@ +from uuid import UUID + +from sqlalchemy import select, update +from sqlalchemy.orm import Session as OrmSession +from sqlalchemy.orm import selectinload +from sqlalchemy.orm.attributes import flag_modified + +from cms_backend.db import count_from_stmt +from cms_backend.db.account import get_account_by_username_or_none +from cms_backend.db.event import create_title_modified_event +from cms_backend.db.exceptions import RecordDoesNotExistError +from cms_backend.db.flavour import get_title_flavour_or_none +from cms_backend.db.models import ( + Account, + Book, + Title, + TitleFlavour, + ZimfarmRecipe, + ZimfarmRecipeHistory, +) +from cms_backend.db.title import get_title_by_id +from cms_backend.schemas.orms import ( + ListResult, + TitleFlavourSchema, + ZimfarmRecipeFullSchema, + ZimfarmRecipeHistorySchema, + ZimfarmRecipeLightSchema, +) +from cms_backend.utils import is_valid_uuid +from cms_backend.utils.datetime import getnow + + +def get_zimfarm_recipe_by_id_or_none( + session: OrmSession, recipe_id: UUID +) -> ZimfarmRecipe | None: + """Get a zimfarm recipe by ID if possible else None""" + return session.scalars( + select(ZimfarmRecipe) + .where(ZimfarmRecipe.id == recipe_id) + .options(selectinload(ZimfarmRecipe.title)) + ).one_or_none() + + +def get_zimfarm_recipe_by_name_or_none( + session: OrmSession, recipe_name: str +) -> ZimfarmRecipe | None: + """Get a zimfarm recipe by ID if possible else None""" + return session.scalars( + select(ZimfarmRecipe) + .where(ZimfarmRecipe.name == recipe_name) + .options( + selectinload(ZimfarmRecipe.title), selectinload(ZimfarmRecipe.flavours) + ) + ).one_or_none() + + +def get_zimfarm_recipe(session: OrmSession, recipe_identifier: str): + """Get a zimfarm recipe by ID/name if possible else raise an exception""" + if is_valid_uuid(recipe_identifier): + zimfarm_recipe = get_zimfarm_recipe_by_id_or_none( + session, UUID(recipe_identifier) + ) + else: + zimfarm_recipe = get_zimfarm_recipe_by_name_or_none(session, recipe_identifier) + + if zimfarm_recipe is None: + raise RecordDoesNotExistError( + f"Zimfarm recipe {recipe_identifier} does not exist" + ) + return zimfarm_recipe + + +def create_zimfarm_recipe_schema( + zimfarm_recipe: ZimfarmRecipe, +) -> ZimfarmRecipeFullSchema: + return ZimfarmRecipeFullSchema( + id=zimfarm_recipe.id, + name=zimfarm_recipe.name, + flavours=[ + TitleFlavourSchema(flavour=tf.flavour, recipe_id=tf.recipe_id) + for tf in zimfarm_recipe.flavours + ], + title_id=zimfarm_recipe.title.id if zimfarm_recipe.title else None, + title_name=zimfarm_recipe.title.name if zimfarm_recipe.title else None, + ) + + +def create_zimfarm_recipe( + session: OrmSession, + *, + recipe_id: str, + recipe_name: str, + title_id: UUID | None | None = None, +) -> ZimfarmRecipe: + """Create a zimfarm recipe in the DB.""" + zimfarm_recipe = ZimfarmRecipe( + id=UUID(recipe_id), + name=recipe_name, + ) + if title_id is not None: + title = get_title_by_id(session, title_id=title_id) + zimfarm_recipe.title = title + session.add(zimfarm_recipe) + session.flush() + maint_scripts = get_account_by_username_or_none(session, username="maint-scripts") + if maint_scripts: + create_zimfarm_recipe_history_entry( + session, + zimfarm_recipe, + maint_scripts.id, + comment="Initial history created by maint-scripts", + ) + return zimfarm_recipe + + +def update_zimfarm_recipe( + session: OrmSession, + *, + recipe: ZimfarmRecipe, + flavours: list[str], + title: Title, + old_recipes: set[UUID], + create_event: bool = True, + author: Account, +) -> ZimfarmRecipe: + """Update a recipe to be associated with the title and flavours. + + - Existing associations with the title's flavours to other recipes + are removed. These other recipe(s) must be in the list of old recipes. + - Old title flavours belonging to the title are deleted and new ones are created + and attached to recipe. + - Books with a matching title name have recipe issues removed + """ + + associated_recipes: set[UUID] = set() + + for flavour in flavours: + title_flavour = get_title_flavour_or_none(session, title.id, flavour) + if title_flavour: + associated_recipes.add(title_flavour.recipe_id) + + if title_flavour.recipe_id != recipe.id: + title_flavour.recipe = recipe + else: + title_flavour = TitleFlavour(flavour=flavour) + title_flavour.recipe = recipe + title_flavour.title = title + session.add(title_flavour) + + if associated_recipes != old_recipes: + raise ValueError("Mismatch between old recipes and title flavour recipes ") + + # Clear title id from recipes with no remaining flavours + recipes_with_flavours: set[UUID] = set( + session.scalars( + select(TitleFlavour.recipe_id).where(TitleFlavour.title_id == title.id) + ) + ) + all_title_recipes: set[UUID] = {recipe.id for recipe in title.zimfarm_recipes} + recipes_to_clear = all_title_recipes - recipes_with_flavours + if recipes_to_clear: + session.execute( + update(ZimfarmRecipe) + .where(ZimfarmRecipe.id.in_(recipes_to_clear)) + .values({"title_id": None}) + ) + + recipe.title = title + + # Create history entries for all the recipes that have been affected + for associated_recipe_id in associated_recipes: + associated_recipe = get_zimfarm_recipe(session, str(associated_recipe_id)) + create_zimfarm_recipe_history_entry( + session, associated_recipe, author.id, comment=None + ) + + if recipe.id not in associated_recipes: + create_zimfarm_recipe_history_entry(session, recipe, author.id, comment=None) + + session.flush() + + if create_event: + # Optimistically remove recipe issues from books that will be processed by the + # event + books = session.scalars( + select(Book).where( + Book.title_id.is_(None), + Book.has_error.is_(False), + Book.name == title.name, + Book.location_kind.not_in(["deleted", "to_delete"]), + Book.issues.contains(["recipe issue"]), + ) + ).all() + for book in books: + book.issues.remove("recipe issue") + flag_modified(book, "issues") + + create_title_modified_event( + session, action="updated", title_name=title.name, title_id=title.id + ) + return recipe + + +def get_zimfarm_recipes( + session: OrmSession, + *, + skip: int, + limit: int, + name: str | None = None, +) -> ListResult[ZimfarmRecipeLightSchema]: + """Get a list of recipes""" + + stmt = select( + ZimfarmRecipe.id.label("recipe_id"), ZimfarmRecipe.name.label("recipe_name") + ).where( + ( + ZimfarmRecipe.name.ilike(f"%{name if name is not None else ''}%") + | (name is None) + ), + ) + + return ListResult[ZimfarmRecipeLightSchema]( + nb_records=count_from_stmt(session, stmt), + records=[ + ZimfarmRecipeLightSchema( + id=recipe_id, + name=recipe_name, + ) + for recipe_id, recipe_name in session.execute( + stmt.offset(skip).limit(limit) + ).all() + ], + ) + + +def create_zimfarm_recipe_history_entry( + session: OrmSession, + recipe: ZimfarmRecipe, + author_id: UUID, + comment: str | None = None, +) -> ZimfarmRecipeHistory: + history_entry = ZimfarmRecipeHistory( + title_name=recipe.title.name if recipe.title else None, + title_id=recipe.title_id, + comment=comment, + created_at=getnow(), + flavours=[tf.flavour for tf in recipe.flavours], + ) + history_entry.zimfarm_recipe = recipe + history_entry.author_id = author_id + session.add(history_entry) + return history_entry + + +def create_zimfarm_recipe_history_schema(entry: ZimfarmRecipeHistory): + return ZimfarmRecipeHistorySchema( + id=entry.id, + title_id=entry.title_id, + title_name=entry.title_name, + flavours=entry.flavours, + created_at=entry.created_at, + author=entry.author.display_name, + comment=entry.comment, + ) + + +def get_zimfarm_recipe_history( + session: OrmSession, *, recipe_identifier: str, skip: int, limit: int +) -> ListResult[ZimfarmRecipeHistorySchema]: + """Get a zimfarm recipe's history""" + recipe = get_zimfarm_recipe(session, recipe_identifier) + stmt = ( + select(ZimfarmRecipeHistory) + .where(ZimfarmRecipeHistory.zimfarm_recipe_id == recipe.id) + .options(selectinload(ZimfarmRecipeHistory.author)) + .order_by(ZimfarmRecipeHistory.created_at.desc()) + ) + return ListResult[ZimfarmRecipeHistorySchema]( + nb_records=count_from_stmt(session, stmt), + records=[ + create_zimfarm_recipe_history_schema(entry) + for entry in session.scalars(stmt.offset(skip).limit(limit)).all() + ], + ) + + +def get_zimfarm_recipe_history_entry_or_none( + session: OrmSession, *, recipe_identifier: str, history_id: UUID +) -> ZimfarmRecipeHistory | None: + """Get a zimfarm recipe's history entry or None if it does not exist""" + recipe = get_zimfarm_recipe(session, recipe_identifier) + return session.scalars( + select(ZimfarmRecipeHistory).where( + ZimfarmRecipeHistory.id == history_id, + ZimfarmRecipeHistory.zimfarm_recipe_id == recipe.id, + ) + ).one_or_none() + + +def get_zimfarm_recipe_history_entry( + session: OrmSession, *, recipe_identifier: str, history_id: UUID +) -> ZimfarmRecipeHistory: + """Get a zimfarm recipe's history entry""" + if history_entry := get_zimfarm_recipe_history_entry_or_none( + session, recipe_identifier=recipe_identifier, history_id=history_id + ): + return history_entry + raise RecordDoesNotExistError( + f"Recipe '{recipe_identifier}' does not have a history entry with id " + f"{history_id}" + ) diff --git a/backend/src/cms_backend/migrations/versions/0c0f13d5733b_create_zimfarm_recipe_and_title_flavour_.py b/backend/src/cms_backend/migrations/versions/0c0f13d5733b_create_zimfarm_recipe_and_title_flavour_.py new file mode 100644 index 0000000..ff44aa1 --- /dev/null +++ b/backend/src/cms_backend/migrations/versions/0c0f13d5733b_create_zimfarm_recipe_and_title_flavour_.py @@ -0,0 +1,119 @@ +"""alembic create recipe and title_flavour table + +Revision ID: 0c0f13d5733b +Revises: a1a5ffedaf5b +Create Date: 2026-06-11 13:15:16.854082 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "0c0f13d5733b" +down_revision = "a1a5ffedaf5b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "zimfarm_recipe", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("title_id", sa.Uuid(), nullable=True), + sa.ForeignKeyConstraint( + ["title_id"], + ["title.id"], + name=op.f("fk_zimfarm_recipe_title_id_title"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_zimfarm_recipe")), + ) + op.create_table( + "title_flavour", + sa.Column("title_id", sa.Uuid(), nullable=False), + sa.Column("flavour", sa.String(), nullable=False), + sa.Column("recipe_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["recipe_id"], + ["zimfarm_recipe.id"], + name=op.f("fk_title_flavour_recipe_id_zimfarm_recipe"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["title_id"], + ["title.id"], + name=op.f("fk_title_flavour_title_id_title"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("title_id", "flavour", name=op.f("pk_title_flavour")), + ) + op.create_table( + "zimfarm_recipe_history", + sa.Column( + "id", + sa.Uuid(), + server_default=sa.text("uuid_generate_v4()"), + nullable=False, + ), + sa.Column("title_name", sa.String(), nullable=True), + sa.Column("title_id", sa.Uuid(), nullable=True), + sa.Column("comment", sa.String(), nullable=True), + sa.Column( + "created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.Column("zimfarm_recipe_id", sa.Uuid(), nullable=False), + sa.Column("author_id", sa.Uuid(), nullable=False), + sa.Column( + "flavours", + postgresql.ARRAY(sa.String()), + server_default="{}", + nullable=False, + ), + sa.ForeignKeyConstraint( + ["author_id"], + ["account.id"], + name=op.f("fk_zimfarm_recipe_history_author_id_account"), + ), + sa.ForeignKeyConstraint( + ["zimfarm_recipe_id"], + ["zimfarm_recipe.id"], + name=op.f("fk_zimfarm_recipe_history_zimfarm_recipe_id_zimfarm_recipe"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_zimfarm_recipe_history")), + ) + op.drop_column("title_history", "flavours") + op.drop_column("title", "flavours") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "title_history", + sa.Column( + "flavours", + postgresql.ARRAY(sa.VARCHAR()), + server_default=sa.text("'{}'::character varying[]"), + autoincrement=False, + nullable=False, + ), + ) + op.add_column( + "title", + sa.Column( + "flavours", + postgresql.ARRAY(sa.VARCHAR()), + server_default=sa.text("'{}'::character varying[]"), + autoincrement=False, + nullable=False, + ), + ) + op.drop_table("zimfarm_recipe_history") + op.drop_table("title_flavour") + op.drop_table("zimfarm_recipe") + # ### end Alembic commands ### diff --git a/backend/src/cms_backend/mill/processors/title.py b/backend/src/cms_backend/mill/processors/title.py index 2ed9c5a..84f8c22 100644 --- a/backend/src/cms_backend/mill/processors/title.py +++ b/backend/src/cms_backend/mill/processors/title.py @@ -2,6 +2,7 @@ from cms_backend import logger from cms_backend.db.book import ( + book_has_recipe_issue, process_book, ) from cms_backend.db.models import Book, Title @@ -9,29 +10,89 @@ apply_retention_rules, title_is_missing_mandatory_metadata, ) +from cms_backend.db.zimfarm_recipe import ( + create_zimfarm_recipe, + get_zimfarm_recipe_by_id_or_none, +) from cms_backend.utils.datetime import getnow from cms_backend.utils.filename import compute_target_filename +from cms_backend.utils.zim import get_missing_keys def add_book_to_title(session: OrmSession, book: Book, title: Title): + """ + Associate a book with a given title, create target locations for move operations. + + When a book is to be added to a book title newly: + - if recipe exists and its configuration matches book name and flavour, proceed + to add it to the title + - if recipe exists and its configuration is NOT matching book name and flavour, + block the book in staging without an associated title and set issue to + 'recipe issue' + - if recipe does not exist, create the recipe (without associated title), block the + book in staging without an associated title and set issue to 'recipe issue' + """ try: # Retrieve name from book.name directly if not book.name: - raise Exception("book name is missing or invalid") + raise ValueError("book name is missing or invalid") # Validate book.date is also present and valid if not book.date: - raise Exception("book date is missing or invalid") + raise ValueError("book date is missing or invalid") + + if not book.zimfarm_notification: + raise ValueError("book is missing zimfarm notification") + + missing_recipe_keys = get_missing_keys( + book.zimfarm_notification.content, "recipe_id", "recipe_name" + ) + if missing_recipe_keys: + raise ValueError( + "book notification is missing recipe details: " + f"{','.join(missing_recipe_keys)}" + ) + + content = book.zimfarm_notification.content + recipe = get_zimfarm_recipe_by_id_or_none(session, content["recipe_id"]) + if recipe is None: + recipe = create_zimfarm_recipe( + session, + recipe_id=content["recipe_id"], + recipe_name=content["recipe_name"], + ) + recipe.name = content["recipe_name"] - title.books.append(book) - book.events.append(f"{getnow()}: book added to title {title.id}") - title.events.append(f"{getnow()}: book {book.id} added to title") + if book_has_recipe_issue(book.flavour, title, recipe): + book.issues = ["recipe issue"] + book.events.append( + f"{getnow()}: cannot add book to title {title.id} " + "because of recipe issue" + ) + else: + title.books.append(book) + book.events.append(f"{getnow()}: book added to title {title.id}") + title.events.append(f"{getnow()}: book {book.id} added to title") + # Update title name should it have changed (e.g. stackexchange domain + # updated + # leading to ZIM name automatically updated as well) + if title.name != book.name: + title.events.append(f"{getnow()}: updating title name to {book.name}") + title.name = book.name - # Update title name should it have changed (e.g. stackexchange domain updated - # leading to ZIM name automatically updated as well) - if title.name != book.name: - title.events.append(f"{getnow()}: updating title name to {book.name}") - title.name = book.name + if title_is_missing_mandatory_metadata(title): + title.title = book.zim_metadata["Title"] + title.creator = book.zim_metadata["Creator"] + title.publisher = book.zim_metadata["Publisher"] + title.description = book.zim_metadata["Description"] + title.language = book.zim_metadata["Language"] + title.illustration_48x48_at_1 = book.zim_metadata[ + "Illustration_48x48@1" + ] + title.long_description = book.zim_metadata.get("LongDescription") + title.license = book.zim_metadata.get("License") + title.relation = book.zim_metadata.get("Relation") + title.source = book.zim_metadata.get("Source") # Compute target filename once for this book book.filename = compute_target_filename( @@ -41,22 +102,8 @@ def add_book_to_title(session: OrmSession, book: Book, title: Title): date=book.date, book_id=book.id, ) - - if title_is_missing_mandatory_metadata(title): - title.title = book.zim_metadata["Title"] - title.creator = book.zim_metadata["Creator"] - title.publisher = book.zim_metadata["Publisher"] - title.description = book.zim_metadata["Description"] - title.language = book.zim_metadata["Language"] - title.illustration_48x48_at_1 = book.zim_metadata["Illustration_48x48@1"] - title.long_description = book.zim_metadata.get("LongDescription") - title.license = book.zim_metadata.get("License") - title.relation = book.zim_metadata.get("Relation") - title.source = book.zim_metadata.get("Source") - process_book(session, book, update_events=True) - if book.location_kind == "prod": - apply_retention_rules(session, title) + apply_retention_rules(session, title) except Exception as exc: book.events.append( diff --git a/backend/src/cms_backend/mill/processors/zimfarm_notification.py b/backend/src/cms_backend/mill/processors/zimfarm_notification.py index 228a593..f196f55 100644 --- a/backend/src/cms_backend/mill/processors/zimfarm_notification.py +++ b/backend/src/cms_backend/mill/processors/zimfarm_notification.py @@ -30,6 +30,8 @@ def process_notification(session: ORMSession, notification: ZimfarmNotification) "zimcheck_url", "folder_name", "filename", + "recipe_id", + "recipe_name", ] if key not in notification.content ] diff --git a/backend/src/cms_backend/roles.py b/backend/src/cms_backend/roles.py index 4f66ec2..4a322ae 100644 --- a/backend/src/cms_backend/roles.py +++ b/backend/src/cms_backend/roles.py @@ -36,6 +36,7 @@ class RoleEnum(StrEnum): "zimfarm_notification": ResourcePermissions.get_all(), "account": ResourcePermissions.get_all(), "collection": ResourcePermissions.get_all(), + "recipe": ResourcePermissions.get_all(), }, RoleEnum.ZIMFARM: { "zimfarm_notification": ResourcePermissions.get(read=True, create=True), diff --git a/backend/src/cms_backend/api/routes/fields.py b/backend/src/cms_backend/schemas/fields.py similarity index 100% rename from backend/src/cms_backend/api/routes/fields.py rename to backend/src/cms_backend/schemas/fields.py diff --git a/backend/src/cms_backend/schemas/models.py b/backend/src/cms_backend/schemas/models.py index d8d60d6..205ec75 100644 --- a/backend/src/cms_backend/schemas/models.py +++ b/backend/src/cms_backend/schemas/models.py @@ -5,9 +5,9 @@ from pydantic import AnyUrl, Field, model_validator -from cms_backend.api.routes.fields import Base64Str, LangCode, NotEmptyString from cms_backend.roles import RoleEnum from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import Base64Str, LangCode, NotEmptyString from cms_backend.schemas.orms import BaseTitleCollectionSchema @@ -67,7 +67,6 @@ class BaseTitleCreateUpdateSchema(BaseModel): publisher: NotEmptyString | None = None language: LangCode | None = None illustration_48x48_at_1: Base64Str | None = None - flavours: list[str] | None = None archived: bool | None = None @model_validator(mode="after") diff --git a/backend/src/cms_backend/schemas/orms.py b/backend/src/cms_backend/schemas/orms.py index 692a1c0..8a2f7ba 100644 --- a/backend/src/cms_backend/schemas/orms.py +++ b/backend/src/cms_backend/schemas/orms.py @@ -3,8 +3,11 @@ from typing import Any, TypeVar from uuid import UUID -from cms_backend.api.routes.fields import NotEmptyString +from pydantic import computed_field + +from cms_backend.context import Context from cms_backend.schemas import BaseModel +from cms_backend.schemas.fields import NotEmptyString T = TypeVar("T") @@ -33,7 +36,6 @@ class TitleLightSchema(BaseModel): license: str | None relation: str | None source: str | None - flavours: list[str] class BaseTitleCollectionSchema(BaseModel): @@ -45,6 +47,11 @@ class TitleCollectionSchema(BaseTitleCollectionSchema): collection_id: UUID +class TitleFlavourSchema(BaseModel): + flavour: str + recipe_id: UUID + + class TitleFullSchema(TitleLightSchema): """ Schema for reading a title model with all fields including books @@ -53,6 +60,7 @@ class TitleFullSchema(TitleLightSchema): events: list[str] books: list["BookLightSchema"] collections: list["TitleCollectionSchema"] + flavours: list[TitleFlavourSchema] class TitleHistorySchema(TitleLightSchema): @@ -146,6 +154,7 @@ class BookFullSchema(BookLightSchema): target_locations: list[BookLocationSchema] title_archived: bool has_backup: bool + recipe_id: UUID | None class BookHistorySchema(BaseModel): @@ -210,3 +219,29 @@ class EventLightSchema(BaseModel): id: UUID created_at: datetime topic: str + + +class ZimfarmRecipeLightSchema(BaseModel): + id: UUID + name: str + + +class ZimfarmRecipeFullSchema(ZimfarmRecipeLightSchema): + title_id: UUID | None + title_name: str | None + flavours: list[TitleFlavourSchema] + + @computed_field + @property + def link(self) -> str: + return f"{Context.zimfarm_api_url}/recipes/{self.name}" + + +class ZimfarmRecipeHistorySchema(BaseModel): + id: UUID + title_id: UUID | None + title_name: str | None + flavours: list[str] + comment: str | None + author: str + created_at: datetime diff --git a/backend/tests/api/routes/test_titles.py b/backend/tests/api/routes/test_titles.py index 3a8b398..3404522 100644 --- a/backend/tests/api/routes/test_titles.py +++ b/backend/tests/api/routes/test_titles.py @@ -18,6 +18,7 @@ Event, Title, Warehouse, + ZimfarmRecipe, ) from cms_backend.db.title import update_title from cms_backend.roles import RoleEnum @@ -70,7 +71,6 @@ def test_get_titles( "relation", "source", "license", - "flavours", } assert data["items"][0]["name"] == "wikipedia_fr_all" @@ -771,13 +771,14 @@ def test_get_title_history_entry( def test_revert_title_required_permissions( dbsession: OrmSession, client: TestClient, + zimfarm_recipe: ZimfarmRecipe, create_account: Callable[..., Account], create_title: Callable[..., Title], permission: RoleEnum, expected_status_code: HTTPStatus, ): """Test reverting a title with different roles""" - title = create_title(name="wikipedia_en_test") + title = create_title(name="wikipedia_en_test", zimfarm_recipe=zimfarm_recipe) account = create_account(permission=permission) access_token = generate_access_token( account_id=str(account.id), issue_time=getnow() @@ -818,6 +819,7 @@ def test_merge_titles_required_permissions( create_collection: Callable[..., Collection], monkeypatch: pytest.MonkeyPatch, create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], permission: RoleEnum, expected_status_code: HTTPStatus, ): @@ -845,6 +847,7 @@ def test_merge_titles_required_permissions( ), } + recipe1 = create_zimfarm_recipe() title1 = create_title( name="test_en_all", flavours=["maxi", "mini"], @@ -854,6 +857,7 @@ def test_merge_titles_required_permissions( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe1, ) book1 = create_book(zim_metadata=content, location_kind="staging") book1.title = title1 @@ -868,6 +872,7 @@ def test_merge_titles_required_permissions( content2 = content.copy() content2["Name"] = "test_eng_all" content2["Date"] = "2025-02-02" + recipe2 = create_zimfarm_recipe() title2 = create_title( name="test_eng_all", flavours=["maxi", "mini"], @@ -877,6 +882,7 @@ def test_merge_titles_required_permissions( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe2, ) book2 = create_book(zim_metadata=content2, location_kind="staging") book2.title = title2 diff --git a/backend/tests/api/routes/test_zimfarm_recipes.py b/backend/tests/api/routes/test_zimfarm_recipes.py new file mode 100644 index 0000000..1e75580 --- /dev/null +++ b/backend/tests/api/routes/test_zimfarm_recipes.py @@ -0,0 +1,134 @@ +from collections.abc import Callable +from http import HTTPStatus +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session as OrmSession + +from cms_backend.api.token import generate_access_token +from cms_backend.db.models import Account, Title, ZimfarmRecipe +from cms_backend.db.zimfarm_recipe import update_zimfarm_recipe +from cms_backend.roles import RoleEnum +from cms_backend.utils.datetime import getnow + + +@pytest.mark.parametrize( + "skip, limit, expected_count", + [ + pytest.param(0, 3, 3, id="first-page"), + pytest.param(3, 3, 3, id="second-page"), + pytest.param(6, 2, 0, id="page-num-too-high-no-results"), + pytest.param(0, 1, 1, id="first-page-with-low-limit"), + pytest.param(0, 10, 6, id="first-page-with-high-limit"), + ], +) +def test_get_zimfarm_recipes( + client: TestClient, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + skip: int, + limit: int, + expected_count: int, +): + for _ in range(6): + create_zimfarm_recipe() + + response = client.get(f"/v1/recipes?skip={skip}&limit={limit}") + assert response.status_code == HTTPStatus.OK + data = response.json() + assert data["meta"]["skip"] == skip + assert data["meta"]["limit"] == limit + assert data["meta"]["page_size"] == expected_count + assert len(data["items"]) == expected_count + + +def test_get_zimfarm_recipe_not_found( + client: TestClient, +): + response = client.get(f"/v1/recipes/{uuid4()}") + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_get_zimfarm_recipe( + client: TestClient, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], +): + recipe = create_zimfarm_recipe() + response = client.get(f"/v1/recipes/{recipe.name}") + assert response.status_code == HTTPStatus.OK + + +@pytest.mark.parametrize( + "skip, limit, expected_count", + [ + pytest.param(0, 3, 3, id="first-page"), + pytest.param(3, 3, 3, id="second-page"), + pytest.param(6, 2, 0, id="page-num-too-high-no-results"), + pytest.param(0, 1, 1, id="first-page-with-low-limit"), + pytest.param(0, 10, 6, id="first-page-with-high-limit"), + ], +) +def test_get_zimfarm_recipe_history( + dbsession: OrmSession, + client: TestClient, + create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + access_token: str, + account: Account, + skip: int, + limit: int, + expected_count: int, +): + """Test retrieving zimfarm recipe history""" + title = create_title(name="wikipedia_en_test") + recipe = create_zimfarm_recipe(title_id=title.id) + for flavour in ["mini", "maxi", "nopic", "", "novid"]: + update_zimfarm_recipe( + dbsession, + recipe=recipe, + author=account, + flavours=[flavour], + title=title, + old_recipes=set(), + create_event=False, + ) + response = client.get( + f"/v1/recipes/{recipe.id}/history?skip={skip}&limit={limit}", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == HTTPStatus.OK + data = response.json() + assert data["meta"]["skip"] == skip + assert data["meta"]["limit"] == limit + assert data["meta"]["page_size"] == expected_count + assert len(data["items"]) == expected_count + + +@pytest.mark.parametrize( + "permission,expected_status_code", + [ + pytest.param(RoleEnum.EDITOR, HTTPStatus.OK, id="editor"), + pytest.param(RoleEnum.VIEWER, HTTPStatus.UNAUTHORIZED, id="viewer"), + ], +) +def test_get_zimfarm_recipe_history_required_permissions( + client: TestClient, + create_account: Callable[..., Account], + create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + permission: RoleEnum, + expected_status_code: HTTPStatus, +): + """Test retrieving zimfarm recipe history with different roles""" + title = create_title(name="wikipedia_en_test") + recipe = create_zimfarm_recipe(title_id=title.id) + + account = create_account(permission=permission) + access_token = generate_access_token( + account_id=str(account.id), issue_time=getnow() + ) + response = client.get( + f"/v1/recipes/{recipe.name}/history?skip=0&limit=10", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == expected_status_code diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 298f88d..562933e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from faker import Faker +from sqlalchemy import select from sqlalchemy.orm import Session as OrmSession from werkzeug.security import generate_password_hash @@ -23,9 +24,12 @@ CollectionTitle, Event, Title, + TitleFlavour, TitleHistory, Warehouse, ZimfarmNotification, + ZimfarmRecipe, + ZimfarmRecipeHistory, ) from cms_backend.roles import RoleEnum from cms_backend.utils.datetime import getnow @@ -184,6 +188,7 @@ def _create_title( relation: str | None = None, source: str | None = None, flavours: list[str] | None = None, + zimfarm_recipe: ZimfarmRecipe | None = None, ) -> Title: db_title = Title( name=name, @@ -198,8 +203,15 @@ def _create_title( license=license, relation=relation, source=source, - flavours=flavours if flavours is not None else [], ) + if flavours: + for flavour in flavours: + title_flavour = TitleFlavour(flavour=flavour) + if zimfarm_recipe: + title_flavour.recipe = zimfarm_recipe + title_flavour.title = db_title + dbsession.add(title_flavour) + history_entry = TitleHistory( name=name, title=title, @@ -213,14 +225,19 @@ def _create_title( license=license, relation=relation, source=source, - flavours=flavours if flavours is not None else [], comment="Initial history entry", ) history_entry.author_id = account.id history_entry.title_ = db_title - dbsession.add(db_title) dbsession.flush() + + if zimfarm_recipe: + zimfarm_recipe.title = db_title + dbsession.add(zimfarm_recipe) + + dbsession.flush() + return db_title return _create_title @@ -471,6 +488,52 @@ def _create_event( return _create_event +@pytest.fixture +def create_zimfarm_recipe( + dbsession: OrmSession, + faker: Faker, + account: Account, +) -> Callable[..., ZimfarmRecipe]: + def _create_zimfarm_recipe( + *, + recipe_id: UUID | None = None, + recipe_name: str | None = None, + title_id: UUID | None = None, + ): + recipe = ZimfarmRecipe( + id=recipe_id or UUID(faker.uuid4()), + name=recipe_name or faker.company(), + ) + if title_id: + recipe.title = dbsession.scalars( + select(Title).where(Title.id == title_id) + ).one() + + history = ZimfarmRecipeHistory( + title_id=title_id, + title_name=recipe.title.name if recipe.title else None, + comment=None, + flavours=[tf.flavour for tf in recipe.flavours], + created_at=getnow(), + ) + history.author_id = account.id + history.zimfarm_recipe = recipe + dbsession.add(history) + dbsession.add(recipe) + dbsession.flush() + + return recipe + + return _create_zimfarm_recipe + + +@pytest.fixture +def zimfarm_recipe( + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], faker: Faker +) -> ZimfarmRecipe: + return create_zimfarm_recipe(recipe_id=faker.uuid4(), recipe_name=faker.name()) + + @pytest.fixture() def illustration_48x48_at_1() -> str: return ( diff --git a/backend/tests/db/test_book.py b/backend/tests/db/test_book.py index 5dd7466..c67953d 100644 --- a/backend/tests/db/test_book.py +++ b/backend/tests/db/test_book.py @@ -28,6 +28,7 @@ Title, Warehouse, ZimfarmNotification, + ZimfarmRecipe, ) from cms_backend.db.rules import has_flavour_mismatch from cms_backend.schemas.models import BookUpdateSchema @@ -291,6 +292,7 @@ def test_revert_book( def test_update_book_flavour_mismatch_issues( dbsession: OrmSession, account: Account, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_title: Callable[..., Title], create_book: Callable[..., Book], ): @@ -313,6 +315,7 @@ def test_update_book_flavour_mismatch_issues( ), } + recipe = create_zimfarm_recipe() title = create_title( name="test_en_all", flavours=["maxi", "mini"], @@ -322,6 +325,7 @@ def test_update_book_flavour_mismatch_issues( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe, ) book = create_book(zim_metadata=content) book.title = title @@ -801,6 +805,7 @@ def test_recover_deleted_book_with_no_backup( def test_update_book_issues_item_count_issues( dbsession: OrmSession, create_book: Callable[..., Book], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_book_location: Callable[..., BookLocation], @@ -827,6 +832,7 @@ def test_update_book_issues_item_count_issues( "BJRU5ErkJggg==" ), } + recipe = create_zimfarm_recipe() title = create_title( name="test_en_all", flavours=["maxi", "mini"], @@ -836,6 +842,7 @@ def test_update_book_issues_item_count_issues( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe, ) # create a collection that tolerates only 10% increase in media and article count create_collection( diff --git a/backend/tests/db/test_books.py b/backend/tests/db/test_books.py index 087bde9..7f0ccba 100644 --- a/backend/tests/db/test_books.py +++ b/backend/tests/db/test_books.py @@ -23,6 +23,7 @@ CollectionTitle, Title, Warehouse, + ZimfarmRecipe, ) from cms_backend.utils.datetime import getnow @@ -626,6 +627,7 @@ def test_move_book_staging_to_prod( dbsession: OrmSession, warehouse: Warehouse, create_book: Callable[..., Book], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_collection_title: Callable[..., CollectionTitle], @@ -633,7 +635,10 @@ def test_move_book_staging_to_prod( monkeypatch: pytest.MonkeyPatch, ): """Test moving a book from staging to prod""" - title = create_title(name="test_en_all", flavours=["mini", "maxi"]) + recipe = create_zimfarm_recipe() + title = create_title( + name="test_en_all", flavours=["mini", "maxi"], zimfarm_recipe=recipe + ) collection = create_collection(warehouse=warehouse) create_collection_title(title=title, collection=collection, path=Path("zim")) @@ -670,6 +675,7 @@ def test_move_book_with_different_flavor_from_title_to_prod( dbsession: OrmSession, warehouse: Warehouse, create_book: Callable[..., Book], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_collection_title: Callable[..., CollectionTitle], @@ -677,7 +683,10 @@ def test_move_book_with_different_flavor_from_title_to_prod( monkeypatch: pytest.MonkeyPatch, ): """Test moving a book with different flavor from it's title from staging to prod""" - title = create_title(name="test_en_all", flavours=["mini", "maxi"]) + recipe = create_zimfarm_recipe() + title = create_title( + name="test_en_all", flavours=["mini", "maxi"], zimfarm_recipe=recipe + ) collection = create_collection(warehouse=warehouse) create_collection_title(title=title, collection=collection, path=Path("zim")) diff --git a/backend/tests/db/test_title.py b/backend/tests/db/test_title.py index c45a80c..41704d9 100644 --- a/backend/tests/db/test_title.py +++ b/backend/tests/db/test_title.py @@ -15,6 +15,7 @@ Event, Title, Warehouse, + ZimfarmRecipe, ) from cms_backend.db.title import ( archive_title, @@ -418,11 +419,12 @@ def test_revert_title( create_collection_title: Callable[..., CollectionTitle], account: Account, illustration_48x48_at_1: str, + zimfarm_recipe: ZimfarmRecipe, ): """Test reverting a title to a previous state""" collection1 = create_collection(name="wikipedia") create_collection(name="gutenberg") - title = create_title(name="wikipedia_en_test") + title = create_title(name="wikipedia_en_test", zimfarm_recipe=zimfarm_recipe) create_collection_title(title, collection1, path="wikis") # Create a history with full metadata @@ -441,7 +443,6 @@ def test_revert_title( license="CC-BY-SA 3.0", relation="wikipedia_v1", source="https://en.wikipedia.org/v1", - flavours=["mini", "nopic"], maturity="stable", comment="First version", ), @@ -467,7 +468,6 @@ def test_revert_title( license="CC-BY-SA 4.0", relation="wikipedia_v2", source="https://en.wikipedia.org/v2", - flavours=["maxi"], maturity="unstable", collection_titles=[ BaseTitleCollectionSchema(collection_name="wikipedia", path="wikis"), @@ -487,7 +487,6 @@ def test_revert_title( assert title.license == "CC-BY-SA 4.0" assert title.relation == "wikipedia_v2" assert title.source == "https://en.wikipedia.org/v2" - assert title.flavours == ["maxi"] assert title.maturity == "unstable" assert len(title.collections) == 2 @@ -511,7 +510,6 @@ def test_revert_title( assert reverted_title.license == "CC-BY-SA 3.0" assert reverted_title.relation == "wikipedia_v1" assert reverted_title.source == "https://en.wikipedia.org/v1" - assert reverted_title.flavours == ["mini", "nopic"] assert reverted_title.maturity == "stable" assert len(reverted_title.collections) == 1 @@ -538,6 +536,7 @@ def test_merge_titles_target_in_sources( def test_merge_titles_fail_because_one_source_book_needs_processing( dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_warehouse: Callable[..., Warehouse], create_title: Callable[..., Title], create_book: Callable[..., Book], @@ -564,6 +563,7 @@ def test_merge_titles_fail_because_one_source_book_needs_processing( ), } + recipe1 = create_zimfarm_recipe() title1 = create_title( name="test_en_all", flavours=["maxi", "mini"], @@ -573,6 +573,7 @@ def test_merge_titles_fail_because_one_source_book_needs_processing( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe1, ) book1 = create_book(zim_metadata=content, location_kind="staging") book1.title = title1 @@ -588,6 +589,7 @@ def test_merge_titles_fail_because_one_source_book_needs_processing( content2 = content.copy() content2["Name"] = "test_eng_all" content2["Date"] = "2025-02-02" + recipe2 = create_zimfarm_recipe() title2 = create_title( name="test_eng_all", flavours=["maxi", "mini"], @@ -597,6 +599,7 @@ def test_merge_titles_fail_because_one_source_book_needs_processing( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe2, ) book2 = create_book(zim_metadata=content2, location_kind="staging") book2.title = title2 @@ -621,6 +624,7 @@ def test_merge_titles_fail_because_one_source_book_needs_processing( def test_merge_titles_success( dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], create_warehouse: Callable[..., Warehouse], create_title: Callable[..., Title], create_book: Callable[..., Book], @@ -647,6 +651,7 @@ def test_merge_titles_success( ), } + recipe1 = create_zimfarm_recipe() title1 = create_title( name="test_en_all", flavours=["maxi", "mini"], @@ -656,6 +661,7 @@ def test_merge_titles_success( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe1, ) book1 = create_book(zim_metadata=content, location_kind="staging") book1.title = title1 @@ -670,6 +676,7 @@ def test_merge_titles_success( content2 = content.copy() content2["Name"] = "test_eng_all" content2["Date"] = "2025-02-02" + recipe2 = create_zimfarm_recipe() title2 = create_title( name="test_eng_all", flavours=["maxi", "mini"], @@ -679,6 +686,7 @@ def test_merge_titles_success( description=content["Description"], language=content["Language"], illustration_48x48_at_1=content["Illustration_48x48@1"], + zimfarm_recipe=recipe2, ) book2 = create_book(zim_metadata=content2, location_kind="staging") book2.title = title2 diff --git a/backend/tests/db/test_zimfarm_recipe.py b/backend/tests/db/test_zimfarm_recipe.py new file mode 100644 index 0000000..f6f37bc --- /dev/null +++ b/backend/tests/db/test_zimfarm_recipe.py @@ -0,0 +1,314 @@ +from collections.abc import Callable +from uuid import uuid4 + +import pytest +from faker import Faker +from sqlalchemy import select +from sqlalchemy.orm import Session as OrmSession + +from cms_backend.db import count_from_stmt +from cms_backend.db.exceptions import RecordDoesNotExistError +from cms_backend.db.models import Account, Book, Title, TitleFlavour, ZimfarmRecipe +from cms_backend.db.zimfarm_recipe import ( + create_zimfarm_recipe, + get_zimfarm_recipe, + get_zimfarm_recipe_by_id_or_none, + get_zimfarm_recipe_history, + get_zimfarm_recipe_history_entry_or_none, + update_zimfarm_recipe, +) + + +def test_get_zimfarm_recipe_or_none(dbsession: OrmSession): + assert get_zimfarm_recipe_by_id_or_none(dbsession, uuid4()) is None + + +def test_get_zimfarm_recipe_not_found(dbsession: OrmSession): + with pytest.raises(RecordDoesNotExistError): + get_zimfarm_recipe(dbsession, str(uuid4())) + + +def test_get_zimfarm_recipe( + dbsession: OrmSession, create_zimfarm_recipe: Callable[..., ZimfarmRecipe] +): + recipe = create_zimfarm_recipe(recipe_id=uuid4(), recipe_name="test_recipe") + db_recipe = get_zimfarm_recipe(dbsession, str(recipe.id)) + assert db_recipe.id == recipe.id + + +def test_create_zimfarm_recipe(dbsession: OrmSession, title: Title, faker: Faker): + recipe = create_zimfarm_recipe( + dbsession, recipe_id=faker.uuid4(), recipe_name=faker.name(), title_id=title.id + ) + assert recipe.title is not None + assert recipe.title.id == title.id + + +def test_update_zimfarm_recipe_mismatch_in_recipes( + dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + create_title: Callable[..., Title], + account: Account, +): + recipe = create_zimfarm_recipe() + title = create_title(flavours=["maxi", "mini"], zimfarm_recipe=recipe) + with pytest.raises(ValueError): + update_zimfarm_recipe( + dbsession, + recipe=recipe, + flavours=["nopic"], + title=title, + old_recipes={uuid4()}, + author=account, + ) + + +def test_update_zimfarm_recipe_update_existing_title_flavours( + dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + create_title: Callable[..., Title], + account: Account, +): + recipe = create_zimfarm_recipe() + title = create_title(flavours=["maxi", "mini"], zimfarm_recipe=recipe) + update_zimfarm_recipe( + dbsession, + recipe=recipe, + flavours=["maxi", "nopic", "mini"], + title=title, + old_recipes={recipe.id}, + author=account, + ) + + assert ( + count_from_stmt( + dbsession, + select(TitleFlavour).where( + TitleFlavour.title_id == title.id, TitleFlavour.recipe_id == recipe.id + ), + ) + == 3 + ) + + +def test_update_zimfarm_recipe_dissociate_recipes_from_flavours_and_title_associations( + dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + create_title: Callable[..., Title], + account: Account, +): + recipe1 = create_zimfarm_recipe() + title = create_title(zimfarm_recipe=recipe1) + # "mini" and "maxi" flavours belong to recipe1 + mini = TitleFlavour(flavour="mini") + mini.title = title + mini.recipe = recipe1 + + maxi = TitleFlavour(flavour="maxi") + maxi.title = title + maxi.recipe = recipe1 + + dbsession.add_all([maxi, mini]) + + # 'nopic' belongs to recipe2 + recipe2 = create_zimfarm_recipe() + nopic = TitleFlavour(flavour="nopic") + nopic.title = title + nopic.recipe = recipe2 + dbsession.add(nopic) + dbsession.flush() + + update_zimfarm_recipe( + dbsession, + recipe=recipe2, + flavours=["maxi", "nopic", "mini"], + title=title, + old_recipes={recipe1.id, recipe2.id}, + author=account, + ) + + # all three flavours now belong to recipe2 + assert ( + count_from_stmt( + dbsession, + select(TitleFlavour).where(TitleFlavour.recipe_id == recipe1.id), + ) + == 0 + ) + assert ( + count_from_stmt( + dbsession, + select(TitleFlavour).where(TitleFlavour.recipe_id == recipe2.id), + ) + == 3 + ) + + dbsession.refresh(recipe1) + # recipe1 is dissociated from title because it has no flavour tied to the title + assert recipe1.title is None + dbsession.refresh(recipe2) + assert recipe2.title is not None + assert recipe2.title.id == title.id + + +def test_update_zimfarm_recipe_remove_book_recipe_issues( + dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + create_title: Callable[..., Title], + create_book: Callable[..., Book], + account: Account, +): + recipe = create_zimfarm_recipe() + title = create_title(flavours=["maxi", "mini"], zimfarm_recipe=recipe) + book = create_book(name=title.name) + book.issues = ["recipe issue"] + dbsession.add(book) + dbsession.flush() + update_zimfarm_recipe( + dbsession, + recipe=recipe, + flavours=["maxi", "nopic", "mini"], + title=title, + old_recipes={recipe.id}, + author=account, + ) + dbsession.refresh(book) + assert len(book.issues) == 0 + + +def test_update_zimfarm_recipe_dissociate_recipes_from_flavours( + dbsession: OrmSession, + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + create_title: Callable[..., Title], + account: Account, +): + recipe1 = create_zimfarm_recipe() + title = create_title(zimfarm_recipe=recipe1) + # "mini", "maxi" and "nopic" flavours belong to recipe1 + mini = TitleFlavour(flavour="mini") + mini.title = title + mini.recipe = recipe1 + + maxi = TitleFlavour(flavour="maxi") + maxi.title = title + maxi.recipe = recipe1 + + nopic = TitleFlavour(flavour="nopic") + nopic.title = title + nopic.recipe = recipe1 + + dbsession.add_all([maxi, mini, nopic]) + + # recipe2 just arrives (with mini now produced by it) + recipe2 = create_zimfarm_recipe() + dbsession.flush() + + update_zimfarm_recipe( + dbsession, + recipe=recipe2, + flavours=["mini"], + title=title, + old_recipes={recipe1.id}, + author=account, + ) + + # recipe1 still has maxi and nopic + assert ( + count_from_stmt( + dbsession, + select(TitleFlavour).where(TitleFlavour.recipe_id == recipe1.id), + ) + == 2 + ) + + # recipe 2 has mini + assert ( + count_from_stmt( + dbsession, + select(TitleFlavour).where(TitleFlavour.recipe_id == recipe2.id), + ) + == 1 + ) + + dbsession.refresh(recipe1) + assert recipe1.title is not None + assert recipe1.title.id is title.id + dbsession.refresh(recipe2) + assert recipe2.title is not None + assert recipe2.title.id == title.id + + +@pytest.mark.parametrize( + "skip, limit, expected_count", + [ + pytest.param(0, 3, 3, id="first-page"), + pytest.param(3, 3, 3, id="second-page"), + pytest.param(6, 2, 0, id="page-num-too-high-no-results"), + pytest.param(0, 1, 1, id="first-page-with-low-limit"), + pytest.param(0, 10, 6, id="first-page-with-high-limit"), + ], +) +def test_get_zimfarm_recipe_history( + dbsession: OrmSession, + create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + account: Account, + skip: int, + limit: int, + expected_count: int, +): + """Test retrieving zimfarm recipe history with pagination""" + title = create_title(name="wikipedia_en_test") + recipe = create_zimfarm_recipe(title_id=title.id) + for flavour in ["mini", "maxi", "nopic", "", "novid"]: + update_zimfarm_recipe( + dbsession, + recipe=recipe, + author=account, + flavours=[flavour], + title=title, + old_recipes=set(), + create_event=False, + ) + + results = get_zimfarm_recipe_history( + dbsession, + recipe_identifier=recipe.name, + skip=skip, + limit=limit, + ) + assert results.nb_records == 6 + assert len(results.records) <= limit + assert len(results.records) == expected_count + + +def test_get_zimfarm_recipe_history_entry_or_none( + dbsession: OrmSession, + create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], + account: Account, +): + """Test retrieving a specific title history entry""" + title = create_title(name="wikipedia_en_test") + recipe = create_zimfarm_recipe(title_id=title.id) + recipe = update_zimfarm_recipe( + dbsession, + recipe=recipe, + author=account, + flavours=["maxi"], + title=title, + old_recipes=set(), + create_event=False, + ) + assert len(recipe.history_entries) == 2 + + history_result = get_zimfarm_recipe_history( + dbsession, recipe_identifier=str(recipe.id), skip=1, limit=1 + ) + history_id = history_result.records[0].id + + history_entry = get_zimfarm_recipe_history_entry_or_none( + dbsession, recipe_identifier=str(recipe.id), history_id=history_id + ) + assert history_entry is not None + assert history_entry.flavours == [] diff --git a/backend/tests/mill/processors/test_zimfarm_notification.py b/backend/tests/mill/processors/test_zimfarm_notification.py index 224280f..6fc07e6 100644 --- a/backend/tests/mill/processors/test_zimfarm_notification.py +++ b/backend/tests/mill/processors/test_zimfarm_notification.py @@ -8,7 +8,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any -from uuid import UUID +from uuid import UUID, uuid4 import pycountry import pytest @@ -24,6 +24,7 @@ Title, Warehouse, ZimfarmNotification, + ZimfarmRecipe, ) from cms_backend.mill.processors.zimfarm_notification import process_notification @@ -48,6 +49,8 @@ "zimcheck_url": "https://www.example.com/zimcheck.json", "folder_name": "test_folder", "filename": "test.zim", + "recipe_id": str(uuid4()), + "recipe_name": "test_en_all", } @@ -280,12 +283,18 @@ def test_set_missing_title_metadata_from_book( warehouse: Warehouse, # noqa: ARG002 create_zimfarm_notification: Callable[..., ZimfarmNotification], create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Set title metadata from book because title has no metadata set """ # Create title that matches book name title = create_title(name="test_en_all") + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) title.maturity = "unstable" dbsession.flush() @@ -314,6 +323,7 @@ def test_preserve_title_metadata( warehouse: Warehouse, # noqa: ARG002 create_zimfarm_notification: Callable[..., ZimfarmNotification], create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], illustration_48x48_at_1: str, ): """ @@ -331,6 +341,11 @@ def test_preserve_title_metadata( illustration_48x48_at_1=illustration_48x48_at_1, ) title.maturity = "unstable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) dbsession.flush() notification = create_zimfarm_notification(content=VALID_NOTIFICATION_CONTENT) @@ -355,6 +370,7 @@ def test_moves_book_to_staging( warehouse: Warehouse, # noqa: ARG002 create_zimfarm_notification: Callable[..., ZimfarmNotification], create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Valid notification + matching unstable maturity title → book moves to staging. @@ -362,6 +378,11 @@ def test_moves_book_to_staging( # Create title that matches book name title = create_title(name="test_en_all") title.maturity = "unstable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) dbsession.flush() notification = create_zimfarm_notification(content=VALID_NOTIFICATION_CONTENT) @@ -398,6 +419,7 @@ def test_moves_book_to_staging_with_empty_folder_name( warehouse: Warehouse, # noqa: ARG002 create_zimfarm_notification: Callable[..., ZimfarmNotification], create_title: Callable[..., Title], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Valid notification with empty folder_name + unstable maturity title → book @@ -406,6 +428,11 @@ def test_moves_book_to_staging_with_empty_folder_name( # Create title that matches book name title = create_title(name="test_en_all") title.maturity = "unstable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) dbsession.flush() content = VALID_NOTIFICATION_CONTENT.copy() @@ -452,11 +479,17 @@ def test_moves_book_to_collection_warehouses( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """Valid notification + stable title → book has collection warehouse targets.""" title = create_title(name="test_en_all") title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -505,6 +538,7 @@ def test_moves_book_to_collection_warehouses_with_empty_folder_name( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Valid notification with empty folder_name + stable title → book has collection @@ -513,6 +547,11 @@ def test_moves_book_to_collection_warehouses_with_empty_folder_name( title = create_title(name="test_en_all") title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -562,6 +601,7 @@ def test_moves_book_to_staging_due_to_diffrent_metadata_from_title( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], illustration_48x48_at_1: str, ): """ @@ -581,6 +621,11 @@ def test_moves_book_to_staging_due_to_diffrent_metadata_from_title( illustration_48x48_at_1=illustration_48x48_at_1, ) title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -621,13 +666,20 @@ def test_moves_book_to_staging_due_to_diffrent_flavour_from_title( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Test that book goes to staging because there is a flavour mismatch between it and it's title """ + recipe = create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + ) - title = create_title(name="test_en_all", flavours=["maxi", "mini"]) + title = create_title( + name="test_en_all", flavours=["maxi", "mini"], zimfarm_recipe=recipe + ) title.maturity = "stable" prod = create_warehouse( @@ -653,10 +705,9 @@ def test_moves_book_to_staging_due_to_diffrent_flavour_from_title( book = dbsession.query(Book).filter_by(id=notification.id).first() assert book is not None - assert book.title_id == title.id assert book.location_kind == "staging" assert len(book.issues) == 1 - assert set(book.issues) == {"flavour mismatch"} + assert set(book.issues) == {"recipe issue"} assert book.has_error is False assert book.needs_file_operation is True assert book.needs_processing is False @@ -669,6 +720,7 @@ def test_moves_book_to_staging_due_to_invalid_language( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Test that book goes to staging because it has an invalid language code @@ -676,6 +728,11 @@ def test_moves_book_to_staging_due_to_invalid_language( title = create_title(name="test_en_all") title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -718,6 +775,7 @@ def test_moves_book_to_prod_due_to_invalid_language_code_being_supported( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Test that book goes to prod even though it's language code is invalid @@ -726,6 +784,11 @@ def test_moves_book_to_prod_due_to_invalid_language_code_being_supported( title = create_title(name="test_en_all") title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -771,6 +834,7 @@ def test_moves_book_to_staging_due_to_valid_language_code_being_disallowed( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): """ Test that book goes to staging because there it's language code is disallowed @@ -779,6 +843,11 @@ def test_moves_book_to_staging_due_to_valid_language_code_being_disallowed( title = create_title(name="test_en_all") title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -828,9 +897,15 @@ def test_book_association_with_archived_title( create_title: Callable[..., Title], create_collection: Callable[..., Collection], create_warehouse: Callable[..., Warehouse], + create_zimfarm_recipe: Callable[..., ZimfarmRecipe], ): title = create_title(name="test_en_all", archived=True) title.maturity = "stable" + create_zimfarm_recipe( + recipe_id=VALID_NOTIFICATION_CONTENT["recipe_id"], + recipe_name=VALID_NOTIFICATION_CONTENT["recipe_name"], + title_id=title.id, + ) prod = create_warehouse( name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") @@ -858,3 +933,48 @@ def test_book_association_with_archived_title( "cannot add book to title because title is archived" in event for event in book.events ) + + +class TestValidNotificationWithRecipeIssues: + def test_recipe_does_not_exist( + self, + dbsession: OrmSession, + warehouse: Warehouse, # noqa: ARG002 + create_zimfarm_notification: Callable[..., ZimfarmNotification], + create_title: Callable[..., Title], + create_collection: Callable[..., Collection], + create_warehouse: Callable[..., Warehouse], + ): + + title = create_title(name="test_en_all") + + title.maturity = "stable" + + prod = create_warehouse( + name="prod", warehouse_id=UUID("00000000-0000-0000-0000-000000000003") + ) + collection = create_collection(warehouse=prod) + + ct = CollectionTitle(path=Path("wikipedia")) + ct.title = title + ct.collection = collection + dbsession.add(ct) + dbsession.flush() + + notification = create_zimfarm_notification(content=VALID_NOTIFICATION_CONTENT) + dbsession.flush() + + process_notification(dbsession, notification) + + assert notification.status == "processed" + + # book is not attached to any title and has recipe issue + book = dbsession.query(Book).filter_by(id=notification.id).first() + assert book is not None + assert book.title_id is None + assert book.issues == ["recipe issue"] + # new recipe is now created + recipe = dbsession.query(ZimfarmRecipe).filter_by( + id=UUID(VALID_NOTIFICATION_CONTENT["recipe_id"]) + ) + assert recipe is not None diff --git a/backend/tests/mill/test_process_title_modifications.py b/backend/tests/mill/test_process_title_modifications.py index 6c01e11..b9c8ab1 100644 --- a/backend/tests/mill/test_process_title_modifications.py +++ b/backend/tests/mill/test_process_title_modifications.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from unittest.mock import patch from uuid import uuid4 from sqlalchemy import select @@ -135,6 +136,7 @@ def test_process_title_modifications_skips_deleted_and_to_delete_books( ) dbsession.add_all([book1, book2, book3]) + dbsession.flush() create_title_modified_event( dbsession, @@ -142,16 +144,11 @@ def test_process_title_modifications_skips_deleted_and_to_delete_books( title_name="wikipedia_en_all", title_id=title.id, ) - process_title_modifications(dbsession) - - dbsession.refresh(book1) - assert book1.title_id is None - - dbsession.refresh(book2) - assert book2.title_id is None - - dbsession.refresh(book3) - assert book3.title_id == title.id + with patch( + "cms_backend.mill.process_title_modifications.process_book" + ) as mock_process_book: + process_title_modifications(dbsession) + mock_process_book.assert_called_once_with(dbsession, book3) def test_process_title_modifications_processes_matching_book( @@ -188,6 +185,8 @@ def test_process_title_modifications_processes_matching_book( title_name="wikipedia_en_all", title_id=title.id, ) - process_title_modifications(dbsession) - dbsession.refresh(book) - assert book.title_id == title.id + with patch( + "cms_backend.mill.process_title_modifications.process_book" + ) as mock_process_book: + process_title_modifications(dbsession) + mock_process_book.assert_called_once_with(dbsession, book) diff --git a/dev/scripts/setup_notifications.py b/dev/scripts/setup_notifications.py index 74c6673..6be3f77 100644 --- a/dev/scripts/setup_notifications.py +++ b/dev/scripts/setup_notifications.py @@ -54,6 +54,8 @@ }, "folder_name": "wikipedia", "filename": "dev_wikipedia_en_all_maxi_2025-01.zim", + "recipe_id": str(uuid4()), + "recipe_name": "dev_wikipedia_en_all", }, { "article_count": 500, @@ -72,6 +74,8 @@ }, "folder_name": "wiktionary", "filename": "dev_wiktionary_fr_all_maxi_2025-01.zim", + "recipe_id": str(uuid4()), + "recipe_name": "dev_wiktionary_fr_all", }, { "article_count": 1500, @@ -90,6 +94,8 @@ }, "folder_name": "", "filename": "dev_wiktionary_en_all_maxi_2025-01.zim", + "recipe_id": str(uuid4()), + "recipe_name": "dev_wiktionary_en_all", }, ] diff --git a/dev/scripts/wipe.py b/dev/scripts/wipe.py index a50c474..f6c9244 100644 --- a/dev/scripts/wipe.py +++ b/dev/scripts/wipe.py @@ -20,6 +20,7 @@ Title, Warehouse, ZimfarmNotification, + ZimfarmRecipe, ) @@ -70,6 +71,12 @@ def wipe_database(session: OrmSession): count = session.execute(delete(Title).where(Title.name.like(DEV_PREFIX))).rowcount print(f" - Deleted {count} Title records") + # 6. Recipe + count = session.execute( + delete(ZimfarmRecipe).where(ZimfarmRecipe.name.like(DEV_PREFIX)) + ).rowcount + print(f" - Deleted {count} ZimfarmRecipe records") + # 7. Collection (depends on Warehouse) count = session.execute( delete(Collection).where(Collection.name.like(DEV_PREFIX)) diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 282575e..46be3ce 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -51,6 +51,14 @@ const navigationItems = computed(() => [ disabled: false, show: true, }, + { + name: 'recipes', + label: 'Recipes', + route: 'recipes', + icon: 'mdi-book-open-variant', + disabled: false, + show: true, + }, { name: 'collections', label: 'Collections', diff --git a/frontend/src/components/BookStatus.vue b/frontend/src/components/BookStatus.vue index ed3dbff..def755f 100644 --- a/frontend/src/components/BookStatus.vue +++ b/frontend/src/components/BookStatus.vue @@ -50,7 +50,7 @@ - + diff --git a/frontend/src/components/HistoryViewer.vue b/frontend/src/components/HistoryViewer.vue index 7245a3e..35d3eb4 100644 --- a/frontend/src/components/HistoryViewer.vue +++ b/frontend/src/components/HistoryViewer.vue @@ -59,7 +59,7 @@ -