diff --git a/services/worker-indexer/Dockerfile b/services/worker-indexer/Dockerfile index d354018..fa67137 100644 --- a/services/worker-indexer/Dockerfile +++ b/services/worker-indexer/Dockerfile @@ -13,8 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app COPY requirements.txt . -# --timeout/--retries: канал до pypi медленный и флоки, без этого сборка падает -RUN pip install --no-cache-dir --timeout 120 --retries 10 -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt COPY . . diff --git a/services/worker-indexer/app/tasks/index.py b/services/worker-indexer/app/tasks/index.py index 3e99f8a..1939558 100644 --- a/services/worker-indexer/app/tasks/index.py +++ b/services/worker-indexer/app/tasks/index.py @@ -7,6 +7,7 @@ from typing import Any from celery.utils.log import get_task_logger from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError from app.algorithms.minhash import add_to_lsh, find_similar from app.algorithms.winnowing import winnow @@ -299,7 +300,16 @@ def add_document(doc_data: dict[str, Any], dispatch_embed: bool = True) -> dict[ doc = Document(**doc_kwargs) session.add(doc) - session.flush() + try: + session.flush() + except IntegrityError: + # Гонка: другой воркер параллельно вставил этот ext_id между нашей + # проверкой existing и flush. Уникальный индекс отработал — это дубль. + session.rollback() + existing = session.execute( + select(Document).where(Document.ext_id == ext_id) + ).scalar_one_or_none() + return {"status": "duplicate", "doc_id": existing.id if existing else None} doc_id = doc.id # Вычислить fingerprints