Compare commits
2 Commits
c4596f2c3a
...
e328236d0f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e328236d0f | ||
|
|
52e5550f91 |
@@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir --timeout 60 -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r requirements.txt
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir --timeout 60 -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r requirements.txt
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir --timeout 60 -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r requirements.txt
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
@@ -13,8 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
# --timeout/--retries: канал до pypi медленный и флоки, без этого сборка падает
|
RUN pip install --no-cache-dir --timeout 60 -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r requirements.txt
|
||||||
RUN pip install --no-cache-dir --timeout 120 --retries 10 -r requirements.txt
|
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from typing import Any
|
|||||||
|
|
||||||
from celery.utils.log import get_task_logger
|
from celery.utils.log import get_task_logger
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
from app.algorithms.minhash import add_to_lsh, find_similar
|
from app.algorithms.minhash import add_to_lsh, find_similar
|
||||||
from app.algorithms.winnowing import winnow
|
from app.algorithms.winnowing import winnow
|
||||||
@@ -299,7 +300,16 @@ def add_document(doc_data: dict[str, Any], dispatch_embed: bool = True) -> dict[
|
|||||||
|
|
||||||
doc = Document(**doc_kwargs)
|
doc = Document(**doc_kwargs)
|
||||||
session.add(doc)
|
session.add(doc)
|
||||||
|
try:
|
||||||
session.flush()
|
session.flush()
|
||||||
|
except IntegrityError:
|
||||||
|
# Гонка: другой воркер параллельно вставил этот ext_id между нашей
|
||||||
|
# проверкой existing и flush. Уникальный индекс отработал — это дубль.
|
||||||
|
session.rollback()
|
||||||
|
existing = session.execute(
|
||||||
|
select(Document).where(Document.ext_id == ext_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
return {"status": "duplicate", "doc_id": existing.id if existing else None}
|
||||||
doc_id = doc.id
|
doc_id = doc.id
|
||||||
|
|
||||||
# Вычислить fingerprints
|
# Вычислить fingerprints
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir --timeout 60 -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r requirements.txt
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user