From 2daaa8c8a461379e0d5377ba727fe04fb23bc517 Mon Sep 17 00:00:00 2001 From: jze9 Date: Tue, 11 Aug 2026 17:20:24 +0500 Subject: [PATCH] =?UTF-8?q?chore(lint):=20ruff-=D0=B3=D0=B5=D0=B9=D1=82=20?= =?UTF-8?q?=D0=B2=20CI=20+=20=D1=84=D0=B8=D0=BA=D1=81=D1=8B=20(0=20=D0=BD?= =?UTF-8?q?=D0=B0=D1=85=D0=BE=D0=B4=D0=BE=D0=BA)=20=E2=80=94=20=D0=B1?= =?UTF-8?q?=D0=BB=D0=BE=D0=BA=D0=B8=D1=80=D1=83=D0=B5=D1=82=20=D0=BA=D1=80?= =?UTF-8?q?=D0=B8=D0=B2=D0=BE=D0=B9=20=D0=B4=D0=B5=D0=BF=D0=BB=D0=BE=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Второй CI-гейт после тестов: ruff как статический анализатор всего Python-кода (services + scripts). Раньше ни линта, ни проверки типов в CI не было вовсе. Конфиг ruff.toml: правила E/F/W/I/UP/B/SIM/C4, line-length 100. Осознанно выключены E501 (длину держит форматтер; длинные RU-комментарии — норма), B008 (Depends()/Query() в дефолтах — идиома FastAPI, не баг) и UP042 ((str, Enum)→StrEnum меняет __str__/сериализацию — не трогаем). Починено под ноль находок: - B904 (11): raise ... from exc / from None — читаемые цепочки исключений в Celery-ретраях и HTTPException, ошибки обработки не маскируют исходные. - SIM105 (5): try/except/pass → contextlib.suppress (faiss remove_ids, lsh.remove, сброс кэша, ws-disconnect, парс года). - C416/SIM108/B905/F841/UP035/UP017/F401/I001: dict(rows), тернарник, zip strict, мёртвая переменная, устаревшие импорты, timezone.utc→UTC, чистка/сортировка. Обвязка: scripts/run_lint.sh (ruff в изолированном python:3.11-slim), шаг «Линт» в job test перед юнит-тестами (падаем раньше). make lint / make lint-fix. Все 41 юнит-тест по-прежнему зелёные, изменённые файлы компилируются. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/deploy.yml | 6 ++++ Makefile | 10 +++++-- ruff.toml | 26 +++++++++++++++++ scripts/parsers/arxiv.py | 6 +--- scripts/parsers/cyberleninka.py | 10 +++---- scripts/parsers/openalex.py | 6 ++-- scripts/run_lint.sh | 22 ++++++++++++++ scripts/run_parser.py | 1 - services/api/alembic/env.py | 9 +++--- .../alembic/versions/001_initial_schema.py | 3 +- .../versions/002_add_task_public_id.py | 3 +- services/api/alembic/versions/003_admin.py | 3 +- services/api/app/api/admin.py | 13 ++++----- services/api/app/api/auth.py | 4 +-- services/api/app/api/documents.py | 2 +- services/api/app/api/reports.py | 1 - services/api/app/core/admin.py | 2 +- services/api/app/core/rate_limiter.py | 4 +-- services/api/app/core/security.py | 6 ++-- services/api/app/core/websocket_manager.py | 5 ++-- services/api/app/main.py | 5 ++-- services/api/app/models/user.py | 4 +-- services/worker-gost/app/db.py | 2 +- .../worker-gost/app/formatters/gost_7_1.py | 6 +--- services/worker-gost/app/tasks/gost.py | 3 +- services/worker-gpu/app/db.py | 2 +- services/worker-gpu/app/es_client.py | 3 +- services/worker-gpu/app/faiss_manager.py | 7 ++--- services/worker-gpu/app/models/base.py | 5 ++-- services/worker-gpu/app/tasks/plagiarism.py | 7 ++--- services/worker-gpu/app/tasks/search.py | 3 +- .../worker-indexer/app/algorithms/minhash.py | 7 ++--- services/worker-indexer/app/db.py | 2 +- .../worker-indexer/app/extractors/docx.py | 1 - services/worker-indexer/app/fulltext.py | 29 +++++++++---------- services/worker-indexer/app/tasks/index.py | 11 +++---- services/worker-notifier/app/db.py | 2 +- services/worker-notifier/app/tasks/notify.py | 5 ++-- 38 files changed, 143 insertions(+), 103 deletions(-) create mode 100644 ruff.toml create mode 100755 scripts/run_lint.sh diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index f24a42e..0ec2af8 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -24,6 +24,12 @@ jobs: git clone --branch main --depth 1 \ https://gitea.jze9.ru/jze9/anti-plagiarism.git "$SRC" + - name: Линт (ruff) — быстрый гейт, падаем раньше тестов + run: | + set -euo pipefail + cd "$SRC" + bash scripts/run_lint.sh + - name: Юнит-тесты (L1 winnowing, L2 minhash, L3 FAISS, ГОСТ) в контейнерах run: | set -euo pipefail diff --git a/Makefile b/Makefile index 9812e27..6dd1d68 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dev build migrate logs shell-api shell-gpu lint test test-one down ps restart clean test-up test-down test-logs test-ps +.PHONY: dev build migrate logs shell-api shell-gpu lint lint-fix test test-one down ps restart clean test-up test-down test-logs test-ps # ─── Переменные ──────────────────────────────────────────────────────────────── # Прод: docker-compose.prod.yml — app-сервисы + локальный ES, остальная инфра @@ -100,9 +100,13 @@ shell-redis: @echo "Redis общий (192.168.1.19) — подключайся напрямую: redis-cli -h 192.168.1.19 -n 3" # ─── Линтинг и тесты ────────────────────────────────────────────────────────── +# Ruff-линт всего Python-кода в изолированном контейнере (конфиг — ruff.toml). +# Тот же скрипт гоняет CI как гейт перед деплоем. Автофиксы: make lint-fix. lint: - $(COMPOSE_PROD) exec api ruff check . --fix - $(COMPOSE_PROD) exec api mypy app/ + bash scripts/run_lint.sh + +lint-fix: + ruff check services/ scripts/ --fix lint-frontend: cd services/frontend && npm run lint diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..babc680 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,26 @@ +# Линтер/форматтер для всех Python-сервисов и скриптов. +# Гоняется как гейт в CI (scripts/run_lint.sh) — падение блокирует деплой. +target-version = "py311" +line-length = 100 + +[lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes (неиспользуемое, неопределённое) + "W", # pycodestyle warnings + "I", # isort (порядок импортов) + "UP", # pyupgrade (устаревшие конструкции) + "B", # flake8-bugbear (частые баги) + "SIM", # flake8-simplify + "C4", # flake8-comprehensions +] +ignore = [ + "E501", # длину строк держит форматтер, а не линтер; длинные RU-комментарии — норма + "B008", # FastAPI-идиома: Depends()/Query() в значениях по умолчанию — не баг + "UP042", # (str, Enum) → StrEnum меняет __str__ (сериализацию) — не трогаем намеренно +] + +[lint.per-file-ignores] +# В тестах допускаем импорт-после-кода (importorskip) и «неотсортированные» блоки +"**/tests/*" = ["E402"] +"**/conftest.py" = ["E402"] diff --git a/scripts/parsers/arxiv.py b/scripts/parsers/arxiv.py index ec08119..fee3d18 100644 --- a/scripts/parsers/arxiv.py +++ b/scripts/parsers/arxiv.py @@ -6,13 +6,12 @@ API: https://info.arxiv.org/help/api/index.html Использует Atom XML API. """ -import time import logging +import time import xml.etree.ElementTree as ET from typing import Any import httpx - from base import BaseParser logger = logging.getLogger(__name__) @@ -203,9 +202,6 @@ class ArxivParser(BaseParser): # Определить язык (arXiv — преимущественно английский) lang = "en" - # Категории как JSON - categories = raw.get("categories", []) - return { "source": self.source_name, "ext_id": f"arxiv:{ext_id}", diff --git a/scripts/parsers/cyberleninka.py b/scripts/parsers/cyberleninka.py index 3a3dd6e..f1b2377 100644 --- a/scripts/parsers/cyberleninka.py +++ b/scripts/parsers/cyberleninka.py @@ -7,15 +7,15 @@ а не недокументированное API. """ +import contextlib +import logging import re import time -import logging from typing import Any import httpx -from bs4 import BeautifulSoup - from base import BaseParser +from bs4 import BeautifulSoup logger = logging.getLogger(__name__) @@ -122,10 +122,8 @@ class CyberLeninkaParser(BaseParser): year_raw = raw.get("year") year = None if year_raw: - try: + with contextlib.suppress(ValueError, TypeError): year = int(str(year_raw)[:4]) - except (ValueError, TypeError): - pass # URL link = raw.get("link", "") diff --git a/scripts/parsers/openalex.py b/scripts/parsers/openalex.py index 2726c26..76ba240 100644 --- a/scripts/parsers/openalex.py +++ b/scripts/parsers/openalex.py @@ -9,12 +9,12 @@ API: https://docs.openalex.org/ - Идемпотентность: проверка по ext_id перед добавлением """ -import time import logging -from typing import Any, Generator +import time +from collections.abc import Generator +from typing import Any import httpx - from base import BaseParser logger = logging.getLogger(__name__) diff --git a/scripts/run_lint.sh b/scripts/run_lint.sh new file mode 100755 index 0000000..e68c155 --- /dev/null +++ b/scripts/run_lint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Ruff-линт всего Python-кода (services + scripts) в изолированном контейнере. +# Конфиг — ruff.toml в корне репозитория. Гоняется как гейт в CI перед деплоем. +# +# PIP_INDEX_URL переопределяется при необходимости (по умолчанию Tsinghua-зеркало). +set -euo pipefail + +MIRROR="${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple/}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUFF_VERSION="0.16.2" + +echo "▶ Ruff-линт (services/ scripts/)" +docker run --rm \ + -v "$ROOT":/repo -w /repo \ + -e PIP_INDEX_URL="$MIRROR" \ + -e PIP_DISABLE_PIP_VERSION_CHECK=1 \ + python:3.11-slim bash -c " + set -e + pip install --no-cache-dir --timeout 60 -q ruff==$RUFF_VERSION + ruff check services/ scripts/ + " +echo "✅ Ruff: все проверки пройдены" diff --git a/scripts/run_parser.py b/scripts/run_parser.py index 032627a..25c6479 100644 --- a/scripts/run_parser.py +++ b/scripts/run_parser.py @@ -10,7 +10,6 @@ import argparse import logging -import os import sys from pathlib import Path diff --git a/services/api/alembic/env.py b/services/api/alembic/env.py index 7ecc219..4f186e0 100644 --- a/services/api/alembic/env.py +++ b/services/api/alembic/env.py @@ -4,14 +4,15 @@ import asyncio import os from logging.config import fileConfig -from alembic import context +import app.models # noqa: F401 — регистрирует все модели + +# Импорт всех моделей для автоопределения изменений +from app.database import Base from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config -# Импорт всех моделей для автоопределения изменений -from app.database import Base -import app.models # noqa: F401 — регистрирует все модели +from alembic import context # Alembic Config config = context.config diff --git a/services/api/alembic/versions/001_initial_schema.py b/services/api/alembic/versions/001_initial_schema.py index e624e8a..88f34fe 100644 --- a/services/api/alembic/versions/001_initial_schema.py +++ b/services/api/alembic/versions/001_initial_schema.py @@ -7,9 +7,10 @@ Create Date: 2024-01-01 00:00:00.000000 Создаёт таблицы: users, tasks, documents, fingerprints, usage_logs """ -from alembic import op import sqlalchemy as sa +from alembic import op + # revision identifiers revision = "001" down_revision = None diff --git a/services/api/alembic/versions/002_add_task_public_id.py b/services/api/alembic/versions/002_add_task_public_id.py index 8c927cc..3c6bad3 100644 --- a/services/api/alembic/versions/002_add_task_public_id.py +++ b/services/api/alembic/versions/002_add_task_public_id.py @@ -7,10 +7,11 @@ Create Date: 2026-05-24 import secrets -from alembic import op import sqlalchemy as sa from sqlalchemy import text +from alembic import op + revision = "002" down_revision = "001" branch_labels = None diff --git a/services/api/alembic/versions/003_admin.py b/services/api/alembic/versions/003_admin.py index e7347a4..e661888 100644 --- a/services/api/alembic/versions/003_admin.py +++ b/services/api/alembic/versions/003_admin.py @@ -5,9 +5,10 @@ Revises: 002 Create Date: 2026-05-30 """ -from alembic import op import sqlalchemy as sa +from alembic import op + revision = "003" down_revision = "002" branch_labels = None diff --git a/services/api/app/api/admin.py b/services/api/app/api/admin.py index 7497ce0..7f3a11c 100644 --- a/services/api/app/api/admin.py +++ b/services/api/app/api/admin.py @@ -4,8 +4,9 @@ дополнительно проверяют секретный код сессии (verify_admin_code). """ +import contextlib import logging -from datetime import datetime, timezone +from datetime import datetime import httpx from fastapi import APIRouter, Depends, HTTPException, Query, status @@ -150,11 +151,11 @@ async def stats(db: AsyncSession = Depends(get_db)) -> AdminStats: users_total = (await db.execute(select(func.count()).select_from(User))).scalar_one() rows = (await db.execute(select(Task.status, func.count()).group_by(Task.status))).all() - tasks_by_status = {s: c for s, c in rows} + tasks_by_status = dict(rows) docs_total = (await db.execute(select(func.count()).select_from(Document))).scalar_one() rows = (await db.execute(select(Document.source, func.count()).group_by(Document.source))).all() - docs_by_source = {s: c for s, c in rows} + docs_by_source = dict(rows) staging_pending = ( await db.execute( @@ -257,10 +258,8 @@ async def update_user( await db.commit() await db.refresh(user) # Сбросить кэш пользователя - try: + with contextlib.suppress(Exception): await get_redis().delete(f"user:cache:{user_id}") - except Exception: - pass return AdminUserResponse.model_validate(user) @@ -533,7 +532,7 @@ async def approve_staging( obj = get_minio().get_object("staging", sw.text_key) full_text = obj.read().decode("utf-8", errors="replace") except Exception as e: - raise HTTPException(status_code=500, detail=f"Не удалось прочитать текст: {e}") + raise HTTPException(status_code=500, detail=f"Не удалось прочитать текст: {e}") from e # Добавить в базу документов через существующую задачу индексатора doc_data = { diff --git a/services/api/app/api/auth.py b/services/api/app/api/auth.py index b54cfc1..e4ada17 100644 --- a/services/api/app/api/auth.py +++ b/services/api/app/api/auth.py @@ -1,7 +1,7 @@ """Роутер аутентификации: регистрация, вход, верификация email.""" -import secrets import logging +import secrets from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy import select @@ -224,7 +224,7 @@ async def resend_verification( raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Не удалось отправить письмо, попробуйте позже", - ) + ) from e @router.post("/verify-email/{token}", status_code=status.HTTP_200_OK) diff --git a/services/api/app/api/documents.py b/services/api/app/api/documents.py index a0e1ec1..92aeadc 100644 --- a/services/api/app/api/documents.py +++ b/services/api/app/api/documents.py @@ -116,7 +116,7 @@ async def upload_for_plagiarism_check( raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Ошибка сохранения файла. Попробуйте позже.", - ) + ) from e # ── 5. Создаём задачу и диспатчим ───────────────────────────────────────── diff --git a/services/api/app/api/reports.py b/services/api/app/api/reports.py index 752722b..d1846ff 100644 --- a/services/api/app/api/reports.py +++ b/services/api/app/api/reports.py @@ -1,7 +1,6 @@ """Роутер для получения отчётов о выполненных задачах.""" import logging -from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy import select diff --git a/services/api/app/core/admin.py b/services/api/app/core/admin.py index 5b231f3..7545713 100644 --- a/services/api/app/core/admin.py +++ b/services/api/app/core/admin.py @@ -1,7 +1,7 @@ """Авторизация админ-панели: проверка роли is_admin и секретного кода сессии.""" import secrets -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from fastapi import Depends, HTTPException, Path, status from sqlalchemy import select diff --git a/services/api/app/core/rate_limiter.py b/services/api/app/core/rate_limiter.py index 910f023..abd4449 100644 --- a/services/api/app/core/rate_limiter.py +++ b/services/api/app/core/rate_limiter.py @@ -9,7 +9,7 @@ Redis гарантирует, что между командами внутри """ import logging -from datetime import datetime, timezone +from datetime import UTC, datetime from app.core.redis_client import get_redis @@ -93,7 +93,7 @@ return 1 def _period_suffix(period: str) -> str: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) return now.strftime("%Y-%m-%d") if period == "day" else now.strftime("%Y-%m") diff --git a/services/api/app/core/security.py b/services/api/app/core/security.py index cf9acd6..bf836ef 100644 --- a/services/api/app/core/security.py +++ b/services/api/app/core/security.py @@ -2,7 +2,7 @@ import json import logging -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from typing import Any from fastapi import Depends, HTTPException, Query, WebSocket, status @@ -35,7 +35,7 @@ def verify_password(plain: str, hashed: str) -> bool: def create_access_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str: payload = data.copy() - expire = datetime.now(timezone.utc) + ( + expire = datetime.now(UTC) + ( expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES) ) payload["exp"] = expire @@ -58,7 +58,7 @@ def _decode_token(token: str) -> int: status_code=status.HTTP_401_UNAUTHORIZED, detail="Невалидный или просроченный токен", headers={"WWW-Authenticate": "Bearer"}, - ) + ) from None async def _load_user(user_id: int, db: AsyncSession): diff --git a/services/api/app/core/websocket_manager.py b/services/api/app/core/websocket_manager.py index 5767a4c..c4924aa 100644 --- a/services/api/app/core/websocket_manager.py +++ b/services/api/app/core/websocket_manager.py @@ -1,5 +1,6 @@ """WebSocket менеджер для real-time обновлений статуса задач.""" +import contextlib import json import logging from typing import Any @@ -27,10 +28,8 @@ class ConnectionManager: def disconnect(self, task_id: str, websocket: WebSocket) -> None: """Удалить соединение из реестра.""" if task_id in self._connections: - try: + with contextlib.suppress(ValueError): self._connections[task_id].remove(websocket) - except ValueError: - pass if not self._connections[task_id]: del self._connections[task_id] logger.info(f"WebSocket отключён от задачи {task_id!r}") diff --git a/services/api/app/main.py b/services/api/app/main.py index bf153ec..01122f5 100644 --- a/services/api/app/main.py +++ b/services/api/app/main.py @@ -1,10 +1,10 @@ """Точка входа FastAPI приложения — Академический помощник.""" import logging +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -from typing import AsyncGenerator -from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends +from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse @@ -98,6 +98,7 @@ async def websocket_task_updates( ownership проверяется до установки соединения. """ from sqlalchemy import select + from app.database import AsyncSessionLocal from app.models.task import Task diff --git a/services/api/app/models/user.py b/services/api/app/models/user.py index fffa553..2628469 100644 --- a/services/api/app/models/user.py +++ b/services/api/app/models/user.py @@ -3,14 +3,14 @@ from datetime import datetime from typing import TYPE_CHECKING -from sqlalchemy import func, String +from sqlalchemy import String, func from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database import Base if TYPE_CHECKING: - from app.models.task import Task from app.models.document import UsageLog + from app.models.task import Task class User(Base): diff --git a/services/worker-gost/app/db.py b/services/worker-gost/app/db.py index 0b45747..402848e 100644 --- a/services/worker-gost/app/db.py +++ b/services/worker-gost/app/db.py @@ -1,7 +1,7 @@ """Подключение к PostgreSQL для gost-воркера.""" +from collections.abc import Generator from contextlib import contextmanager -from typing import Generator from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker diff --git a/services/worker-gost/app/formatters/gost_7_1.py b/services/worker-gost/app/formatters/gost_7_1.py index d39235b..48e96ac 100644 --- a/services/worker-gost/app/formatters/gost_7_1.py +++ b/services/worker-gost/app/formatters/gost_7_1.py @@ -16,7 +16,6 @@ - Место публикации через "/" """ -import re def _format_author(author: dict) -> str: @@ -245,10 +244,7 @@ def _get_sort_key(doc: dict) -> str: Строка для сортировки """ authors = doc.get("authors", []) - if authors: - last_name = authors[0].get("last_name", "") - else: - last_name = doc.get("title", "") + last_name = authors[0].get("last_name", "") if authors else doc.get("title", "") if not last_name: return "яяя" # В конец diff --git a/services/worker-gost/app/tasks/gost.py b/services/worker-gost/app/tasks/gost.py index 88c0403..5ec3be5 100644 --- a/services/worker-gost/app/tasks/gost.py +++ b/services/worker-gost/app/tasks/gost.py @@ -1,6 +1,5 @@ """Celery задача форматирования библиографии по ГОСТ.""" -import logging from typing import Any from celery.utils.log import get_task_logger @@ -139,4 +138,4 @@ def format_bibliography( except Exception as db_exc: logger.error(f"Не удалось обновить статус задачи: {db_exc}") - raise self.retry(exc=exc, countdown=30) + raise self.retry(exc=exc, countdown=30) from exc diff --git a/services/worker-gpu/app/db.py b/services/worker-gpu/app/db.py index 32e3444..78e572c 100644 --- a/services/worker-gpu/app/db.py +++ b/services/worker-gpu/app/db.py @@ -1,8 +1,8 @@ """Синхронное подключение к PostgreSQL для Celery воркеров.""" import logging +from collections.abc import Generator from contextlib import contextmanager -from typing import Generator import redis as redis_lib from sqlalchemy import create_engine diff --git a/services/worker-gpu/app/es_client.py b/services/worker-gpu/app/es_client.py index cef6436..bde9650 100644 --- a/services/worker-gpu/app/es_client.py +++ b/services/worker-gpu/app/es_client.py @@ -3,7 +3,8 @@ import logging from typing import Any -from elasticsearch import Elasticsearch, exceptions as es_exceptions +from elasticsearch import Elasticsearch +from elasticsearch import exceptions as es_exceptions from app.config import settings diff --git a/services/worker-gpu/app/faiss_manager.py b/services/worker-gpu/app/faiss_manager.py index 7500b02..a31438a 100644 --- a/services/worker-gpu/app/faiss_manager.py +++ b/services/worker-gpu/app/faiss_manager.py @@ -12,6 +12,7 @@ миллионов документов) полный перебор по FlatIP по скорости приемлем. """ +import contextlib import logging import os from pathlib import Path @@ -88,7 +89,7 @@ class FAISSManager: distances, ids = cls._index.search(query, min(k, cls._index.ntotal)) results = [] - for idx, dist in zip(ids[0], distances[0]): + for idx, dist in zip(ids[0], distances[0], strict=False): if idx == -1: continue # Для IDMap2 idx — это уже doc_id из PostgreSQL @@ -120,10 +121,8 @@ class FAISSManager: ids = np.asarray(doc_ids, dtype=np.int64) # Удалить существующие id, чтобы повторный эмбеддинг не создавал дубли - try: + with contextlib.suppress(Exception): cls._index.remove_ids(ids) - except Exception: - pass cls._index.add_with_ids(vectors, ids) logger.info(f"Добавлено {len(doc_ids)} векторов в FAISS. Всего: {cls._index.ntotal}") diff --git a/services/worker-gpu/app/models/base.py b/services/worker-gpu/app/models/base.py index f54fe42..40cf924 100644 --- a/services/worker-gpu/app/models/base.py +++ b/services/worker-gpu/app/models/base.py @@ -1,10 +1,9 @@ """Базовые модели данных для GPU воркера (минимальный набор для работы с БД).""" from datetime import datetime -from uuid import uuid4 -from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String, Text, func -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from sqlalchemy import JSON, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column class Base(DeclarativeBase): diff --git a/services/worker-gpu/app/tasks/plagiarism.py b/services/worker-gpu/app/tasks/plagiarism.py index 199918d..14b33a8 100644 --- a/services/worker-gpu/app/tasks/plagiarism.py +++ b/services/worker-gpu/app/tasks/plagiarism.py @@ -1,6 +1,5 @@ """Celery задачи проверки плагиата (уровни 3 и 4) и построения эмбеддингов.""" -import logging from typing import Any from celery.utils.log import get_task_logger @@ -209,7 +208,7 @@ def check_plagiarism( except Exception as db_exc: logger.error(f"Не удалось обновить статус задачи: {db_exc}") - raise self.retry(exc=exc, countdown=120) + raise self.retry(exc=exc, countdown=120) from exc @celery_app.task(name="gpu.embed_documents") @@ -225,10 +224,9 @@ def embed_documents(doc_ids: list[int]) -> dict[str, Any]: if not doc_ids: return {"status": "ok", "embedded": 0} - from app.models import Document from app.faiss_manager import FAISSManager from app.model_manager import ModelManager - from sqlalchemy import select + from app.models import Document logger.info(f"Построение эмбеддингов для {len(doc_ids)} документов...") @@ -248,7 +246,6 @@ def embed_documents(doc_ids: list[int]) -> dict[str, Any]: ] ids = [d.id for d in docs] - import numpy as np vectors = ModelManager.encode(texts) FAISSManager.add_vectors(vectors, ids) diff --git a/services/worker-gpu/app/tasks/search.py b/services/worker-gpu/app/tasks/search.py index fa38cd9..8f8a40b 100644 --- a/services/worker-gpu/app/tasks/search.py +++ b/services/worker-gpu/app/tasks/search.py @@ -2,7 +2,6 @@ import hashlib import json -import logging from typing import Any from celery.utils.log import get_task_logger @@ -278,4 +277,4 @@ def search_semantic( logger.error(f"Не удалось обновить статус задачи: {db_exc}") # Повторить попытку - raise self.retry(exc=exc, countdown=60) + raise self.retry(exc=exc, countdown=60) from exc diff --git a/services/worker-indexer/app/algorithms/minhash.py b/services/worker-indexer/app/algorithms/minhash.py index 474653b..1dd974b 100644 --- a/services/worker-indexer/app/algorithms/minhash.py +++ b/services/worker-indexer/app/algorithms/minhash.py @@ -9,6 +9,7 @@ локальный и теряется при рестарте), чтобы воркер не падал целиком. """ +import contextlib import logging from urllib.parse import urlparse @@ -118,10 +119,8 @@ def add_to_lsh(doc_key: str, text: str) -> None: lsh = get_lsh() m = text_to_minhash(text) try: - try: - lsh.remove(doc_key) # снять прежнюю версию, если была - except Exception: - pass # ключа не было — это норма + with contextlib.suppress(Exception): + lsh.remove(doc_key) # снять прежнюю версию, если была (ключа могло не быть) lsh.insert(doc_key, m) except Exception as e: logger.warning(f"MinHash LSH: не удалось добавить {doc_key!r}: {e}") diff --git a/services/worker-indexer/app/db.py b/services/worker-indexer/app/db.py index 5eee69a..94cf6c9 100644 --- a/services/worker-indexer/app/db.py +++ b/services/worker-indexer/app/db.py @@ -1,8 +1,8 @@ """Синхронное подключение к PostgreSQL и MinIO для индексер-воркера.""" import logging +from collections.abc import Generator from contextlib import contextmanager -from typing import Generator from minio import Minio from sqlalchemy import create_engine diff --git a/services/worker-indexer/app/extractors/docx.py b/services/worker-indexer/app/extractors/docx.py index 219d8bd..2ea5930 100644 --- a/services/worker-indexer/app/extractors/docx.py +++ b/services/worker-indexer/app/extractors/docx.py @@ -22,7 +22,6 @@ def extract_text_from_docx(data: bytes) -> str: ValueError: Если не удалось открыть DOCX """ from docx import Document as DocxDocument - from docx.oxml.ns import qn try: doc = DocxDocument(io.BytesIO(data)) diff --git a/services/worker-indexer/app/fulltext.py b/services/worker-indexer/app/fulltext.py index a47589a..be47821 100644 --- a/services/worker-indexer/app/fulltext.py +++ b/services/worker-indexer/app/fulltext.py @@ -38,22 +38,21 @@ def fetch_full_text(url: str) -> str | None: follow_redirects=True, timeout=settings.FULL_TEXT_TIMEOUT, headers=_HEADERS, - ) as client: - with client.stream("GET", url) as resp: - resp.raise_for_status() - ctype = resp.headers.get("content-type", "").lower() + ) as client, client.stream("GET", url) as resp: + resp.raise_for_status() + ctype = resp.headers.get("content-type", "").lower() - # Скачиваем с ограничением размера - buf = bytearray() - for chunk in resp.iter_bytes(): - buf += chunk - if len(buf) > settings.FULL_TEXT_MAX_BYTES: - logger.info( - f"full-text превысил лимит {settings.FULL_TEXT_MAX_BYTES} байт, " - f"обрезаю: {url}" - ) - break - data = bytes(buf) + # Скачиваем с ограничением размера + buf = bytearray() + for chunk in resp.iter_bytes(): + buf += chunk + if len(buf) > settings.FULL_TEXT_MAX_BYTES: + logger.info( + f"full-text превысил лимит {settings.FULL_TEXT_MAX_BYTES} байт, " + f"обрезаю: {url}" + ) + break + data = bytes(buf) except Exception as e: logger.info(f"full-text: скачать не удалось {url!r}: {type(e).__name__}: {str(e)[:120]}") return None diff --git a/services/worker-indexer/app/tasks/index.py b/services/worker-indexer/app/tasks/index.py index 1939558..28b66f0 100644 --- a/services/worker-indexer/app/tasks/index.py +++ b/services/worker-indexer/app/tasks/index.py @@ -1,7 +1,7 @@ """Celery задачи индексации документов и проверки плагиата (уровни 1-2).""" import io -import logging +from datetime import UTC from pathlib import Path from typing import Any @@ -253,7 +253,7 @@ def extract_and_check( except Exception as exc: logger.error(f"Ошибка при обработке задачи {task_id!r}: {exc}", exc_info=True) update_task_status(task_id, "failed", str(exc)) - raise self.retry(exc=exc, countdown=60) + raise self.retry(exc=exc, countdown=60) from exc @celery_app.task(name="index.add_document") @@ -330,6 +330,7 @@ def add_document(doc_data: dict[str, Any], dispatch_embed: bool = True) -> dict[ # Индексация в Elasticsearch try: from elasticsearch import Elasticsearch + from app.config import settings as cfg es = Elasticsearch(cfg.ELASTICSEARCH_URL) @@ -409,7 +410,7 @@ def enrich_full_text(self, doc_id: int, url: str) -> dict[str, Any]: ) except Exception as exc: logger.error(f"enrich_full_text: не удалось сохранить текст в MinIO для {doc_id}: {exc}") - raise self.retry(exc=exc, countdown=120) + raise self.retry(exc=exc, countdown=120) from exc # Пересчитать fingerprints по полному тексту doc_fp = winnow(text) @@ -501,7 +502,7 @@ def run_parser(source_id: int) -> dict[str, Any]: задачу add_document для каждого полученного документа. """ import sys - from datetime import datetime, timezone + from datetime import datetime from app.models import ParseSource @@ -594,7 +595,7 @@ def run_parser(source_id: int) -> dict[str, Any]: src.last_status = "error" if error_msg else "done" src.last_error = error_msg src.docs_added = (src.docs_added or 0) + added - src.last_run_at = datetime.now(timezone.utc) + src.last_run_at = datetime.now(UTC) session.commit() return {"status": "error" if error_msg else "done", "added": added, "error": error_msg} diff --git a/services/worker-notifier/app/db.py b/services/worker-notifier/app/db.py index 0d6b682..82e2fd5 100644 --- a/services/worker-notifier/app/db.py +++ b/services/worker-notifier/app/db.py @@ -1,7 +1,7 @@ """Подключение к PostgreSQL для notifier-воркера.""" +from collections.abc import Generator from contextlib import contextmanager -from typing import Generator from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker diff --git a/services/worker-notifier/app/tasks/notify.py b/services/worker-notifier/app/tasks/notify.py index 9cb8b2b..591c703 100644 --- a/services/worker-notifier/app/tasks/notify.py +++ b/services/worker-notifier/app/tasks/notify.py @@ -1,6 +1,5 @@ """Celery задачи отправки email уведомлений.""" -import logging from celery.utils.log import get_task_logger @@ -67,7 +66,7 @@ def send_task_done(self, task_id: str) -> dict: except Exception as exc: logger.error(f"Ошибка отправки уведомления для задачи {task_id!r}: {exc}", exc_info=True) - raise self.retry(exc=exc, countdown=30) + raise self.retry(exc=exc, countdown=30) from exc def _build_summary(task) -> str: @@ -144,4 +143,4 @@ def send_verification(self, user_email: str, user_name: str, token: str) -> dict return {"status": "sent"} except Exception as exc: logger.error(f"Ошибка отправки верификации на {user_email!r}: {exc}", exc_info=True) - raise self.retry(exc=exc, countdown=60) + raise self.retry(exc=exc, countdown=60) from exc