feat: initial microservices project structure
Services: - api: FastAPI gateway with JWT auth, async endpoints, WebSocket - worker-gpu: CUDA sentence-transformers, FAISS IVFFlat, Ollama LLM - worker-indexer: Winnowing+MinHash plagiarism detection, PDF/DOCX extraction - worker-notifier: SMTP email notifications - worker-gost: GOST 7.1-2003 and GOST R 7.0.5-2008 formatting Infrastructure: - docker-compose.yml (production) + docker-compose.dev.yml (hot reload) - Nginx reverse proxy + WebSocket support - PostgreSQL 16 with Alembic migrations - Elasticsearch 8 with Russian/English analyzers - MinIO, RabbitMQ, Redis, Ollama Frontend: - React 18 + Vite + TypeScript + TailwindCSS + Zustand + React Query v5 - 9 pages: Home, Search, Cabinet, Task, Check, Bibliography, Pricing, Login, Register Scripts: - Parser stubs: OpenAlex, КиберЛенинка, arXiv (Phase 0 - to be filled) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
40
.env.example
Normal file
40
.env.example
Normal file
@@ -0,0 +1,40 @@
|
||||
# PostgreSQL
|
||||
POSTGRES_HOST=postgres
|
||||
POSTGRES_PORT=5432
|
||||
POSTGRES_DB=antiplagiator
|
||||
POSTGRES_USER=antiplagiator
|
||||
POSTGRES_PASSWORD=changeme
|
||||
|
||||
# Redis
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
|
||||
# RabbitMQ
|
||||
RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672/
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT=minio:9000
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=changeme
|
||||
MINIO_BUCKET_DOCS=documents
|
||||
MINIO_BUCKET_BACKUPS=backups
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_URL=http://elasticsearch:9200
|
||||
|
||||
# Ollama
|
||||
OLLAMA_URL=http://ollama:11434
|
||||
|
||||
# JWT
|
||||
SECRET_KEY=change-me-in-production-use-openssl-rand-hex-32
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES=10080
|
||||
|
||||
# SMTP (Yandex)
|
||||
SMTP_HOST=smtp.yandex.ru
|
||||
SMTP_PORT=465
|
||||
SMTP_USER=noreply@jze9.ru
|
||||
SMTP_PASSWORD=changeme
|
||||
SMTP_FROM=noreply@jze9.ru
|
||||
|
||||
# App
|
||||
APP_URL=https://academic.jze9.ru
|
||||
ENVIRONMENT=development
|
||||
98
.gitignore
vendored
Normal file
98
.gitignore
vendored
Normal file
@@ -0,0 +1,98 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# Virtual environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Environment files
|
||||
*.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Node / Frontend
|
||||
node_modules/
|
||||
dist/
|
||||
build/
|
||||
.npm
|
||||
.eslintcache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Docker data volumes (монтируются, не хранятся в репо)
|
||||
data/
|
||||
|
||||
# FAISS индексы
|
||||
*.faiss
|
||||
*.index
|
||||
|
||||
# Database dumps
|
||||
*.sql.gz
|
||||
*.dump
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Testing
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
|
||||
# Alembic
|
||||
# (миграции коммитятся, только __pycache__ игнорируется)
|
||||
|
||||
# MinIO / S3 кэш
|
||||
.minio/
|
||||
|
||||
# ML модели (скачиваются при старте)
|
||||
models/
|
||||
*.bin
|
||||
*.safetensors
|
||||
|
||||
# Jupyter
|
||||
.ipynb_checkpoints/
|
||||
*.ipynb
|
||||
|
||||
# Temp
|
||||
tmp/
|
||||
temp/
|
||||
132
Makefile
Normal file
132
Makefile
Normal file
@@ -0,0 +1,132 @@
|
||||
.PHONY: dev build migrate logs shell-api shell-gpu lint test down ps restart clean
|
||||
|
||||
# ─── Переменные ────────────────────────────────────────────────────────────────
|
||||
COMPOSE = docker compose
|
||||
COMPOSE_DEV = docker compose -f docker-compose.yml -f docker-compose.dev.yml
|
||||
PROJECT = anti-plagiarism
|
||||
|
||||
# ─── Разработка ────────────────────────────────────────────────────────────────
|
||||
dev:
|
||||
@cp -n .env.example .env 2>/dev/null || true
|
||||
$(COMPOSE_DEV) up --build
|
||||
|
||||
dev-d:
|
||||
@cp -n .env.example .env 2>/dev/null || true
|
||||
$(COMPOSE_DEV) up --build -d
|
||||
|
||||
# ─── Продакшн ─────────────────────────────────────────────────────────────────
|
||||
build:
|
||||
$(COMPOSE) build
|
||||
|
||||
up:
|
||||
$(COMPOSE) up -d
|
||||
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
|
||||
restart:
|
||||
$(COMPOSE) restart
|
||||
|
||||
ps:
|
||||
$(COMPOSE) ps
|
||||
|
||||
# ─── Миграции ─────────────────────────────────────────────────────────────────
|
||||
migrate:
|
||||
$(COMPOSE) exec api alembic upgrade head
|
||||
|
||||
migrate-dev:
|
||||
$(COMPOSE_DEV) exec api alembic upgrade head
|
||||
|
||||
makemigration:
|
||||
@read -p "Migration name: " name; \
|
||||
$(COMPOSE) exec api alembic revision --autogenerate -m "$$name"
|
||||
|
||||
downgrade:
|
||||
$(COMPOSE) exec api alembic downgrade -1
|
||||
|
||||
# ─── Логи ─────────────────────────────────────────────────────────────────────
|
||||
logs:
|
||||
$(COMPOSE) logs -f
|
||||
|
||||
logs-api:
|
||||
$(COMPOSE) logs -f api
|
||||
|
||||
logs-gpu:
|
||||
$(COMPOSE) logs -f worker-gpu
|
||||
|
||||
logs-indexer:
|
||||
$(COMPOSE) logs -f worker-indexer
|
||||
|
||||
logs-notifier:
|
||||
$(COMPOSE) logs -f worker-notifier
|
||||
|
||||
logs-gost:
|
||||
$(COMPOSE) logs -f worker-gost
|
||||
|
||||
# ─── Шеллы ────────────────────────────────────────────────────────────────────
|
||||
shell-api:
|
||||
$(COMPOSE) exec api bash
|
||||
|
||||
shell-gpu:
|
||||
$(COMPOSE) exec worker-gpu bash
|
||||
|
||||
shell-indexer:
|
||||
$(COMPOSE) exec worker-indexer bash
|
||||
|
||||
shell-db:
|
||||
$(COMPOSE) exec postgres psql -U antiplagiator antiplagiator
|
||||
|
||||
shell-redis:
|
||||
$(COMPOSE) exec redis redis-cli
|
||||
|
||||
# ─── Линтинг и тесты ──────────────────────────────────────────────────────────
|
||||
lint:
|
||||
$(COMPOSE) exec api ruff check . --fix
|
||||
$(COMPOSE) exec api mypy app/
|
||||
|
||||
lint-frontend:
|
||||
cd services/frontend && npm run lint
|
||||
|
||||
test:
|
||||
$(COMPOSE) exec api pytest tests/ -v
|
||||
|
||||
test-cov:
|
||||
$(COMPOSE) exec api pytest tests/ -v --cov=app --cov-report=html
|
||||
|
||||
# ─── Утилиты ──────────────────────────────────────────────────────────────────
|
||||
clean:
|
||||
$(COMPOSE) down -v --remove-orphans
|
||||
docker system prune -f
|
||||
|
||||
# Запустить парсер (пример: make parse PARSER=openalex ARGS="--limit 1000")
|
||||
parse:
|
||||
$(COMPOSE) exec api python /scripts/run_parser.py $(PARSER) $(ARGS)
|
||||
|
||||
# Создать MinIO бакеты
|
||||
minio-init:
|
||||
$(COMPOSE) exec minio mc alias set local http://localhost:9000 minioadmin changeme
|
||||
$(COMPOSE) exec minio mc mb local/documents || true
|
||||
$(COMPOSE) exec minio mc mb local/backups || true
|
||||
|
||||
# Создать ES индекс
|
||||
es-init:
|
||||
bash infra/elasticsearch/setup.sh
|
||||
|
||||
# Статистика Celery
|
||||
flower:
|
||||
@echo "Flower доступен по адресу http://localhost:5555"
|
||||
@$(COMPOSE) ps flower
|
||||
|
||||
help:
|
||||
@echo "Академический помощник — команды make:"
|
||||
@echo ""
|
||||
@echo " make dev — запустить в режиме разработки (hot reload)"
|
||||
@echo " make build — собрать Docker образы"
|
||||
@echo " make up — запустить в продакшн режиме"
|
||||
@echo " make down — остановить все сервисы"
|
||||
@echo " make migrate — применить миграции Alembic"
|
||||
@echo " make logs — показать логи всех сервисов"
|
||||
@echo " make shell-api — открыть shell в контейнере api"
|
||||
@echo " make lint — запустить линтер"
|
||||
@echo " make test — запустить тесты"
|
||||
@echo " make clean — удалить все контейнеры и volumes"
|
||||
154
README.md
Normal file
154
README.md
Normal file
@@ -0,0 +1,154 @@
|
||||
# Академический помощник
|
||||
|
||||
Микросервисная система антиплагиата и поиска академических источников.
|
||||
|
||||
Студент вводит тему → система ищет источники → проверяет плагиат → форматирует ГОСТ-библиографию.
|
||||
Всё асинхронно: студент закрыл браузер, получил email когда готово.
|
||||
|
||||
## Архитектура
|
||||
|
||||
```
|
||||
┌─────────────────┐
|
||||
│ Frontend │ React + Vite + TypeScript
|
||||
│ (React SPA) │
|
||||
└────────┬────────┘
|
||||
│ HTTPS
|
||||
┌────────▼────────┐
|
||||
│ API Gateway │ FastAPI, порт 8000
|
||||
│ (FastAPI) │ JWT, Rate Limit, WebSocket
|
||||
└──┬─────────┬───┘
|
||||
│ RabbitMQ│ Celery задачи
|
||||
┌────────────▼──┐ ┌──▼──────────────┐
|
||||
│ worker-gpu │ │ worker-indexer │
|
||||
│ (CUDA/FAISS) │ │ (PDF/DOCX parse) │
|
||||
│ Sem. search │ │ Winnowing/MinHash │
|
||||
│ LLM paraphrase│ └──────────────────┘
|
||||
└────────────────┘
|
||||
│
|
||||
┌─────────▼──────────┐ ┌────────────────────┐
|
||||
│ worker-notifier │ │ worker-gost │
|
||||
│ (SMTP email) │ │ (ГОСТ 7.1/7.0.5) │
|
||||
└────────────────────┘ └────────────────────┘
|
||||
|
||||
Инфраструктура:
|
||||
PostgreSQL 16 · Redis 7 · RabbitMQ 3 · Elasticsearch 8
|
||||
MinIO (4TB) · Ollama (llama3:8b) · NVIDIA RTX 2060
|
||||
```
|
||||
|
||||
## Быстрый старт
|
||||
|
||||
```bash
|
||||
# 1. Клонировать репозиторий
|
||||
git clone https://github.com/jze9/anti-plagiarism.git
|
||||
cd anti-plagiarism
|
||||
|
||||
# 2. Создать файл конфигурации
|
||||
cp .env.example .env
|
||||
# Отредактировать .env — сменить пароли и ключи!
|
||||
|
||||
# 3. Запустить в режиме разработки
|
||||
make dev
|
||||
|
||||
# 4. Применить миграции базы данных
|
||||
make migrate-dev
|
||||
|
||||
# 5. Создать ES индекс
|
||||
make es-init
|
||||
|
||||
# 6. Открыть браузер
|
||||
# Frontend: http://localhost:5173
|
||||
# API docs: http://localhost:8000/api/docs
|
||||
# RabbitMQ: http://localhost:15672 (guest/guest)
|
||||
# Flower: http://localhost:5555
|
||||
# MinIO: http://localhost:9001
|
||||
```
|
||||
|
||||
## Команды
|
||||
|
||||
```bash
|
||||
make dev # Запуск в dev режиме (hot reload)
|
||||
make build # Сборка Docker образов
|
||||
make up # Запуск в продакшн режиме
|
||||
make down # Остановить все сервисы
|
||||
make migrate # Применить Alembic миграции
|
||||
make logs # Логи всех сервисов
|
||||
make shell-api # Shell в контейнере API
|
||||
make shell-gpu # Shell в контейнере GPU воркера
|
||||
make lint # Запустить линтер
|
||||
make test # Запустить тесты
|
||||
make clean # Удалить контейнеры и volumes
|
||||
```
|
||||
|
||||
## Стек технологий
|
||||
|
||||
| Компонент | Технологии |
|
||||
|-----------|-----------|
|
||||
| API Gateway | FastAPI 0.111, Python 3.11, SQLAlchemy 2.0, Alembic |
|
||||
| GPU Worker | sentence-transformers, FAISS-GPU (IVFFlat), Ollama (llama3:8b) |
|
||||
| Indexer | PyMuPDF, python-docx, Winnowing, MinHash LSH |
|
||||
| Очереди | RabbitMQ (брокер) + Celery 5 (воркеры) + Redis (результаты) |
|
||||
| База данных | PostgreSQL 16 |
|
||||
| Поиск | Elasticsearch 8 (BM25) + FAISS GPU (cosine) |
|
||||
| Хранилище | MinIO (S3-совместимый) |
|
||||
| Frontend | React 18, Vite, TypeScript, TailwindCSS, Zustand, React Query v5 |
|
||||
|
||||
## Проверка плагиата (4 уровня)
|
||||
|
||||
1. **Winnowing + MinHash** (~мс) — точное совпадение по fingerprints
|
||||
2. **n-граммы + Jaccard** (~сек) — нечёткое совпадение
|
||||
3. **FAISS GPU cosine** (~мс) — семантическое сходство (порог 0.75)
|
||||
4. **Ollama Llama3** (~2сек) — LLM анализ парафраза (confidence > 0.7)
|
||||
|
||||
## Тарифные планы
|
||||
|
||||
| Тариф | Цена | Поиск/день | Изложений/мес | Плагиат/мес | Одновременно |
|
||||
|-------|------|-----------|--------------|-------------|-------------|
|
||||
| Бесплатный | 0₽ | 10 | 3 | 1 | 1 |
|
||||
| Студенческий | 199₽ | безлимит | 30 | 10 | 2 |
|
||||
| Премиум | 499₽ | безлимит | безлимит | 50 | 5 |
|
||||
| Научный | 999₽ | безлимит | безлимит | безлимит | 10 |
|
||||
|
||||
## Парсеры источников
|
||||
|
||||
```bash
|
||||
# Запарсить OpenAlex
|
||||
python scripts/run_parser.py openalex \
|
||||
--query "машинное обучение" \
|
||||
--limit 10000 \
|
||||
--output /data/processed
|
||||
|
||||
# КиберЛенинка
|
||||
python scripts/run_parser.py cyberleninka \
|
||||
--query "нейронные сети" \
|
||||
--limit 1000
|
||||
|
||||
# arXiv
|
||||
python scripts/run_parser.py arxiv \
|
||||
--query "deep learning" \
|
||||
--categories cs.AI cs.LG \
|
||||
--limit 5000
|
||||
```
|
||||
|
||||
## Переменные окружения
|
||||
|
||||
Смотри `.env.example` для полного списка переменных.
|
||||
Обязательно смените `SECRET_KEY`, `POSTGRES_PASSWORD`, `MINIO_SECRET_KEY`.
|
||||
|
||||
## Продакшн деплой
|
||||
|
||||
```bash
|
||||
# На сервере
|
||||
cp .env.example .env
|
||||
nano .env # Настроить все пароли и ключи
|
||||
|
||||
make build
|
||||
make up
|
||||
make migrate
|
||||
|
||||
# SSL сертификат (Let's Encrypt)
|
||||
certbot --nginx -d academic.jze9.ru
|
||||
```
|
||||
|
||||
## Лицензия
|
||||
|
||||
MIT
|
||||
63
docker-compose.dev.yml
Normal file
63
docker-compose.dev.yml
Normal file
@@ -0,0 +1,63 @@
|
||||
version: "3.9"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Override для разработки: hot reload, volume mounts, DEBUG режим
|
||||
# Использование: docker compose -f docker-compose.yml -f docker-compose.dev.yml up
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
services:
|
||||
|
||||
api:
|
||||
build:
|
||||
context: ./services/api
|
||||
dockerfile: Dockerfile
|
||||
command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir /app/app
|
||||
volumes:
|
||||
- ./services/api:/app
|
||||
environment:
|
||||
DEBUG: "true"
|
||||
ENVIRONMENT: development
|
||||
|
||||
worker-gpu:
|
||||
build:
|
||||
context: ./services/worker-gpu
|
||||
dockerfile: Dockerfile
|
||||
command: celery -A app.celery_app worker -Q queue.gpu -c 1 -n gpu@%h --loglevel=debug
|
||||
volumes:
|
||||
- ./services/worker-gpu:/app
|
||||
environment:
|
||||
DEBUG: "true"
|
||||
ENVIRONMENT: development
|
||||
|
||||
worker-indexer:
|
||||
build:
|
||||
context: ./services/worker-indexer
|
||||
dockerfile: Dockerfile
|
||||
command: celery -A app.celery_app worker -Q queue.index -c 2 -n indexer@%h --loglevel=debug
|
||||
volumes:
|
||||
- ./services/worker-indexer:/app
|
||||
environment:
|
||||
DEBUG: "true"
|
||||
ENVIRONMENT: development
|
||||
|
||||
worker-notifier:
|
||||
build:
|
||||
context: ./services/worker-notifier
|
||||
dockerfile: Dockerfile
|
||||
command: celery -A app.celery_app worker -Q queue.notify -c 2 -n notifier@%h --loglevel=debug
|
||||
volumes:
|
||||
- ./services/worker-notifier:/app
|
||||
environment:
|
||||
DEBUG: "true"
|
||||
ENVIRONMENT: development
|
||||
|
||||
worker-gost:
|
||||
build:
|
||||
context: ./services/worker-gost
|
||||
dockerfile: Dockerfile
|
||||
command: celery -A app.celery_app worker -Q queue.gost -c 2 -n gost@%h --loglevel=debug
|
||||
volumes:
|
||||
- ./services/worker-gost:/app
|
||||
environment:
|
||||
DEBUG: "true"
|
||||
ENVIRONMENT: development
|
||||
259
docker-compose.yml
Normal file
259
docker-compose.yml
Normal file
@@ -0,0 +1,259 @@
|
||||
version: "3.9"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Академический помощник — Production Docker Compose
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
networks:
|
||||
antiplagiator:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
rabbitmq_data:
|
||||
elasticsearch_data:
|
||||
minio_data:
|
||||
ollama_data:
|
||||
faiss_index:
|
||||
|
||||
# ─── Общие переменные окружения для app-сервисов ────────────────────────────
|
||||
x-app-env: &app-env
|
||||
env_file: .env
|
||||
networks:
|
||||
- antiplagiator
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# ИНФРАСТРУКТУРА
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
postgres:
|
||||
image: postgres:16
|
||||
container_name: antiplagiator-postgres
|
||||
environment:
|
||||
POSTGRES_DB: antiplagiator
|
||||
POSTGRES_USER: antiplagiator
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme}
|
||||
volumes:
|
||||
- ./data/postgres:/var/lib/postgresql/data
|
||||
- ./infra/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
networks:
|
||||
- antiplagiator
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U antiplagiator -d antiplagiator"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: antiplagiator-redis
|
||||
command: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru
|
||||
volumes:
|
||||
- ./data/redis:/data
|
||||
networks:
|
||||
- antiplagiator
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3-management
|
||||
container_name: antiplagiator-rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-guest}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-guest}
|
||||
volumes:
|
||||
- ./data/rabbitmq:/var/lib/rabbitmq
|
||||
networks:
|
||||
- antiplagiator
|
||||
ports:
|
||||
- "15672:15672" # Management UI
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "check_port_connectivity"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
elasticsearch:
|
||||
image: elasticsearch:8.13.0
|
||||
container_name: antiplagiator-elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- xpack.ml.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms2g -Xmx2g
|
||||
- cluster.name=antiplagiator
|
||||
- bootstrap.memory_lock=true
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- ./data/elasticsearch:/usr/share/elasticsearch/data
|
||||
networks:
|
||||
- antiplagiator
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -qv '\"status\":\"red\"'"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: antiplagiator-minio
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-changeme}
|
||||
volumes:
|
||||
- ./data/minio:/data
|
||||
networks:
|
||||
- antiplagiator
|
||||
ports:
|
||||
- "9001:9001" # Console UI
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
ollama:
|
||||
image: ollama/ollama:latest
|
||||
container_name: antiplagiator-ollama
|
||||
volumes:
|
||||
- ./data/ollama:/root/.ollama
|
||||
networks:
|
||||
- antiplagiator
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# ПРИЛОЖЕНИЕ
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
api:
|
||||
build:
|
||||
context: ./services/api
|
||||
dockerfile: Dockerfile
|
||||
container_name: antiplagiator-api
|
||||
<<: *app-env
|
||||
ports:
|
||||
- "8000:8000"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
|
||||
worker-gpu:
|
||||
build:
|
||||
context: ./services/worker-gpu
|
||||
dockerfile: Dockerfile
|
||||
container_name: antiplagiator-worker-gpu
|
||||
<<: *app-env
|
||||
command: celery -A app.celery_app worker -Q queue.gpu -c 1 -n gpu@%h --loglevel=info
|
||||
volumes:
|
||||
- faiss_index:/data/index
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
worker-indexer:
|
||||
build:
|
||||
context: ./services/worker-indexer
|
||||
dockerfile: Dockerfile
|
||||
container_name: antiplagiator-worker-indexer
|
||||
<<: *app-env
|
||||
command: celery -A app.celery_app worker -Q queue.index -c 4 -n indexer@%h --loglevel=info
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
|
||||
worker-notifier:
|
||||
build:
|
||||
context: ./services/worker-notifier
|
||||
dockerfile: Dockerfile
|
||||
container_name: antiplagiator-worker-notifier
|
||||
<<: *app-env
|
||||
command: celery -A app.celery_app worker -Q queue.notify -c 16 -n notifier@%h --loglevel=info
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
|
||||
worker-gost:
|
||||
build:
|
||||
context: ./services/worker-gost
|
||||
dockerfile: Dockerfile
|
||||
container_name: antiplagiator-worker-gost
|
||||
<<: *app-env
|
||||
command: celery -A app.celery_app worker -Q queue.gost -c 8 -n gost@%h --loglevel=info
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
|
||||
flower:
|
||||
image: mher/flower:2.0
|
||||
container_name: antiplagiator-flower
|
||||
command: celery --broker=${RABBITMQ_URL:-amqp://guest:guest@rabbitmq:5672/} flower --port=5555
|
||||
networks:
|
||||
- antiplagiator
|
||||
ports:
|
||||
- "5555:5555"
|
||||
depends_on:
|
||||
- rabbitmq
|
||||
- redis
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CELERY_BROKER_URL: ${RABBITMQ_URL:-amqp://guest:guest@rabbitmq:5672/}
|
||||
CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
|
||||
75
infra/elasticsearch/mappings.json
Normal file
75
infra/elasticsearch/mappings.json
Normal file
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"russian_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "russian_stop", "russian_stemmer"]
|
||||
},
|
||||
"english_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "english_stop", "english_stemmer"]
|
||||
},
|
||||
"multilingual_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "russian_stop", "russian_stemmer", "english_stop"]
|
||||
}
|
||||
},
|
||||
"filter": {
|
||||
"russian_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_russian_"
|
||||
},
|
||||
"russian_stemmer": {
|
||||
"type": "stemmer",
|
||||
"language": "russian"
|
||||
},
|
||||
"english_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_english_"
|
||||
},
|
||||
"english_stemmer": {
|
||||
"type": "stemmer",
|
||||
"language": "english"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"doc_id": {"type": "long"},
|
||||
"source": {"type": "keyword"},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer",
|
||||
"fields": {
|
||||
"keyword": {"type": "keyword", "ignore_above": 512},
|
||||
"ru": {"type": "text", "analyzer": "russian_analyzer"},
|
||||
"en": {"type": "text", "analyzer": "english_analyzer"}
|
||||
}
|
||||
},
|
||||
"abstract": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer",
|
||||
"fields": {
|
||||
"ru": {"type": "text", "analyzer": "russian_analyzer"},
|
||||
"en": {"type": "text", "analyzer": "english_analyzer"}
|
||||
}
|
||||
},
|
||||
"authors": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer"
|
||||
},
|
||||
"year": {"type": "integer"},
|
||||
"lang": {"type": "keyword"},
|
||||
"journal": {"type": "keyword", "ignore_above": 512},
|
||||
"doi": {"type": "keyword"},
|
||||
"url": {"type": "keyword", "index": false}
|
||||
}
|
||||
}
|
||||
}
|
||||
36
infra/elasticsearch/setup.sh
Executable file
36
infra/elasticsearch/setup.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Создание ES индекса с маппингом для документов
|
||||
# Запускать после старта контейнера Elasticsearch
|
||||
|
||||
set -e
|
||||
|
||||
ES_URL="${ELASTICSEARCH_URL:-http://localhost:9200}"
|
||||
INDEX_NAME="documents"
|
||||
MAPPINGS_FILE="$(dirname "$0")/mappings.json"
|
||||
|
||||
echo "Ожидание готовности Elasticsearch..."
|
||||
until curl -s "$ES_URL/_cluster/health" | grep -qv '"status":"red"'; do
|
||||
sleep 2
|
||||
done
|
||||
echo "Elasticsearch готов."
|
||||
|
||||
# Проверить существование индекса
|
||||
if curl -s -o /dev/null -w "%{http_code}" "$ES_URL/$INDEX_NAME" | grep -q "200"; then
|
||||
echo "Индекс '$INDEX_NAME' уже существует."
|
||||
read -p "Пересоздать? (y/N): " confirm
|
||||
if [[ $confirm != "y" ]]; then
|
||||
echo "Пропускаем."
|
||||
exit 0
|
||||
fi
|
||||
echo "Удаление старого индекса..."
|
||||
curl -s -X DELETE "$ES_URL/$INDEX_NAME"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "Создание индекса '$INDEX_NAME'..."
|
||||
curl -s -X PUT "$ES_URL/$INDEX_NAME" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @"$MAPPINGS_FILE" | python3 -m json.tool
|
||||
|
||||
echo ""
|
||||
echo "Индекс '$INDEX_NAME' создан."
|
||||
113
infra/nginx/nginx.conf
Normal file
113
infra/nginx/nginx.conf
Normal file
@@ -0,0 +1,113 @@
|
||||
# Nginx конфиг для academic.jze9.ru
|
||||
# Продакшн: SSL + proxy to FastAPI + React static
|
||||
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
# Логи
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent"';
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# Gzip
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
|
||||
gzip_min_length 1024;
|
||||
gzip_vary on;
|
||||
|
||||
# Лимиты
|
||||
client_max_body_size 100M;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 30s;
|
||||
proxy_send_timeout 300s;
|
||||
|
||||
# Upstream
|
||||
upstream api {
|
||||
server api:8000;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# HTTP → HTTPS редирект
|
||||
server {
|
||||
listen 80;
|
||||
server_name academic.jze9.ru;
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
|
||||
# HTTPS основной сервер
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name academic.jze9.ru;
|
||||
|
||||
# SSL (Let's Encrypt)
|
||||
ssl_certificate /etc/letsencrypt/live/academic.jze9.ru/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/academic.jze9.ru/privkey.pem;
|
||||
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
|
||||
ssl_prefer_server_ciphers off;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
ssl_session_timeout 1d;
|
||||
|
||||
# HSTS
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
|
||||
# Security headers
|
||||
add_header X-Content-Type-Options nosniff;
|
||||
add_header X-Frame-Options DENY;
|
||||
add_header X-XSS-Protection "1; mode=block";
|
||||
|
||||
# ── API proxy ──────────────────────────────────────────────────────────
|
||||
location /api/ {
|
||||
proxy_pass http://api/api/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
}
|
||||
|
||||
# ── WebSocket ──────────────────────────────────────────────────────────
|
||||
location /ws/ {
|
||||
proxy_pass http://api/ws/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_read_timeout 86400;
|
||||
}
|
||||
|
||||
# ── React SPA (статика) ────────────────────────────────────────────────
|
||||
root /var/www/academic;
|
||||
index index.html;
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
expires 1d;
|
||||
add_header Cache-Control "public, no-transform";
|
||||
}
|
||||
|
||||
# Кэшировать статические ресурсы
|
||||
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff2|woff)$ {
|
||||
expires 30d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
}
|
||||
14
infra/postgres/init.sql
Normal file
14
infra/postgres/init.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
-- Инициализация PostgreSQL для Академического помощника
|
||||
-- Выполняется при первом старте контейнера
|
||||
|
||||
-- Расширения
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm"; -- Для нечёткого поиска по тексту
|
||||
|
||||
-- Настройки подключения (уже создан через POSTGRES_USER/POSTGRES_DB env)
|
||||
-- Дополнительные гранты
|
||||
GRANT ALL PRIVILEGES ON DATABASE antiplagiator TO antiplagiator;
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO antiplagiator;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO antiplagiator;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO antiplagiator;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO antiplagiator;
|
||||
218
scripts/parsers/arxiv.py
Normal file
218
scripts/parsers/arxiv.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""Парсер arXiv API.
|
||||
|
||||
arXiv — открытый препринт-сервер в физике, математике, CS и биологии.
|
||||
API: https://info.arxiv.org/help/api/index.html
|
||||
|
||||
Использует Atom XML API.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from base import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ARXIV_API = "https://export.arxiv.org/api/query"
|
||||
RATE_LIMIT_DELAY = 3.0 # arXiv требует не более 1 запроса/3сек
|
||||
|
||||
# Namespace XML
|
||||
NS = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
"dc": "http://purl.org/dc/elements/1.1/",
|
||||
"opensearch": "http://a9.com/-/spec/opensearch/1.1/",
|
||||
}
|
||||
|
||||
|
||||
class ArxivParser(BaseParser):
|
||||
"""Парсер arXiv API."""
|
||||
|
||||
source_name = "arxiv"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.client = httpx.Client(
|
||||
headers={"User-Agent": "AcademicHelper/1.0 (noreply@jze9.ru)"},
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
query: str = "",
|
||||
limit: int = 500,
|
||||
categories: list[str] | None = None,
|
||||
year_from: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Получить препринты из arXiv.
|
||||
|
||||
Args:
|
||||
query: Поисковый запрос
|
||||
limit: Максимальное количество документов
|
||||
categories: Список категорий arXiv (cs.AI, math.ST и т.д.)
|
||||
year_from: Год публикации от
|
||||
|
||||
Returns:
|
||||
Список сырых словарей
|
||||
"""
|
||||
results = []
|
||||
start = 0
|
||||
max_results = min(100, limit)
|
||||
|
||||
# Формируем запрос
|
||||
search_query = ""
|
||||
if query:
|
||||
search_query = f"all:{query}"
|
||||
if categories:
|
||||
cat_query = " OR ".join(f"cat:{c}" for c in categories)
|
||||
search_query = f"({search_query}) AND ({cat_query})" if search_query else f"({cat_query})"
|
||||
|
||||
if not search_query:
|
||||
search_query = "all:neural network" # Default query
|
||||
|
||||
while len(results) < limit:
|
||||
try:
|
||||
params = {
|
||||
"search_query": search_query,
|
||||
"start": start,
|
||||
"max_results": min(max_results, limit - len(results)),
|
||||
"sortBy": "submittedDate",
|
||||
"sortOrder": "descending",
|
||||
}
|
||||
|
||||
response = self.client.get(ARXIV_API, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
entries = self._parse_xml(response.text)
|
||||
if not entries:
|
||||
break
|
||||
|
||||
results.extend(entries)
|
||||
start += len(entries)
|
||||
|
||||
if len(entries) < max_results:
|
||||
break
|
||||
|
||||
time.sleep(RATE_LIMIT_DELAY)
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"arXiv HTTP ошибка: {e.response.status_code}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка запроса arXiv: {e}")
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
def _parse_xml(self, xml_text: str) -> list[dict[str, Any]]:
|
||||
"""Парсить Atom XML ответ arXiv."""
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"Ошибка парсинга XML arXiv: {e}")
|
||||
return []
|
||||
|
||||
entries = []
|
||||
for entry in root.findall("atom:entry", NS):
|
||||
entries.append(self._parse_entry(entry))
|
||||
|
||||
return entries
|
||||
|
||||
def _parse_entry(self, entry: ET.Element) -> dict[str, Any]:
|
||||
"""Парсить один entry из Atom XML."""
|
||||
def text(path: str) -> str | None:
|
||||
elem = entry.find(path, NS)
|
||||
return elem.text.strip() if elem is not None and elem.text else None
|
||||
|
||||
arxiv_id = text("atom:id") or ""
|
||||
# Нормализовать: убрать версию
|
||||
if "abs/" in arxiv_id:
|
||||
arxiv_id = arxiv_id.split("abs/")[-1].split("v")[0]
|
||||
|
||||
# Авторы
|
||||
authors = []
|
||||
for author_elem in entry.findall("atom:author", NS):
|
||||
name = text("atom:name") if author_elem.find("atom:name", NS) is not None else None
|
||||
if name:
|
||||
parts = name.strip().split()
|
||||
if len(parts) >= 2:
|
||||
authors.append({
|
||||
"last_name": parts[-1],
|
||||
"first_name": " ".join(parts[:-1]),
|
||||
})
|
||||
elif parts:
|
||||
authors.append({"last_name": parts[0], "first_name": ""})
|
||||
|
||||
# Год из published
|
||||
published = text("atom:published") or ""
|
||||
year = int(published[:4]) if published[:4].isdigit() else None
|
||||
|
||||
# DOI
|
||||
doi = None
|
||||
for link in entry.findall("atom:link", NS):
|
||||
if link.get("title") == "doi":
|
||||
doi = link.get("href", "").replace("http://dx.doi.org/", "")
|
||||
break
|
||||
|
||||
# URL
|
||||
url = None
|
||||
for link in entry.findall("atom:link", NS):
|
||||
if link.get("type") == "text/html":
|
||||
url = link.get("href")
|
||||
break
|
||||
|
||||
# Категории
|
||||
categories = [
|
||||
cat.get("term", "")
|
||||
for cat in entry.findall("atom:category", NS)
|
||||
]
|
||||
|
||||
return {
|
||||
"id": arxiv_id,
|
||||
"title": text("atom:title") or "",
|
||||
"abstract": text("atom:summary") or "",
|
||||
"authors": authors,
|
||||
"year": year,
|
||||
"published": published,
|
||||
"doi": doi,
|
||||
"url": url,
|
||||
"categories": categories,
|
||||
"journal": "arXiv",
|
||||
}
|
||||
|
||||
def transform(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Преобразовать документ arXiv в унифицированный формат."""
|
||||
ext_id = raw.get("id", "")
|
||||
if not ext_id:
|
||||
return {}
|
||||
|
||||
# Нормализовать авторов
|
||||
authors = self.normalize_authors(raw.get("authors", []))
|
||||
|
||||
# Определить язык (arXiv — преимущественно английский)
|
||||
lang = "en"
|
||||
|
||||
# Категории как JSON
|
||||
categories = raw.get("categories", [])
|
||||
|
||||
return {
|
||||
"source": self.source_name,
|
||||
"ext_id": f"arxiv:{ext_id}",
|
||||
"doi": raw.get("doi") or None,
|
||||
"title": (raw.get("title") or "").replace("\n", " ").strip() or None,
|
||||
"authors": authors,
|
||||
"year": raw.get("year"),
|
||||
"lang": lang,
|
||||
"journal": "arXiv",
|
||||
"volume": None,
|
||||
"issue": None,
|
||||
"pages": None,
|
||||
"abstract": (raw.get("abstract") or "").replace("\n", " ").strip() or None,
|
||||
"url": raw.get("url"),
|
||||
"full_text": None,
|
||||
}
|
||||
166
scripts/parsers/base.py
Normal file
166
scripts/parsers/base.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""Базовый класс для всех парсеров источников.
|
||||
|
||||
Определяет унифицированный интерфейс и формат документов.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Унифицированный формат документа
|
||||
UNIFIED_SCHEMA = {
|
||||
"source": str, # openalex, cyberleninka, arxiv, wikipedia_ru, wikipedia_en
|
||||
"ext_id": str, # Внешний ID (уникален в рамках источника)
|
||||
"doi": "str|None",
|
||||
"title": str,
|
||||
"authors": list, # [{"last_name": str, "first_name": str, "initials": str}]
|
||||
"year": "int|None",
|
||||
"lang": "str|None", # ru, en
|
||||
"journal": "str|None",
|
||||
"volume": "str|None",
|
||||
"issue": "str|None",
|
||||
"pages": "str|None",
|
||||
"abstract": "str|None",
|
||||
"url": "str|None",
|
||||
"full_text": "str|None", # Для fingerprinting (не хранится в PostgreSQL)
|
||||
}
|
||||
|
||||
|
||||
class BaseParser(ABC):
|
||||
"""Базовый класс парсера источника."""
|
||||
|
||||
source_name: str = "unknown"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.logger = logging.getLogger(f"parser.{self.source_name}")
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self, **kwargs) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Получить сырые документы из источника.
|
||||
|
||||
Args:
|
||||
**kwargs: Специфичные для источника параметры (query, limit и т.д.)
|
||||
|
||||
Returns:
|
||||
Список сырых словарей из API источника
|
||||
"""
|
||||
...
|
||||
|
||||
def transform(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Привести сырой документ к унифицированному формату.
|
||||
|
||||
Args:
|
||||
raw: Сырой словарь из API источника
|
||||
|
||||
Returns:
|
||||
Документ в унифицированном формате
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"Парсер {self.source_name!r} должен реализовать метод transform()"
|
||||
)
|
||||
|
||||
def save_jsonl(self, docs: list[dict[str, Any]], output_path: Path) -> None:
|
||||
"""
|
||||
Сохранить документы в JSONL формате (один JSON объект на строку).
|
||||
|
||||
Args:
|
||||
docs: Список документов
|
||||
output_path: Путь к выходному файлу (дополняется, не перезаписывается)
|
||||
"""
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output_path.open("a", encoding="utf-8") as f:
|
||||
for doc in docs:
|
||||
f.write(json.dumps(doc, ensure_ascii=False) + "\n")
|
||||
|
||||
def run(self, output_dir: Path, **kwargs) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Запустить парсер: fetch → transform → save.
|
||||
|
||||
Args:
|
||||
output_dir: Директория для сохранения результатов
|
||||
**kwargs: Параметры для метода fetch()
|
||||
|
||||
Returns:
|
||||
Список трансформированных документов
|
||||
"""
|
||||
self.logger.info(f"[{self.source_name}] Начало парсинга...")
|
||||
|
||||
raw_docs = self.fetch(**kwargs)
|
||||
self.logger.info(f"[{self.source_name}] Получено {len(raw_docs)} документов")
|
||||
|
||||
transformed = []
|
||||
errors = 0
|
||||
for raw in raw_docs:
|
||||
try:
|
||||
doc = self.transform(raw)
|
||||
if doc and doc.get("title") and doc.get("ext_id"):
|
||||
transformed.append(doc)
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
self.logger.warning(f"Ошибка трансформации: {e}")
|
||||
|
||||
if errors:
|
||||
self.logger.warning(f"[{self.source_name}] Ошибок трансформации: {errors}")
|
||||
|
||||
out_file = output_dir / f"{self.source_name}.jsonl"
|
||||
self.save_jsonl(transformed, out_file)
|
||||
|
||||
self.logger.info(
|
||||
f"[{self.source_name}] Сохранено {len(transformed)} документов → {out_file}"
|
||||
)
|
||||
|
||||
return transformed
|
||||
|
||||
@staticmethod
|
||||
def normalize_authors(raw_authors: list) -> list[dict[str, str]]:
|
||||
"""
|
||||
Нормализовать список авторов к единому формату.
|
||||
|
||||
Args:
|
||||
raw_authors: Авторы из API (разный формат у каждого источника)
|
||||
|
||||
Returns:
|
||||
Список {"last_name": str, "first_name": str, "initials": str}
|
||||
"""
|
||||
result = []
|
||||
for author in raw_authors:
|
||||
if isinstance(author, str):
|
||||
parts = author.strip().split()
|
||||
last_name = parts[0] if parts else ""
|
||||
first_name = " ".join(parts[1:]) if len(parts) > 1 else ""
|
||||
elif isinstance(author, dict):
|
||||
last_name = (
|
||||
author.get("last_name") or
|
||||
author.get("family") or
|
||||
author.get("surname") or ""
|
||||
).strip()
|
||||
first_name = (
|
||||
author.get("first_name") or
|
||||
author.get("given") or ""
|
||||
).strip()
|
||||
else:
|
||||
continue
|
||||
|
||||
if not last_name:
|
||||
continue
|
||||
|
||||
# Инициалы
|
||||
initials = ""
|
||||
if first_name:
|
||||
parts = first_name.split()
|
||||
initials = "".join(p[0].upper() + "." for p in parts if p)
|
||||
|
||||
result.append({
|
||||
"last_name": last_name,
|
||||
"first_name": first_name,
|
||||
"initials": initials,
|
||||
})
|
||||
|
||||
return result
|
||||
233
scripts/parsers/cyberleninka.py
Normal file
233
scripts/parsers/cyberleninka.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""Парсер КиберЛенинки.
|
||||
|
||||
КиберЛенинка — крупнейший русскоязычный открытый репозиторий научных статей.
|
||||
Сайт: https://cyberleninka.ru
|
||||
|
||||
Используем парсинг HTML страниц со строгим rate limiting (1 req/sec),
|
||||
а не недокументированное API.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from base import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_URL = "https://cyberleninka.ru"
|
||||
SEARCH_URL = f"{BASE_URL}/api/search"
|
||||
ARTICLE_URL = f"{BASE_URL}/article"
|
||||
RATE_LIMIT_DELAY = 1.0 # секунд между запросами
|
||||
|
||||
|
||||
class CyberLeninkaParser(BaseParser):
|
||||
"""Парсер КиберЛенинки через HTML + API поиска."""
|
||||
|
||||
source_name = "cyberleninka"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.client = httpx.Client(
|
||||
headers={
|
||||
"User-Agent": "AcademicHelper/1.0 (noreply@jze9.ru)",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||||
},
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
query: str = "",
|
||||
limit: int = 500,
|
||||
subject: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Получить статьи из КиберЛенинки.
|
||||
|
||||
Args:
|
||||
query: Поисковый запрос
|
||||
limit: Максимальное количество статей
|
||||
subject: Предметная область (опционально)
|
||||
|
||||
Returns:
|
||||
Список сырых словарей статей
|
||||
"""
|
||||
results = []
|
||||
page = 0
|
||||
|
||||
while len(results) < limit:
|
||||
try:
|
||||
params: dict[str, Any] = {
|
||||
"q": query,
|
||||
"size": min(10, limit - len(results)),
|
||||
"from": page * 10,
|
||||
}
|
||||
|
||||
response = self.client.get(SEARCH_URL, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
items = data.get("articles") or []
|
||||
|
||||
if not items:
|
||||
break
|
||||
|
||||
results.extend(items)
|
||||
page += 1
|
||||
|
||||
if len(items) < 10:
|
||||
break
|
||||
|
||||
time.sleep(RATE_LIMIT_DELAY)
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"КиберЛенинка HTTP ошибка: {e.response.status_code}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка запроса КиберЛенинки: {e}")
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
def transform(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Преобразовать статью КиберЛенинки в унифицированный формат.
|
||||
|
||||
Структура ответа API поиска КиберЛенинки:
|
||||
- id: числовой ID
|
||||
- name: название
|
||||
- authors: строка с авторами
|
||||
- journal: название журнала
|
||||
- year: год
|
||||
- annotation: аннотация
|
||||
- link: путь к статье
|
||||
"""
|
||||
raw_id = raw.get("id") or raw.get("link", "")
|
||||
ext_id = str(raw_id)
|
||||
|
||||
if not ext_id:
|
||||
return {}
|
||||
|
||||
# Авторы (строка вида "Иванов И.И., Петров П.П.")
|
||||
authors_str = raw.get("authors", "") or ""
|
||||
authors = _parse_cyberleninka_authors(authors_str)
|
||||
|
||||
# Год
|
||||
year_raw = raw.get("year")
|
||||
year = None
|
||||
if year_raw:
|
||||
try:
|
||||
year = int(str(year_raw)[:4])
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# URL
|
||||
link = raw.get("link", "")
|
||||
url = f"{BASE_URL}{link}" if link.startswith("/") else link or None
|
||||
|
||||
return {
|
||||
"source": self.source_name,
|
||||
"ext_id": ext_id,
|
||||
"doi": raw.get("doi") or None,
|
||||
"title": (raw.get("name") or "").strip() or None,
|
||||
"authors": authors,
|
||||
"year": year,
|
||||
"lang": "ru", # КиберЛенинка — только русскоязычные
|
||||
"journal": raw.get("journal") or None,
|
||||
"volume": raw.get("volume") or None,
|
||||
"issue": raw.get("number") or None,
|
||||
"pages": raw.get("pages") or None,
|
||||
"abstract": (raw.get("annotation") or "").strip() or None,
|
||||
"url": url,
|
||||
"full_text": None,
|
||||
}
|
||||
|
||||
def fetch_article_details(self, url: str) -> dict[str, Any]:
|
||||
"""
|
||||
Получить детали статьи из HTML страницы.
|
||||
|
||||
Извлекает метатеги citation_* по Dublin Core / Google Scholar стандарту.
|
||||
|
||||
Args:
|
||||
url: URL страницы статьи
|
||||
|
||||
Returns:
|
||||
Словарь с дополнительными метаданными
|
||||
"""
|
||||
try:
|
||||
time.sleep(RATE_LIMIT_DELAY)
|
||||
response = self.client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
meta = {}
|
||||
|
||||
# Извлечь citation_* мета теги
|
||||
for tag in soup.find_all("meta"):
|
||||
name = tag.get("name", "")
|
||||
content = tag.get("content", "")
|
||||
if name.startswith("citation_") and content:
|
||||
key = name[9:] # Убрать "citation_"
|
||||
meta[key] = content
|
||||
|
||||
return {
|
||||
"doi": meta.get("doi"),
|
||||
"title": meta.get("title"),
|
||||
"abstract": meta.get("abstract"),
|
||||
"year": meta.get("publication_date", "")[:4] or None,
|
||||
"journal": meta.get("journal_title"),
|
||||
"volume": meta.get("volume"),
|
||||
"issue": meta.get("issue"),
|
||||
"pages": f"{meta.get('firstpage', '')}-{meta.get('lastpage', '')}".strip("-") or None,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Ошибка получения деталей статьи {url}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def _parse_cyberleninka_authors(authors_str: str) -> list[dict[str, str]]:
|
||||
"""
|
||||
Разбить строку авторов КиберЛенинки на список.
|
||||
|
||||
Форматы: "Иванов И.И.", "Иванов И.И., Петров П.П."
|
||||
"""
|
||||
if not authors_str.strip():
|
||||
return []
|
||||
|
||||
results = []
|
||||
# Разделитель — запятая, но не внутри инициалов
|
||||
parts = re.split(r",\s+(?=[А-ЯA-Z])", authors_str.strip())
|
||||
|
||||
for part in parts:
|
||||
part = part.strip().rstrip(",")
|
||||
words = part.split()
|
||||
|
||||
if not words:
|
||||
continue
|
||||
|
||||
# Ищем паттерн: Фамилия Инициалы (последнее слово — инициалы)
|
||||
if len(words) >= 2:
|
||||
# Проверить, последнее ли слово — инициалы (содержит точки)
|
||||
if "." in words[-1]:
|
||||
last_name = " ".join(words[:-1])
|
||||
initials = words[-1]
|
||||
else:
|
||||
last_name = words[0]
|
||||
initials = "".join(w[0].upper() + "." for w in words[1:] if w)
|
||||
else:
|
||||
last_name = words[0]
|
||||
initials = ""
|
||||
|
||||
results.append({
|
||||
"last_name": last_name,
|
||||
"first_name": "",
|
||||
"initials": initials,
|
||||
})
|
||||
|
||||
return results
|
||||
245
scripts/parsers/openalex.py
Normal file
245
scripts/parsers/openalex.py
Normal file
@@ -0,0 +1,245 @@
|
||||
"""Парсер OpenAlex API.
|
||||
|
||||
OpenAlex — открытая академическая база данных с >250 млн работ.
|
||||
API: https://docs.openalex.org/
|
||||
|
||||
Особенности:
|
||||
- Cursor-based пагинация (не offset, чтобы не было дублей)
|
||||
- Rate limit: 100,000 запросов/день без ключа
|
||||
- Идемпотентность: проверка по ext_id перед добавлением
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Any, Generator
|
||||
|
||||
import httpx
|
||||
|
||||
from base import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENALEX_API = "https://api.openalex.org"
|
||||
DEFAULT_EMAIL = "noreply@jze9.ru" # Для вежливого агента
|
||||
|
||||
|
||||
class OpenAlexParser(BaseParser):
|
||||
"""Парсер OpenAlex API с cursor-based пагинацией."""
|
||||
|
||||
source_name = "openalex"
|
||||
|
||||
def __init__(self, email: str = DEFAULT_EMAIL) -> None:
|
||||
super().__init__()
|
||||
self.email = email
|
||||
self.client = httpx.Client(
|
||||
headers={"User-Agent": f"AcademicHelper/1.0 ({email})"},
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
query: str = "",
|
||||
limit: int = 1000,
|
||||
lang: str | None = None,
|
||||
year_from: int | None = None,
|
||||
year_to: int | None = None,
|
||||
type_filter: str = "journal-article",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Получить документы из OpenAlex.
|
||||
|
||||
Args:
|
||||
query: Поисковый запрос
|
||||
limit: Максимальное количество документов
|
||||
lang: Язык ('ru', 'en', None = все)
|
||||
year_from: Год публикации от
|
||||
year_to: Год публикации до
|
||||
type_filter: Тип документа (journal-article, book, и т.д.)
|
||||
|
||||
Returns:
|
||||
Список сырых словарей из OpenAlex API
|
||||
"""
|
||||
results = []
|
||||
|
||||
for page in self._paginate(
|
||||
query=query,
|
||||
limit=limit,
|
||||
lang=lang,
|
||||
year_from=year_from,
|
||||
year_to=year_to,
|
||||
type_filter=type_filter,
|
||||
):
|
||||
results.extend(page)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
def _paginate(
|
||||
self,
|
||||
query: str,
|
||||
limit: int,
|
||||
lang: str | None,
|
||||
year_from: int | None,
|
||||
year_to: int | None,
|
||||
type_filter: str,
|
||||
) -> Generator[list[dict], None, None]:
|
||||
"""Cursor-based пагинация OpenAlex."""
|
||||
cursor = "*"
|
||||
per_page = min(200, limit)
|
||||
total_fetched = 0
|
||||
|
||||
while total_fetched < limit:
|
||||
params: dict[str, Any] = {
|
||||
"per-page": per_page,
|
||||
"cursor": cursor,
|
||||
"mailto": self.email,
|
||||
}
|
||||
|
||||
# Фильтры
|
||||
filters = [f"type:{type_filter}", "is_oa:true"]
|
||||
|
||||
if query:
|
||||
params["search"] = query
|
||||
|
||||
if lang:
|
||||
filters.append(f"language:{lang}")
|
||||
|
||||
if year_from and year_to:
|
||||
filters.append(f"publication_year:{year_from}-{year_to}")
|
||||
elif year_from:
|
||||
filters.append(f"publication_year:>{year_from}")
|
||||
elif year_to:
|
||||
filters.append(f"publication_year:<{year_to}")
|
||||
|
||||
params["filter"] = ",".join(filters)
|
||||
|
||||
try:
|
||||
response = self.client.get(f"{OPENALEX_API}/works", params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
works = data.get("results", [])
|
||||
if not works:
|
||||
break
|
||||
|
||||
yield works
|
||||
total_fetched += len(works)
|
||||
|
||||
# Следующий курсор
|
||||
cursor = data.get("meta", {}).get("next_cursor")
|
||||
if not cursor:
|
||||
break
|
||||
|
||||
# Rate limiting: 10 запросов/сек без ключа
|
||||
time.sleep(0.1)
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"OpenAlex HTTP ошибка: {e.response.status_code}")
|
||||
if e.response.status_code == 429:
|
||||
logger.warning("Rate limit! Ожидаем 60 секунд...")
|
||||
time.sleep(60)
|
||||
continue
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка запроса OpenAlex: {e}")
|
||||
break
|
||||
|
||||
def transform(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Преобразовать документ OpenAlex в унифицированный формат.
|
||||
|
||||
OpenAlex структура:
|
||||
- id: строка вида "https://openalex.org/W..."
|
||||
- doi: DOI ссылка
|
||||
- title: название
|
||||
- authorships: список авторов
|
||||
- publication_year: год
|
||||
- primary_location.source.display_name: журнал
|
||||
- open_access.oa_url: ссылка на PDF
|
||||
- abstract_inverted_index: инвертированный индекс аннотации
|
||||
"""
|
||||
# Нормализовать ext_id (убрать URL-часть)
|
||||
raw_id = raw.get("id", "")
|
||||
ext_id = raw_id.replace("https://openalex.org/", "") if raw_id else ""
|
||||
|
||||
if not ext_id:
|
||||
return {}
|
||||
|
||||
# Авторы
|
||||
authorships = raw.get("authorships", [])
|
||||
authors_raw = []
|
||||
for a in authorships[:10]: # Максимум 10 авторов
|
||||
author = a.get("author", {})
|
||||
display_name = author.get("display_name", "")
|
||||
if display_name:
|
||||
# OpenAlex даёт "Иван Иванов" → разбиваем
|
||||
parts = display_name.strip().split()
|
||||
if len(parts) >= 2:
|
||||
authors_raw.append({
|
||||
"last_name": parts[-1],
|
||||
"first_name": " ".join(parts[:-1]),
|
||||
})
|
||||
elif parts:
|
||||
authors_raw.append({"last_name": parts[0], "first_name": ""})
|
||||
|
||||
authors = self.normalize_authors(authors_raw)
|
||||
|
||||
# Журнал
|
||||
primary_location = raw.get("primary_location") or {}
|
||||
source = primary_location.get("source") or {}
|
||||
journal = source.get("display_name")
|
||||
|
||||
# URL на полный текст
|
||||
oa = raw.get("open_access") or {}
|
||||
url = oa.get("oa_url") or primary_location.get("landing_page_url")
|
||||
|
||||
# Аннотация (восстановить из инвертированного индекса)
|
||||
abstract = None
|
||||
inv_index = raw.get("abstract_inverted_index")
|
||||
if inv_index:
|
||||
abstract = _reconstruct_abstract(inv_index)
|
||||
|
||||
# Бибинформация
|
||||
biblio = raw.get("biblio") or {}
|
||||
|
||||
# DOI
|
||||
doi = raw.get("doi", "")
|
||||
if doi and doi.startswith("https://doi.org/"):
|
||||
doi = doi.replace("https://doi.org/", "")
|
||||
|
||||
# Язык
|
||||
lang = raw.get("language")
|
||||
|
||||
return {
|
||||
"source": self.source_name,
|
||||
"ext_id": ext_id,
|
||||
"doi": doi or None,
|
||||
"title": (raw.get("title") or "").strip() or None,
|
||||
"authors": authors,
|
||||
"year": raw.get("publication_year"),
|
||||
"lang": lang,
|
||||
"journal": journal,
|
||||
"volume": biblio.get("volume"),
|
||||
"issue": biblio.get("issue"),
|
||||
"pages": f"{biblio.get('first_page', '')}-{biblio.get('last_page', '')}".strip("-") or None,
|
||||
"abstract": abstract,
|
||||
"url": url,
|
||||
"full_text": None, # Полный текст скачивается отдельно
|
||||
}
|
||||
|
||||
|
||||
def _reconstruct_abstract(inverted_index: dict) -> str:
|
||||
"""
|
||||
Восстановить аннотацию из инвертированного индекса OpenAlex.
|
||||
|
||||
Инвертированный индекс: {слово: [позиция1, позиция2, ...], ...}
|
||||
"""
|
||||
try:
|
||||
positions: dict[int, str] = {}
|
||||
for word, pos_list in inverted_index.items():
|
||||
for pos in pos_list:
|
||||
positions[pos] = word
|
||||
return " ".join(positions[k] for k in sorted(positions.keys()))
|
||||
except Exception:
|
||||
return ""
|
||||
152
scripts/run_parser.py
Normal file
152
scripts/run_parser.py
Normal file
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CLI для запуска парсеров источников.
|
||||
|
||||
Использование:
|
||||
python run_parser.py openalex --limit 10000 --query "машинное обучение"
|
||||
python run_parser.py cyberleninka --limit 1000 --query "нейронные сети"
|
||||
python run_parser.py arxiv --limit 5000 --categories cs.AI cs.LG
|
||||
python run_parser.py all --limit 1000
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Добавить директорию парсеров в путь
|
||||
sys.path.insert(0, str(Path(__file__).parent / "parsers"))
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_openalex(args: argparse.Namespace, output_dir: Path) -> None:
|
||||
from openalex import OpenAlexParser
|
||||
|
||||
parser = OpenAlexParser()
|
||||
parser.run(
|
||||
output_dir=output_dir,
|
||||
query=args.query,
|
||||
limit=args.limit,
|
||||
lang=args.lang,
|
||||
year_from=args.year_from,
|
||||
year_to=args.year_to,
|
||||
)
|
||||
|
||||
|
||||
def run_cyberleninka(args: argparse.Namespace, output_dir: Path) -> None:
|
||||
from cyberleninka import CyberLeninkaParser
|
||||
|
||||
parser = CyberLeninkaParser()
|
||||
parser.run(
|
||||
output_dir=output_dir,
|
||||
query=args.query,
|
||||
limit=args.limit,
|
||||
)
|
||||
|
||||
|
||||
def run_arxiv(args: argparse.Namespace, output_dir: Path) -> None:
|
||||
from arxiv import ArxivParser
|
||||
|
||||
parser = ArxivParser()
|
||||
parser.run(
|
||||
output_dir=output_dir,
|
||||
query=args.query,
|
||||
limit=args.limit,
|
||||
categories=args.categories,
|
||||
year_from=args.year_from,
|
||||
)
|
||||
|
||||
|
||||
def run_all(args: argparse.Namespace, output_dir: Path) -> None:
|
||||
"""Запустить все парсеры последовательно."""
|
||||
logger.info("Запуск всех парсеров...")
|
||||
run_openalex(args, output_dir)
|
||||
run_cyberleninka(args, output_dir)
|
||||
run_arxiv(args, output_dir)
|
||||
|
||||
|
||||
PARSERS = {
|
||||
"openalex": run_openalex,
|
||||
"cyberleninka": run_cyberleninka,
|
||||
"arxiv": run_arxiv,
|
||||
"all": run_all,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Запуск парсеров академических источников",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"source",
|
||||
choices=list(PARSERS.keys()),
|
||||
help="Источник для парсинга",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query", "-q",
|
||||
default="",
|
||||
help="Поисковый запрос",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", "-n",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Максимальное количество документов (default: 1000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", "-o",
|
||||
default="/data/processed",
|
||||
help="Директория для сохранения JSONL (default: /data/processed)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lang",
|
||||
choices=["ru", "en"],
|
||||
default=None,
|
||||
help="Язык документов",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--year-from",
|
||||
type=int,
|
||||
default=None,
|
||||
dest="year_from",
|
||||
help="Год публикации от",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--year-to",
|
||||
type=int,
|
||||
default=None,
|
||||
dest="year_to",
|
||||
help="Год публикации до",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--categories",
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="arXiv категории (например: cs.AI cs.LG math.ST)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
output_dir = Path(args.output)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"Парсер: {args.source}, лимит: {args.limit}, запрос: {args.query!r}")
|
||||
logger.info(f"Выходная директория: {output_dir}")
|
||||
|
||||
run_fn = PARSERS[args.source]
|
||||
run_fn(args, output_dir)
|
||||
|
||||
logger.info("Парсинг завершён.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
20
services/api/Dockerfile
Normal file
20
services/api/Dockerfile
Normal file
@@ -0,0 +1,20 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Установка системных зависимостей
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
libpq-dev \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# Создать директорию для логов
|
||||
RUN mkdir -p /app/logs
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
46
services/api/alembic.ini
Normal file
46
services/api/alembic.ini
Normal file
@@ -0,0 +1,46 @@
|
||||
# Alembic Configuration File
|
||||
|
||||
[alembic]
|
||||
# Путь к директории с миграциями
|
||||
script_location = alembic
|
||||
|
||||
# Формат имени файла миграции
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# Часовой пояс для временных меток
|
||||
timezone = Europe/Moscow
|
||||
|
||||
# Логирование
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
0
services/api/alembic/__init__.py
Normal file
0
services/api/alembic/__init__.py
Normal file
90
services/api/alembic/env.py
Normal file
90
services/api/alembic/env.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""Alembic env.py — настройка среды для миграций (async режим)."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
# Импорт всех моделей для автоопределения изменений
|
||||
from app.database import Base
|
||||
import app.models # noqa: F401 — регистрирует все модели
|
||||
|
||||
# Alembic Config
|
||||
config = context.config
|
||||
|
||||
# Настройка логирования
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# Целевые метаданные для автогенерации
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# Читаем DATABASE URL из переменной окружения (синхронный psycopg2 для Alembic)
|
||||
def get_url() -> str:
|
||||
host = os.getenv("POSTGRES_HOST", "postgres")
|
||||
port = os.getenv("POSTGRES_PORT", "5432")
|
||||
db = os.getenv("POSTGRES_DB", "antiplagiator")
|
||||
user = os.getenv("POSTGRES_USER", "antiplagiator")
|
||||
password = os.getenv("POSTGRES_PASSWORD", "changeme")
|
||||
return f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{db}"
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Запустить миграции в 'offline' режиме (без реального соединения с БД)."""
|
||||
url = get_url()
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
compare_server_default=True,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
"""Выполнить миграции с реальным соединением."""
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
compare_server_default=True,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
"""Запустить миграции в async режиме."""
|
||||
configuration = config.get_section(config.config_ini_section) or {}
|
||||
configuration["sqlalchemy.url"] = get_url()
|
||||
|
||||
connectable = async_engine_from_config(
|
||||
configuration,
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Запустить миграции в 'online' режиме."""
|
||||
asyncio.run(run_async_migrations())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
24
services/api/alembic/script.py.mako
Normal file
24
services/api/alembic/script.py.mako
Normal file
@@ -0,0 +1,24 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
113
services/api/alembic/versions/001_initial_schema.py
Normal file
113
services/api/alembic/versions/001_initial_schema.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Начальная схема базы данных.
|
||||
|
||||
Revision ID: 001
|
||||
Revises:
|
||||
Create Date: 2024-01-01 00:00:00.000000
|
||||
|
||||
Создаёт таблицы: users, tasks, documents, fingerprints, usage_logs
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers
|
||||
revision = "001"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Создать все таблицы."""
|
||||
|
||||
# ── users ──────────────────────────────────────────────────────────────────
|
||||
op.create_table(
|
||||
"users",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("email", sa.String(255), nullable=False),
|
||||
sa.Column("hashed_password", sa.String(255), nullable=False),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("is_verified", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("plan", sa.String(20), nullable=False, server_default="free"),
|
||||
sa.Column("verification_token", sa.String(255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_users_email", "users", ["email"], unique=True)
|
||||
|
||||
# ── tasks ──────────────────────────────────────────────────────────────────
|
||||
op.create_table(
|
||||
"tasks",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE")),
|
||||
sa.Column("type", sa.String(20), nullable=False),
|
||||
sa.Column("status", sa.String(20), nullable=False, server_default="queued"),
|
||||
sa.Column("celery_task_id", sa.String(255), nullable=True),
|
||||
sa.Column("input_data", sa.JSON(), nullable=False, server_default="{}"),
|
||||
sa.Column("result", sa.JSON(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("queue_position", sa.Integer(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tasks_user_id", "tasks", ["user_id"])
|
||||
op.create_index("ix_tasks_status", "tasks", ["status"])
|
||||
|
||||
# ── documents ──────────────────────────────────────────────────────────────
|
||||
op.create_table(
|
||||
"documents",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("source", sa.String(50), nullable=False),
|
||||
sa.Column("ext_id", sa.String(255), nullable=False),
|
||||
sa.Column("doi", sa.String(255), nullable=True),
|
||||
sa.Column("title", sa.Text(), nullable=False),
|
||||
sa.Column("authors", sa.JSON(), nullable=False, server_default="[]"),
|
||||
sa.Column("year", sa.Integer(), nullable=True),
|
||||
sa.Column("lang", sa.String(10), nullable=True),
|
||||
sa.Column("journal", sa.Text(), nullable=True),
|
||||
sa.Column("volume", sa.String(50), nullable=True),
|
||||
sa.Column("issue", sa.String(50), nullable=True),
|
||||
sa.Column("pages", sa.String(50), nullable=True),
|
||||
sa.Column("abstract", sa.Text(), nullable=True),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column("minio_key", sa.Text(), nullable=True),
|
||||
sa.Column("faiss_id", sa.Integer(), nullable=True),
|
||||
sa.Column("indexed_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_documents_ext_id", "documents", ["ext_id"], unique=True)
|
||||
op.create_index("ix_documents_doi", "documents", ["doi"])
|
||||
op.create_index("ix_documents_source", "documents", ["source"])
|
||||
op.create_index("ix_documents_year", "documents", ["year"])
|
||||
op.create_index("ix_documents_lang", "documents", ["lang"])
|
||||
op.create_index("ix_documents_faiss_id", "documents", ["faiss_id"])
|
||||
|
||||
# ── fingerprints ───────────────────────────────────────────────────────────
|
||||
op.create_table(
|
||||
"fingerprints",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("doc_id", sa.Integer(), sa.ForeignKey("documents.id", ondelete="CASCADE")),
|
||||
sa.Column("hash_value", sa.BigInteger(), nullable=False),
|
||||
sa.Column("position", sa.Integer(), nullable=False),
|
||||
)
|
||||
op.create_index("ix_fingerprints_doc_id", "fingerprints", ["doc_id"])
|
||||
op.create_index("ix_fingerprints_hash_value", "fingerprints", ["hash_value"])
|
||||
|
||||
# ── usage_logs ─────────────────────────────────────────────────────────────
|
||||
op.create_table(
|
||||
"usage_logs",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE")),
|
||||
sa.Column("action", sa.String(50), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_usage_logs_user_id", "usage_logs", ["user_id"])
|
||||
op.create_index("ix_usage_logs_created_at", "usage_logs", ["created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Удалить все таблицы."""
|
||||
op.drop_table("usage_logs")
|
||||
op.drop_table("fingerprints")
|
||||
op.drop_table("documents")
|
||||
op.drop_table("tasks")
|
||||
op.drop_table("users")
|
||||
0
services/api/alembic/versions/__init__.py
Normal file
0
services/api/alembic/versions/__init__.py
Normal file
0
services/api/app/__init__.py
Normal file
0
services/api/app/__init__.py
Normal file
1
services/api/app/api/__init__.py
Normal file
1
services/api/app/api/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# API роутеры
|
||||
119
services/api/app/api/auth.py
Normal file
119
services/api/app/api/auth.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Роутер аутентификации: регистрация, вход, верификация email."""
|
||||
|
||||
import secrets
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.celery_app import celery_app
|
||||
from app.core.security import (
|
||||
create_access_token,
|
||||
get_current_user,
|
||||
hash_password,
|
||||
verify_password,
|
||||
)
|
||||
from app.database import get_db
|
||||
from app.models.user import User
|
||||
from app.schemas.auth import LoginRequest, RegisterRequest, TokenResponse, UserInToken
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
|
||||
@router.post("/register", response_model=TokenResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def register(data: RegisterRequest, db: AsyncSession = Depends(get_db)) -> TokenResponse:
|
||||
"""
|
||||
Регистрация нового пользователя.
|
||||
|
||||
Создаёт аккаунт и отправляет письмо с подтверждением email.
|
||||
"""
|
||||
# Проверить уникальность email
|
||||
existing = await db.execute(select(User).where(User.email == data.email.lower()))
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Пользователь с таким email уже существует",
|
||||
)
|
||||
|
||||
# Создать токен верификации
|
||||
verification_token = secrets.token_urlsafe(32)
|
||||
|
||||
user = User(
|
||||
email=data.email.lower(),
|
||||
hashed_password=hash_password(data.password),
|
||||
name=data.name,
|
||||
verification_token=verification_token,
|
||||
is_verified=False,
|
||||
plan="free",
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush() # Получить ID без коммита
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
|
||||
# Диспатч email верификации через воркер
|
||||
try:
|
||||
celery_app.send_task(
|
||||
"notify.send_verification",
|
||||
args=[user.email, user.name, verification_token],
|
||||
queue="queue.notify",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Не удалось поставить задачу верификации email: {e}")
|
||||
|
||||
access_token = create_access_token({"sub": str(user.id)})
|
||||
return TokenResponse(
|
||||
access_token=access_token,
|
||||
token_type="bearer",
|
||||
user=UserInToken.model_validate(user),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
async def login(data: LoginRequest, db: AsyncSession = Depends(get_db)) -> TokenResponse:
|
||||
"""Вход в систему. Возвращает JWT токен."""
|
||||
result = await db.execute(select(User).where(User.email == data.email.lower()))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if user is None or not verify_password(data.password, user.hashed_password):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Неверный email или пароль",
|
||||
)
|
||||
|
||||
access_token = create_access_token({"sub": str(user.id)})
|
||||
return TokenResponse(
|
||||
access_token=access_token,
|
||||
token_type="bearer",
|
||||
user=UserInToken.model_validate(user),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserInToken)
|
||||
async def get_me(current_user: User = Depends(get_current_user)) -> UserInToken:
|
||||
"""Получить информацию о текущем пользователе."""
|
||||
return UserInToken.model_validate(current_user)
|
||||
|
||||
|
||||
@router.post("/verify-email/{token}", status_code=status.HTTP_200_OK)
|
||||
async def verify_email(token: str, db: AsyncSession = Depends(get_db)) -> dict:
|
||||
"""Подтвердить email по токену из письма."""
|
||||
result = await db.execute(
|
||||
select(User).where(User.verification_token == token)
|
||||
)
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if user is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Неверный или просроченный токен верификации",
|
||||
)
|
||||
|
||||
user.is_verified = True
|
||||
user.verification_token = None
|
||||
await db.commit()
|
||||
|
||||
return {"message": "Email успешно подтверждён"}
|
||||
155
services/api/app/api/documents.py
Normal file
155
services/api/app/api/documents.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""Роутер загрузки документов на проверку плагиата."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.core.celery_app import celery_app
|
||||
from app.core.rate_limiter import check_and_increment_limit, check_concurrent_limit
|
||||
from app.core.security import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.task import Task
|
||||
from app.models.user import User
|
||||
from app.schemas.tasks import TaskResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/documents", tags=["documents"])
|
||||
|
||||
# Допустимые форматы
|
||||
ALLOWED_EXTENSIONS = {".pdf", ".docx", ".txt"}
|
||||
MAX_FILE_SIZE_MB = 100
|
||||
MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
|
||||
|
||||
|
||||
def get_minio_client() -> Minio:
|
||||
"""Создать MinIO клиент."""
|
||||
return Minio(
|
||||
settings.MINIO_ENDPOINT,
|
||||
access_key=settings.MINIO_ACCESS_KEY,
|
||||
secret_key=settings.MINIO_SECRET_KEY,
|
||||
secure=False,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/check", response_model=TaskResponse, status_code=status.HTTP_202_ACCEPTED)
|
||||
async def upload_for_plagiarism_check(
|
||||
file: UploadFile = File(..., description="PDF, DOCX или TXT файл до 100 МБ"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> TaskResponse:
|
||||
"""
|
||||
Загрузить документ для проверки на плагиат.
|
||||
|
||||
Файл сохраняется в MinIO, затем диспатчится задача index.extract_and_check.
|
||||
Результат доступен через GET /tasks/{task_id}.
|
||||
"""
|
||||
# Проверить расширение файла
|
||||
if not file.filename:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Имя файла не указано",
|
||||
)
|
||||
|
||||
ext = Path(file.filename).suffix.lower()
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail=f"Неподдерживаемый формат файла. Допустимые форматы: {', '.join(ALLOWED_EXTENSIONS)}",
|
||||
)
|
||||
|
||||
# Проверить лимит по тарифу
|
||||
limit_check = check_and_increment_limit(current_user.id, "plagiarism", current_user.plan)
|
||||
if not limit_check["allowed"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=(
|
||||
f"Превышен месячный лимит проверок плагиата для тарифа '{current_user.plan}'. "
|
||||
f"Использовано {limit_check['current']} из {limit_check['limit']}."
|
||||
),
|
||||
)
|
||||
|
||||
# Проверить лимит одновременных задач
|
||||
if not check_concurrent_limit(current_user.id, current_user.plan):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail="Превышен лимит одновременных задач.",
|
||||
)
|
||||
|
||||
# Прочитать файл
|
||||
file_data = await file.read()
|
||||
|
||||
if len(file_data) > MAX_FILE_SIZE_BYTES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail=f"Файл слишком большой. Максимальный размер: {MAX_FILE_SIZE_MB} МБ",
|
||||
)
|
||||
|
||||
# Загрузить в MinIO
|
||||
minio_key = f"uploads/{current_user.id}/{uuid.uuid4()}{ext}"
|
||||
minio_client = get_minio_client()
|
||||
|
||||
try:
|
||||
# Создать бакет если не существует
|
||||
if not minio_client.bucket_exists(settings.MINIO_BUCKET_DOCS):
|
||||
minio_client.make_bucket(settings.MINIO_BUCKET_DOCS)
|
||||
|
||||
import io
|
||||
minio_client.put_object(
|
||||
bucket_name=settings.MINIO_BUCKET_DOCS,
|
||||
object_name=minio_key,
|
||||
data=io.BytesIO(file_data),
|
||||
length=len(file_data),
|
||||
content_type=file.content_type or "application/octet-stream",
|
||||
)
|
||||
logger.info(f"Файл загружен в MinIO: {minio_key}")
|
||||
|
||||
except S3Error as e:
|
||||
logger.error(f"Ошибка загрузки в MinIO: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Ошибка сохранения файла. Попробуйте позже.",
|
||||
)
|
||||
|
||||
# Создать задачу в БД
|
||||
task = Task(
|
||||
user_id=current_user.id,
|
||||
type="plagiarism",
|
||||
status="queued",
|
||||
input_data={
|
||||
"filename": file.filename,
|
||||
"minio_key": minio_key,
|
||||
"file_size_bytes": len(file_data),
|
||||
"content_type": file.content_type,
|
||||
},
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
task_id = task.id
|
||||
|
||||
# Диспатч в индексер воркер
|
||||
celery_result = celery_app.send_task(
|
||||
"index.extract_and_check",
|
||||
args=[task_id, minio_key, file.filename],
|
||||
queue="queue.index",
|
||||
)
|
||||
|
||||
task.celery_task_id = celery_result.id
|
||||
task.queue_position = 1
|
||||
task.eta_seconds = 120 # ~2 минуты с учётом 4 уровней проверки
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
logger.info(
|
||||
f"Задача проверки плагиата {task_id!r} создана для пользователя {current_user.id}, "
|
||||
f"файл: {file.filename!r}"
|
||||
)
|
||||
|
||||
return TaskResponse.model_validate(task)
|
||||
77
services/api/app/api/reports.py
Normal file
77
services/api/app/api/reports.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Роутер для получения отчётов о выполненных задачах."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.task import Task
|
||||
from app.models.user import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/reports", tags=["reports"])
|
||||
|
||||
|
||||
@router.get("/{task_id}")
|
||||
async def get_report(
|
||||
task_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""
|
||||
Получить готовый отчёт по задаче.
|
||||
|
||||
Возвращает task.result в зависимости от типа задачи:
|
||||
- search: список источников с ГОСТ-цитатами
|
||||
- plagiarism: детальный отчёт с совпадениями
|
||||
- gost: отформатированная библиография
|
||||
- summarize: краткое изложение
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(Task).where(Task.id == task_id, Task.user_id == current_user.id)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if task is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Задача не найдена",
|
||||
)
|
||||
|
||||
if task.status == "queued":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
detail="Задача ещё в очереди",
|
||||
)
|
||||
|
||||
if task.status == "processing":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
detail="Задача ещё выполняется",
|
||||
)
|
||||
|
||||
if task.status == "failed":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Задача завершилась с ошибкой: {task.error}",
|
||||
)
|
||||
|
||||
if task.result is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Результат недоступен",
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": task.id,
|
||||
"type": task.type,
|
||||
"status": task.status,
|
||||
"created_at": task.created_at.isoformat() if task.created_at else None,
|
||||
"updated_at": task.updated_at.isoformat() if task.updated_at else None,
|
||||
"result": task.result,
|
||||
}
|
||||
95
services/api/app/api/search.py
Normal file
95
services/api/app/api/search.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Роутер семантического поиска источников."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.celery_app import celery_app
|
||||
from app.core.rate_limiter import check_and_increment_limit, check_concurrent_limit
|
||||
from app.core.security import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.task import Task
|
||||
from app.models.user import User
|
||||
from app.schemas.tasks import SearchRequest, TaskResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/search", tags=["search"])
|
||||
|
||||
# Примерное время ожидания в секундах для каждой позиции в очереди
|
||||
ETA_PER_POSITION_SECONDS = 15
|
||||
|
||||
|
||||
@router.post("/", response_model=TaskResponse, status_code=status.HTTP_202_ACCEPTED)
|
||||
async def create_search_task(
|
||||
data: SearchRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> TaskResponse:
|
||||
"""
|
||||
Создать задачу семантического поиска источников.
|
||||
|
||||
Возвращает TaskResponse с queue_position и eta_seconds сразу.
|
||||
Результат доступен через GET /tasks/{task_id} или WebSocket /ws/tasks/{task_id}.
|
||||
"""
|
||||
# Проверить лимит по тарифу
|
||||
limit_check = check_and_increment_limit(current_user.id, "search", current_user.plan)
|
||||
if not limit_check["allowed"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=(
|
||||
f"Превышен дневной лимит поиска для тарифа '{current_user.plan}'. "
|
||||
f"Использовано {limit_check['current']} из {limit_check['limit']}."
|
||||
),
|
||||
)
|
||||
|
||||
# Проверить лимит одновременных задач
|
||||
if not check_concurrent_limit(current_user.id, current_user.plan):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail="Превышен лимит одновременных задач. Дождитесь завершения текущих.",
|
||||
)
|
||||
|
||||
# Создать задачу в БД
|
||||
task = Task(
|
||||
user_id=current_user.id,
|
||||
type="search",
|
||||
status="queued",
|
||||
input_data={
|
||||
"query": data.query,
|
||||
"lang": data.lang,
|
||||
"year_from": data.year_from,
|
||||
"year_to": data.year_to,
|
||||
"category": data.category,
|
||||
},
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
task_id = task.id
|
||||
|
||||
# Диспатч в GPU воркер
|
||||
celery_result = celery_app.send_task(
|
||||
"gpu.search_semantic",
|
||||
args=[task_id, data.query],
|
||||
kwargs={
|
||||
"lang": data.lang,
|
||||
"year_from": data.year_from,
|
||||
"year_to": data.year_to,
|
||||
},
|
||||
queue="queue.gpu",
|
||||
)
|
||||
|
||||
task.celery_task_id = celery_result.id
|
||||
task.queue_position = 1 # TODO: реальный подсчёт через Redis
|
||||
task.eta_seconds = ETA_PER_POSITION_SECONDS
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
logger.info(
|
||||
f"Задача поиска {task_id!r} создана для пользователя {current_user.id}, "
|
||||
f"запрос: {data.query[:50]!r}"
|
||||
)
|
||||
|
||||
return TaskResponse.model_validate(task)
|
||||
89
services/api/app/api/tasks.py
Normal file
89
services/api/app/api/tasks.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""Роутер для управления задачами пользователя."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.task import Task
|
||||
from app.models.user import User
|
||||
from app.schemas.tasks import TaskResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/tasks", tags=["tasks"])
|
||||
|
||||
|
||||
@router.get("/", response_model=list[TaskResponse])
|
||||
async def list_tasks(
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[TaskResponse]:
|
||||
"""Получить список задач текущего пользователя (от новых к старым)."""
|
||||
result = await db.execute(
|
||||
select(Task)
|
||||
.where(Task.user_id == current_user.id)
|
||||
.order_by(Task.created_at.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
tasks = result.scalars().all()
|
||||
return [TaskResponse.model_validate(t) for t in tasks]
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=TaskResponse)
|
||||
async def get_task(
|
||||
task_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> TaskResponse:
|
||||
"""Получить детали конкретной задачи."""
|
||||
result = await db.execute(
|
||||
select(Task).where(Task.id == task_id, Task.user_id == current_user.id)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if task is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Задача не найдена",
|
||||
)
|
||||
|
||||
return TaskResponse.model_validate(task)
|
||||
|
||||
|
||||
@router.delete("/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_task(
|
||||
task_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> None:
|
||||
"""
|
||||
Удалить задачу.
|
||||
|
||||
Нельзя удалить задачу в статусе 'processing'.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(Task).where(Task.id == task_id, Task.user_id == current_user.id)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if task is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Задача не найдена",
|
||||
)
|
||||
|
||||
if task.status == "processing":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Нельзя удалить задачу в процессе выполнения",
|
||||
)
|
||||
|
||||
await db.delete(task)
|
||||
await db.commit()
|
||||
82
services/api/app/config.py
Normal file
82
services/api/app/config.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Конфигурация приложения через Pydantic Settings."""
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Настройки приложения, читаемые из переменных окружения."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
# PostgreSQL
|
||||
POSTGRES_HOST: str = "postgres"
|
||||
POSTGRES_PORT: int = 5432
|
||||
POSTGRES_DB: str = "antiplagiator"
|
||||
POSTGRES_USER: str = "antiplagiator"
|
||||
POSTGRES_PASSWORD: str = "changeme"
|
||||
|
||||
# Redis
|
||||
REDIS_URL: str = "redis://redis:6379/0"
|
||||
|
||||
# RabbitMQ
|
||||
RABBITMQ_URL: str = "amqp://guest:guest@rabbitmq:5672/"
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: str = "minio:9000"
|
||||
MINIO_ACCESS_KEY: str = "minioadmin"
|
||||
MINIO_SECRET_KEY: str = "changeme"
|
||||
MINIO_BUCKET_DOCS: str = "documents"
|
||||
MINIO_BUCKET_BACKUPS: str = "backups"
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_URL: str = "http://elasticsearch:9200"
|
||||
|
||||
# Ollama
|
||||
OLLAMA_URL: str = "http://ollama:11434"
|
||||
|
||||
# JWT
|
||||
SECRET_KEY: str = "change-me-in-production-use-openssl-rand-hex-32"
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 10080 # 7 дней
|
||||
|
||||
# SMTP
|
||||
SMTP_HOST: str = "smtp.yandex.ru"
|
||||
SMTP_PORT: int = 465
|
||||
SMTP_USER: str = "noreply@jze9.ru"
|
||||
SMTP_PASSWORD: str = "changeme"
|
||||
SMTP_FROM: str = "noreply@jze9.ru"
|
||||
|
||||
# App
|
||||
APP_URL: str = "https://academic.jze9.ru"
|
||||
ENVIRONMENT: str = "development"
|
||||
DEBUG: bool = False
|
||||
|
||||
# CORS
|
||||
CORS_ORIGINS: list[str] = [
|
||||
"http://localhost:5173",
|
||||
"http://localhost:3000",
|
||||
"https://academic.jze9.ru",
|
||||
]
|
||||
|
||||
@property
|
||||
def database_url(self) -> str:
|
||||
"""URL для asyncpg (асинхронные запросы)."""
|
||||
return (
|
||||
f"postgresql+asyncpg://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}"
|
||||
f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
@property
|
||||
def database_url_sync(self) -> str:
|
||||
"""URL для psycopg2 (Alembic и синхронные операции)."""
|
||||
return (
|
||||
f"postgresql+psycopg2://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}"
|
||||
f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
0
services/api/app/core/__init__.py
Normal file
0
services/api/app/core/__init__.py
Normal file
39
services/api/app/core/celery_app.py
Normal file
39
services/api/app/core/celery_app.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Celery приложение для API-диспатчера. Только определение — задачи находятся в воркерах."""
|
||||
|
||||
from celery import Celery
|
||||
|
||||
from app.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"api_dispatcher",
|
||||
broker=settings.RABBITMQ_URL,
|
||||
backend=settings.REDIS_URL,
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="Europe/Moscow",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
# Маршрутизация задач по очередям воркеров
|
||||
task_routes={
|
||||
"gpu.*": {"queue": "queue.gpu"},
|
||||
"index.*": {"queue": "queue.index"},
|
||||
"notify.*": {"queue": "queue.notify"},
|
||||
"gost.*": {"queue": "queue.gost"},
|
||||
},
|
||||
# Настройки очередей
|
||||
task_queues={
|
||||
"queue.gpu": {"exchange": "queue.gpu", "routing_key": "queue.gpu"},
|
||||
"queue.index": {"exchange": "queue.index", "routing_key": "queue.index"},
|
||||
"queue.notify": {"exchange": "queue.notify", "routing_key": "queue.notify"},
|
||||
"queue.gost": {"exchange": "queue.gost", "routing_key": "queue.gost"},
|
||||
},
|
||||
# Результаты хранить 24 часа
|
||||
result_expires=86400,
|
||||
# Повторные попытки
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
)
|
||||
167
services/api/app/core/rate_limiter.py
Normal file
167
services/api/app/core/rate_limiter.py
Normal file
@@ -0,0 +1,167 @@
|
||||
"""Redis-based rate limiter для проверки лимитов по тарифному плану."""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import redis
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# Лимиты по тарифным планам
|
||||
PLAN_LIMITS: dict[str, dict[str, int | None]] = {
|
||||
"free": {
|
||||
"search_per_day": 10,
|
||||
"summarize_per_month": 3,
|
||||
"plagiarism_per_month": 1,
|
||||
"concurrent": 1,
|
||||
},
|
||||
"student": {
|
||||
"search_per_day": None, # None = безлимит
|
||||
"summarize_per_month": 30,
|
||||
"plagiarism_per_month": 10,
|
||||
"concurrent": 2,
|
||||
},
|
||||
"premium": {
|
||||
"search_per_day": None,
|
||||
"summarize_per_month": None,
|
||||
"plagiarism_per_month": 50,
|
||||
"concurrent": 5,
|
||||
},
|
||||
"science": {
|
||||
"search_per_day": None,
|
||||
"summarize_per_month": None,
|
||||
"plagiarism_per_month": None,
|
||||
"concurrent": 10,
|
||||
},
|
||||
}
|
||||
|
||||
# Маппинг действий на ключи лимитов
|
||||
ACTION_TO_LIMIT: dict[str, tuple[str, str]] = {
|
||||
"search": ("search_per_day", "day"),
|
||||
"summarize": ("summarize_per_month", "month"),
|
||||
"plagiarism": ("plagiarism_per_month", "month"),
|
||||
"gost": ("gost_per_month", "month"), # ГОСТ всегда разрешён
|
||||
}
|
||||
|
||||
|
||||
def get_redis_client() -> redis.Redis:
|
||||
"""Создать синхронный Redis клиент."""
|
||||
return redis.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def _get_period_key(period: str) -> str:
|
||||
"""Получить строку периода для Redis ключа."""
|
||||
now = datetime.now(timezone.utc)
|
||||
if period == "day":
|
||||
return now.strftime("%Y-%m-%d")
|
||||
elif period == "month":
|
||||
return now.strftime("%Y-%m")
|
||||
return now.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def check_and_increment_limit(user_id: int, action: str, plan: str) -> dict:
|
||||
"""
|
||||
Проверить лимит и инкрементировать счётчик.
|
||||
|
||||
Args:
|
||||
user_id: ID пользователя
|
||||
action: Действие (search, plagiarism, summarize, gost)
|
||||
plan: Тарифный план пользователя
|
||||
|
||||
Returns:
|
||||
dict с полями:
|
||||
- allowed: bool — разрешено ли действие
|
||||
- current: int — текущее количество использований
|
||||
- limit: int | None — лимит (None = безлимит)
|
||||
- remaining: int | None — осталось использований
|
||||
- reset_at: str — когда сбрасывается счётчик
|
||||
"""
|
||||
limits = PLAN_LIMITS.get(plan, PLAN_LIMITS["free"])
|
||||
|
||||
if action not in ACTION_TO_LIMIT:
|
||||
# Неизвестное действие — разрешаем
|
||||
return {"allowed": True, "current": 0, "limit": None, "remaining": None}
|
||||
|
||||
limit_key, period = ACTION_TO_LIMIT[action]
|
||||
limit_value = limits.get(limit_key)
|
||||
|
||||
# Безлимитный план
|
||||
if limit_value is None:
|
||||
return {
|
||||
"allowed": True,
|
||||
"current": 0,
|
||||
"limit": None,
|
||||
"remaining": None,
|
||||
"reset_at": None,
|
||||
}
|
||||
|
||||
period_str = _get_period_key(period)
|
||||
redis_key = f"rate:{user_id}:{action}:{period_str}"
|
||||
|
||||
r = get_redis_client()
|
||||
|
||||
# Атомарно инкрементировать
|
||||
pipe = r.pipeline()
|
||||
pipe.incr(redis_key)
|
||||
# Устанавливаем TTL: для дня — 86400 сек, для месяца — 32 дня
|
||||
ttl = 86400 if period == "day" else 86400 * 32
|
||||
pipe.expire(redis_key, ttl)
|
||||
results = pipe.execute()
|
||||
|
||||
current = results[0]
|
||||
|
||||
if current > limit_value:
|
||||
# Декрементировать обратно (не считать запрещённые)
|
||||
r.decr(redis_key)
|
||||
current -= 1
|
||||
return {
|
||||
"allowed": False,
|
||||
"current": current,
|
||||
"limit": limit_value,
|
||||
"remaining": 0,
|
||||
"reset_at": period_str,
|
||||
}
|
||||
|
||||
return {
|
||||
"allowed": True,
|
||||
"current": current,
|
||||
"limit": limit_value,
|
||||
"remaining": limit_value - current,
|
||||
"reset_at": period_str,
|
||||
}
|
||||
|
||||
|
||||
def check_concurrent_limit(user_id: int, plan: str) -> bool:
|
||||
"""
|
||||
Проверить лимит одновременных задач.
|
||||
|
||||
Returns:
|
||||
True если можно создать новую задачу, False если превышен лимит.
|
||||
"""
|
||||
limits = PLAN_LIMITS.get(plan, PLAN_LIMITS["free"])
|
||||
max_concurrent = limits.get("concurrent", 1)
|
||||
|
||||
r = get_redis_client()
|
||||
key = f"concurrent:{user_id}"
|
||||
current = r.get(key)
|
||||
|
||||
return (current is None) or (int(current) < max_concurrent)
|
||||
|
||||
|
||||
def increment_concurrent(user_id: int) -> None:
|
||||
"""Увеличить счётчик одновременных задач (при создании задачи)."""
|
||||
r = get_redis_client()
|
||||
key = f"concurrent:{user_id}"
|
||||
pipe = r.pipeline()
|
||||
pipe.incr(key)
|
||||
pipe.expire(key, 3600) # Автосброс через 1 час
|
||||
pipe.execute()
|
||||
|
||||
|
||||
def decrement_concurrent(user_id: int) -> None:
|
||||
"""Уменьшить счётчик одновременных задач (при завершении задачи)."""
|
||||
r = get_redis_client()
|
||||
key = f"concurrent:{user_id}"
|
||||
current = r.get(key)
|
||||
if current and int(current) > 0:
|
||||
r.decr(key)
|
||||
110
services/api/app/core/security.py
Normal file
110
services/api/app/core/security.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Утилиты безопасности: JWT, хэширование паролей, dependency для получения текущего пользователя."""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
|
||||
# Контекст хэширования паролей (bcrypt)
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
# OAuth2 схема
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Хэшировать пароль через bcrypt."""
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
"""Проверить соответствие пароля его хэшу."""
|
||||
return pwd_context.verify(plain_password, hashed_password)
|
||||
|
||||
|
||||
def create_access_token(data: dict[str, Any], expires_delta: timedelta | None = None) -> str:
|
||||
"""Создать JWT токен доступа."""
|
||||
to_encode = data.copy()
|
||||
expire = datetime.now(timezone.utc) + (
|
||||
expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
)
|
||||
to_encode.update({"exp": expire})
|
||||
return jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
|
||||
|
||||
|
||||
def verify_token(token: str) -> dict[str, Any]:
|
||||
"""
|
||||
Декодировать и верифицировать JWT токен.
|
||||
|
||||
Raises:
|
||||
HTTPException: если токен невалиден или просрочен.
|
||||
"""
|
||||
try:
|
||||
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
|
||||
user_id: int | None = payload.get("sub")
|
||||
if user_id is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Неверный токен аутентификации",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
return payload
|
||||
except JWTError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Невалидный или просроченный токен",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
FastAPI dependency: получить текущего авторизованного пользователя из JWT токена.
|
||||
|
||||
Raises:
|
||||
HTTPException 401: если токен невалиден.
|
||||
HTTPException 404: если пользователь не найден.
|
||||
"""
|
||||
from app.models.user import User
|
||||
|
||||
payload = verify_token(token)
|
||||
user_id: int = int(payload.get("sub"))
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if user is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Пользователь не найден",
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
async def get_current_verified_user(
|
||||
current_user=Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
FastAPI dependency: только верифицированные пользователи.
|
||||
|
||||
Raises:
|
||||
HTTPException 403: если email не подтверждён.
|
||||
"""
|
||||
if not current_user.is_verified:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Необходимо подтвердить email адрес",
|
||||
)
|
||||
return current_user
|
||||
86
services/api/app/core/websocket_manager.py
Normal file
86
services/api/app/core/websocket_manager.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""WebSocket менеджер для real-time обновлений статуса задач."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import WebSocket
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""Менеджер WebSocket соединений с поддержкой подписки на задачи."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# task_id -> список активных соединений
|
||||
self._connections: dict[str, list[WebSocket]] = {}
|
||||
|
||||
async def connect(self, task_id: str, websocket: WebSocket) -> None:
|
||||
"""Принять WebSocket соединение и зарегистрировать его для задачи."""
|
||||
await websocket.accept()
|
||||
if task_id not in self._connections:
|
||||
self._connections[task_id] = []
|
||||
self._connections[task_id].append(websocket)
|
||||
logger.info(f"WebSocket подключён к задаче {task_id!r}")
|
||||
|
||||
def disconnect(self, task_id: str, websocket: WebSocket) -> None:
|
||||
"""Удалить соединение из реестра."""
|
||||
if task_id in self._connections:
|
||||
try:
|
||||
self._connections[task_id].remove(websocket)
|
||||
except ValueError:
|
||||
pass
|
||||
if not self._connections[task_id]:
|
||||
del self._connections[task_id]
|
||||
logger.info(f"WebSocket отключён от задачи {task_id!r}")
|
||||
|
||||
async def send_task_update(self, task_id: str, data: dict[str, Any]) -> None:
|
||||
"""
|
||||
Отправить обновление статуса задачи всем подключённым клиентам.
|
||||
|
||||
Args:
|
||||
task_id: ID задачи
|
||||
data: Словарь с обновлением (status, queue_position, result и т.д.)
|
||||
"""
|
||||
connections = self._connections.get(task_id, [])
|
||||
if not connections:
|
||||
return
|
||||
|
||||
message = json.dumps(data, ensure_ascii=False, default=str)
|
||||
dead_connections = []
|
||||
|
||||
for websocket in connections:
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
except Exception as e:
|
||||
logger.warning(f"Ошибка отправки WebSocket сообщения: {e}")
|
||||
dead_connections.append(websocket)
|
||||
|
||||
# Очистить мёртвые соединения
|
||||
for ws in dead_connections:
|
||||
self.disconnect(task_id, ws)
|
||||
|
||||
async def broadcast(self, data: dict[str, Any]) -> None:
|
||||
"""Отправить сообщение всем подключённым клиентам."""
|
||||
message = json.dumps(data, ensure_ascii=False, default=str)
|
||||
all_dead = []
|
||||
|
||||
for task_id, connections in self._connections.items():
|
||||
for websocket in connections:
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
except Exception:
|
||||
all_dead.append((task_id, websocket))
|
||||
|
||||
for task_id, ws in all_dead:
|
||||
self.disconnect(task_id, ws)
|
||||
|
||||
@property
|
||||
def active_connections_count(self) -> int:
|
||||
"""Количество активных WebSocket соединений."""
|
||||
return sum(len(conns) for conns in self._connections.values())
|
||||
|
||||
|
||||
# Глобальный экземпляр менеджера
|
||||
ws_manager = ConnectionManager()
|
||||
53
services/api/app/database.py
Normal file
53
services/api/app/database.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""Настройка базы данных: AsyncEngine, AsyncSession, dependency для FastAPI."""
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""Базовый класс для всех SQLAlchemy моделей."""
|
||||
pass
|
||||
|
||||
|
||||
# Асинхронный движок
|
||||
engine = create_async_engine(
|
||||
settings.database_url,
|
||||
echo=settings.DEBUG,
|
||||
pool_size=10,
|
||||
max_overflow=20,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
)
|
||||
|
||||
# Фабрика сессий
|
||||
AsyncSessionLocal = async_sessionmaker(
|
||||
bind=engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autocommit=False,
|
||||
autoflush=False,
|
||||
)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""
|
||||
FastAPI dependency для получения сессии БД.
|
||||
Сессия автоматически закрывается после запроса.
|
||||
"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
113
services/api/app/main.py
Normal file
113
services/api/app/main.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Точка входа FastAPI приложения — Академический помощник (anti-plagiarism)."""
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api import auth, documents, reports, search, tasks
|
||||
from app.config import settings
|
||||
from app.core.websocket_manager import ws_manager
|
||||
from app.database import engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""
|
||||
Lifecycle менеджер приложения.
|
||||
Выполняет инициализацию при старте и очистку при остановке.
|
||||
"""
|
||||
logger.info("Запуск Академического помощника...")
|
||||
|
||||
# Проверка соединений при старте
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(__import__("sqlalchemy").text("SELECT 1"))
|
||||
logger.info("PostgreSQL: соединение установлено")
|
||||
except Exception as e:
|
||||
logger.error(f"PostgreSQL: ошибка соединения: {e}")
|
||||
|
||||
yield
|
||||
|
||||
# Очистка при остановке
|
||||
logger.info("Остановка приложения...")
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Академический помощник API",
|
||||
description=(
|
||||
"Сервис антиплагиата и поиска академических источников. "
|
||||
"Семантический поиск через FAISS GPU + Elasticsearch, "
|
||||
"проверка плагиата в 4 уровня, ГОСТ библиография."
|
||||
),
|
||||
version="0.1.0",
|
||||
lifespan=lifespan,
|
||||
docs_url="/api/docs",
|
||||
redoc_url="/api/redoc",
|
||||
openapi_url="/api/openapi.json",
|
||||
)
|
||||
|
||||
# ─── CORS ─────────────────────────────────────────────────────────────────────
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# ─── Роутеры ──────────────────────────────────────────────────────────────────
|
||||
app.include_router(auth.router, prefix="/api")
|
||||
app.include_router(tasks.router, prefix="/api")
|
||||
app.include_router(search.router, prefix="/api")
|
||||
app.include_router(documents.router, prefix="/api")
|
||||
app.include_router(reports.router, prefix="/api")
|
||||
|
||||
|
||||
# ─── WebSocket ────────────────────────────────────────────────────────────────
|
||||
@app.websocket("/ws/tasks/{task_id}")
|
||||
async def websocket_task_updates(websocket: WebSocket, task_id: str) -> None:
|
||||
"""
|
||||
WebSocket endpoint для real-time обновлений статуса задачи.
|
||||
|
||||
Клиент подключается и получает обновления при изменении статуса задачи.
|
||||
Соединение закрывается при получении статуса done/failed.
|
||||
"""
|
||||
await ws_manager.connect(task_id, websocket)
|
||||
try:
|
||||
while True:
|
||||
# Ждём входящих сообщений (ping/pong для поддержания соединения)
|
||||
data = await websocket.receive_text()
|
||||
if data == "ping":
|
||||
await websocket.send_text("pong")
|
||||
except WebSocketDisconnect:
|
||||
ws_manager.disconnect(task_id, websocket)
|
||||
logger.info(f"WebSocket клиент отключился от задачи {task_id!r}")
|
||||
|
||||
|
||||
# ─── Health check ──────────────────────────────────────────────────────────────
|
||||
@app.get("/health", tags=["monitoring"])
|
||||
async def health_check() -> JSONResponse:
|
||||
"""Проверка работоспособности сервиса."""
|
||||
return JSONResponse(
|
||||
content={
|
||||
"status": "ok",
|
||||
"service": "api",
|
||||
"version": "0.1.0",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
async def root() -> JSONResponse:
|
||||
"""Корневой endpoint — редирект к документации."""
|
||||
return JSONResponse(
|
||||
content={"message": "Академический помощник API", "docs": "/api/docs"}
|
||||
)
|
||||
0
services/api/app/schemas/__init__.py
Normal file
0
services/api/app/schemas/__init__.py
Normal file
38
services/api/app/schemas/auth.py
Normal file
38
services/api/app/schemas/auth.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""Схемы для аутентификации и авторизации."""
|
||||
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
|
||||
|
||||
class RegisterRequest(BaseModel):
|
||||
"""Запрос на регистрацию нового пользователя."""
|
||||
|
||||
email: EmailStr
|
||||
password: str = Field(min_length=8, max_length=128)
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
"""Запрос на вход в систему."""
|
||||
|
||||
email: EmailStr
|
||||
password: str
|
||||
|
||||
|
||||
class UserInToken(BaseModel):
|
||||
"""Минимальная информация о пользователе в JWT токене."""
|
||||
|
||||
id: int
|
||||
email: str
|
||||
name: str
|
||||
plan: str
|
||||
is_verified: bool
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
"""Ответ с JWT токеном и данными пользователя."""
|
||||
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
user: UserInToken
|
||||
46
services/api/app/schemas/reports.py
Normal file
46
services/api/app/schemas/reports.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Схемы для отчётов о плагиате и результатов поиска."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class PlagiarismMatch(BaseModel):
|
||||
"""Совпадение фрагмента с источником."""
|
||||
|
||||
fragment: str
|
||||
position_start: int
|
||||
position_end: int
|
||||
similarity: float = Field(ge=0.0, le=100.0, description="Схожесть в процентах")
|
||||
method: str = Field(description="Метод обнаружения: exact, fuzzy, semantic+llm")
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
reason: str | None = None
|
||||
source_title: str
|
||||
source_url: str | None = None
|
||||
source_db: str = Field(description="База данных: openalex, cyberleninka, arxiv, и т.д.")
|
||||
|
||||
|
||||
class PlagiarismReport(BaseModel):
|
||||
"""Полный отчёт о плагиате."""
|
||||
|
||||
task_id: str
|
||||
overall_similarity: float = Field(ge=0.0, le=100.0)
|
||||
matches: list[PlagiarismMatch] = Field(default_factory=list)
|
||||
total_fragments: int
|
||||
flagged_fragments: int
|
||||
checked_at: datetime
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""Источник в результатах поиска."""
|
||||
|
||||
id: int
|
||||
title: str
|
||||
authors: list[dict] = Field(default_factory=list)
|
||||
year: int | None = None
|
||||
journal: str | None = None
|
||||
abstract: str | None = None
|
||||
url: str | None = None
|
||||
relevance_score: float = Field(ge=0.0, le=1.0)
|
||||
gost_citation: str = Field(description="Отформатированная ГОСТ-цитата")
|
||||
source_db: str
|
||||
39
services/api/app/schemas/tasks.py
Normal file
39
services/api/app/schemas/tasks.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Схемы для задач и поиска."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class TaskCreate(BaseModel):
|
||||
"""Создание задачи."""
|
||||
|
||||
type: str
|
||||
input_data: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class TaskResponse(BaseModel):
|
||||
"""Ответ с информацией о задаче."""
|
||||
|
||||
id: str
|
||||
type: str
|
||||
status: str
|
||||
queue_position: int | None = None
|
||||
eta_seconds: int | None = None
|
||||
input_data: dict[str, Any] = Field(default_factory=dict)
|
||||
result: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Запрос на семантический поиск источников."""
|
||||
|
||||
query: str = Field(min_length=3, max_length=1000)
|
||||
lang: str | None = Field(default=None, description="Язык: ru, en или None для всех")
|
||||
year_from: int | None = Field(default=None, ge=1900, le=2100)
|
||||
year_to: int | None = Field(default=None, ge=1900, le=2100)
|
||||
category: str | None = Field(default=None, description="Тематическая категория")
|
||||
17
services/api/requirements.txt
Normal file
17
services/api/requirements.txt
Normal file
@@ -0,0 +1,17 @@
|
||||
fastapi==0.111.0
|
||||
uvicorn[standard]==0.30.0
|
||||
sqlalchemy==2.0.30
|
||||
alembic==1.13.1
|
||||
asyncpg==0.29.0
|
||||
psycopg2-binary==2.9.9
|
||||
redis==5.0.4
|
||||
celery==5.4.0
|
||||
pydantic==2.7.1
|
||||
pydantic-settings==2.2.1
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
python-multipart==0.0.9
|
||||
minio==7.2.7
|
||||
httpx==0.27.0
|
||||
aiofiles==23.2.1
|
||||
websockets==12.0
|
||||
14
services/frontend/index.html
Normal file
14
services/frontend/index.html
Normal file
@@ -0,0 +1,14 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Академический помощник — поиск источников, проверка плагиата, ГОСТ-библиография" />
|
||||
<title>Академический помощник</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
36
services/frontend/package.json
Normal file
36
services/frontend/package.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "academic-helper-frontend",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"lint": "eslint . --ext ts,tsx",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^18.3.0",
|
||||
"react-dom": "^18.3.0",
|
||||
"react-router-dom": "^6.23.0",
|
||||
"@tanstack/react-query": "^5.40.0",
|
||||
"zustand": "^4.5.2",
|
||||
"axios": "^1.7.2",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"react-hot-toast": "^2.4.1",
|
||||
"lucide-react": "^0.390.0",
|
||||
"clsx": "^2.1.1",
|
||||
"tailwind-merge": "^2.3.0",
|
||||
"date-fns": "^3.6.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.3.3",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react": "^4.3.0",
|
||||
"typescript": "^5.4.5",
|
||||
"vite": "^5.2.12",
|
||||
"tailwindcss": "^3.4.4",
|
||||
"postcss": "^8.4.38",
|
||||
"autoprefixer": "^10.4.19",
|
||||
"eslint": "^9.4.0"
|
||||
}
|
||||
}
|
||||
6
services/frontend/postcss.config.js
Normal file
6
services/frontend/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
80
services/frontend/src/api/client.ts
Normal file
80
services/frontend/src/api/client.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import axios from 'axios';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
|
||||
export const api = axios.create({
|
||||
baseURL: import.meta.env.VITE_API_URL || '/api',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
// Добавить JWT токен к каждому запросу
|
||||
api.interceptors.request.use((config) => {
|
||||
const token = useAuthStore.getState().token;
|
||||
if (token) {
|
||||
config.headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
return config;
|
||||
});
|
||||
|
||||
// Обработка ответов: при 401 — разлогинить
|
||||
api.interceptors.response.use(
|
||||
(response) => response,
|
||||
(error) => {
|
||||
if (error.response?.status === 401) {
|
||||
useAuthStore.getState().logout();
|
||||
}
|
||||
return Promise.reject(error);
|
||||
}
|
||||
);
|
||||
|
||||
// ─── API методы ───────────────────────────────────────────────────────────────
|
||||
|
||||
export const authApi = {
|
||||
register: (data: { email: string; password: string; name: string }) =>
|
||||
api.post('/auth/register', data),
|
||||
|
||||
login: (data: { email: string; password: string }) =>
|
||||
api.post('/auth/login', data),
|
||||
|
||||
me: () => api.get('/auth/me'),
|
||||
|
||||
verifyEmail: (token: string) =>
|
||||
api.post(`/auth/verify-email/${token}`),
|
||||
};
|
||||
|
||||
export const tasksApi = {
|
||||
list: (limit = 20, offset = 0) =>
|
||||
api.get('/tasks/', { params: { limit, offset } }),
|
||||
|
||||
get: (taskId: string) =>
|
||||
api.get(`/tasks/${taskId}`),
|
||||
|
||||
delete: (taskId: string) =>
|
||||
api.delete(`/tasks/${taskId}`),
|
||||
};
|
||||
|
||||
export const searchApi = {
|
||||
create: (data: {
|
||||
query: string;
|
||||
lang?: string;
|
||||
year_from?: number;
|
||||
year_to?: number;
|
||||
category?: string;
|
||||
}) => api.post('/search/', data),
|
||||
};
|
||||
|
||||
export const documentsApi = {
|
||||
uploadForCheck: (file: File) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return api.post('/documents/check', formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' },
|
||||
timeout: 120000, // 2 минуты для загрузки
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export const reportsApi = {
|
||||
get: (taskId: string) =>
|
||||
api.get(`/reports/${taskId}`),
|
||||
};
|
||||
104
services/frontend/src/components/DropZone.tsx
Normal file
104
services/frontend/src/components/DropZone.tsx
Normal file
@@ -0,0 +1,104 @@
|
||||
import React, { useCallback } from 'react';
|
||||
import { useDropzone } from 'react-dropzone';
|
||||
import { Upload, FileText, X } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
|
||||
interface DropZoneProps {
|
||||
onFile: (file: File) => void;
|
||||
file?: File | null;
|
||||
onClear?: () => void;
|
||||
}
|
||||
|
||||
const ACCEPTED_TYPES = {
|
||||
'application/pdf': ['.pdf'],
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
|
||||
'text/plain': ['.txt'],
|
||||
};
|
||||
|
||||
const MAX_SIZE_BYTES = 100 * 1024 * 1024; // 100 MB
|
||||
const MAX_SIZE_LABEL = '100 МБ';
|
||||
|
||||
function formatFileSize(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} Б`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} КБ`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} МБ`;
|
||||
}
|
||||
|
||||
export function DropZone({ onFile, file, onClear }: DropZoneProps) {
|
||||
const onDrop = useCallback(
|
||||
(acceptedFiles: File[]) => {
|
||||
if (acceptedFiles.length > 0) {
|
||||
onFile(acceptedFiles[0]);
|
||||
}
|
||||
},
|
||||
[onFile]
|
||||
);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive, fileRejections } = useDropzone({
|
||||
onDrop,
|
||||
accept: ACCEPTED_TYPES,
|
||||
maxSize: MAX_SIZE_BYTES,
|
||||
maxFiles: 1,
|
||||
});
|
||||
|
||||
const rejectionError = fileRejections[0]?.errors[0]?.message;
|
||||
|
||||
if (file) {
|
||||
return (
|
||||
<div className="flex items-center gap-3 p-4 bg-brand-50 border border-brand-200 rounded-xl">
|
||||
<div className="p-2 bg-brand-100 rounded-lg">
|
||||
<FileText className="w-6 h-6 text-brand-600" />
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-medium text-gray-900 truncate">{file.name}</p>
|
||||
<p className="text-xs text-gray-500">{formatFileSize(file.size)}</p>
|
||||
</div>
|
||||
{onClear && (
|
||||
<button
|
||||
onClick={onClear}
|
||||
className="p-1.5 rounded-lg text-gray-400 hover:text-gray-600 hover:bg-white transition-colors"
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div
|
||||
{...getRootProps()}
|
||||
className={clsx(
|
||||
'border-2 border-dashed rounded-xl p-10 text-center cursor-pointer transition-all duration-200',
|
||||
isDragActive
|
||||
? 'border-brand-400 bg-brand-50'
|
||||
: 'border-gray-200 hover:border-brand-300 hover:bg-gray-50'
|
||||
)}
|
||||
>
|
||||
<input {...getInputProps()} />
|
||||
<Upload
|
||||
className={clsx(
|
||||
'w-10 h-10 mx-auto mb-3',
|
||||
isDragActive ? 'text-brand-500' : 'text-gray-300'
|
||||
)}
|
||||
/>
|
||||
{isDragActive ? (
|
||||
<p className="text-base font-medium text-brand-600">Отпустите файл для загрузки</p>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-base font-medium text-gray-700 mb-1">
|
||||
Перетащите файл или нажмите для выбора
|
||||
</p>
|
||||
<p className="text-sm text-gray-400">
|
||||
PDF, DOCX, TXT — до {MAX_SIZE_LABEL}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
{rejectionError && (
|
||||
<p className="mt-2 text-sm text-red-500">{rejectionError}</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
41
services/frontend/src/components/GostCitation.tsx
Normal file
41
services/frontend/src/components/GostCitation.tsx
Normal file
@@ -0,0 +1,41 @@
|
||||
import React from 'react';
|
||||
import { Copy, Check } from 'lucide-react';
|
||||
import { useState } from 'react';
|
||||
|
||||
interface GostCitationProps {
|
||||
citation: string;
|
||||
number?: number;
|
||||
}
|
||||
|
||||
export function GostCitation({ citation, number }: GostCitationProps) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const handleCopy = () => {
|
||||
navigator.clipboard.writeText(citation).then(() => {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="group flex items-start gap-3 py-2 px-3 rounded-lg hover:bg-gray-50 transition-colors">
|
||||
{number !== undefined && (
|
||||
<span className="flex-shrink-0 text-sm text-gray-400 font-mono w-6 pt-0.5 text-right">
|
||||
{number}.
|
||||
</span>
|
||||
)}
|
||||
<p className="flex-1 text-sm text-gray-700 leading-relaxed">{citation}</p>
|
||||
<button
|
||||
onClick={handleCopy}
|
||||
className="flex-shrink-0 p-1 rounded text-gray-300 hover:text-gray-600 opacity-0 group-hover:opacity-100 transition-all"
|
||||
title="Копировать"
|
||||
>
|
||||
{copied ? (
|
||||
<Check className="w-4 h-4 text-green-500" />
|
||||
) : (
|
||||
<Copy className="w-4 h-4" />
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
149
services/frontend/src/components/Layout.tsx
Normal file
149
services/frontend/src/components/Layout.tsx
Normal file
@@ -0,0 +1,149 @@
|
||||
import React from 'react';
|
||||
import { Link, NavLink, useNavigate } from 'react-router-dom';
|
||||
import { GraduationCap, Search, BookOpen, Upload, LayoutDashboard, LogOut, User, BookMarked } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
|
||||
interface LayoutProps {
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
export function Layout({ children }: LayoutProps) {
|
||||
const { isAuthenticated, user, logout } = useAuthStore();
|
||||
const navigate = useNavigate();
|
||||
|
||||
const handleLogout = () => {
|
||||
logout();
|
||||
navigate('/');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50">
|
||||
{/* Навигация */}
|
||||
<nav className="bg-white border-b border-gray-100 sticky top-0 z-50">
|
||||
<div className="max-w-6xl mx-auto px-4 sm:px-6 lg:px-8">
|
||||
<div className="flex items-center justify-between h-16">
|
||||
{/* Логотип */}
|
||||
<Link
|
||||
to="/"
|
||||
className="flex items-center gap-2.5 text-gray-900 hover:text-brand-600 transition-colors"
|
||||
>
|
||||
<div className="p-1.5 bg-brand-600 rounded-lg">
|
||||
<GraduationCap className="w-5 h-5 text-white" />
|
||||
</div>
|
||||
<span className="font-semibold text-base hidden sm:block">Академический помощник</span>
|
||||
<span className="font-semibold text-base sm:hidden">АкадПомощник</span>
|
||||
</Link>
|
||||
|
||||
{/* Центральная навигация */}
|
||||
<div className="flex items-center gap-1">
|
||||
<NavLink
|
||||
to="/search"
|
||||
className={({ isActive }) =>
|
||||
clsx(
|
||||
'flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm font-medium transition-colors',
|
||||
isActive
|
||||
? 'bg-brand-50 text-brand-700'
|
||||
: 'text-gray-600 hover:bg-gray-100'
|
||||
)
|
||||
}
|
||||
>
|
||||
<Search className="w-4 h-4" />
|
||||
<span className="hidden md:block">Поиск</span>
|
||||
</NavLink>
|
||||
|
||||
<NavLink
|
||||
to="/check"
|
||||
className={({ isActive }) =>
|
||||
clsx(
|
||||
'flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm font-medium transition-colors',
|
||||
isActive
|
||||
? 'bg-brand-50 text-brand-700'
|
||||
: 'text-gray-600 hover:bg-gray-100'
|
||||
)
|
||||
}
|
||||
>
|
||||
<Upload className="w-4 h-4" />
|
||||
<span className="hidden md:block">Плагиат</span>
|
||||
</NavLink>
|
||||
|
||||
<NavLink
|
||||
to="/bibliography"
|
||||
className={({ isActive }) =>
|
||||
clsx(
|
||||
'flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm font-medium transition-colors',
|
||||
isActive
|
||||
? 'bg-brand-50 text-brand-700'
|
||||
: 'text-gray-600 hover:bg-gray-100'
|
||||
)
|
||||
}
|
||||
>
|
||||
<BookOpen className="w-4 h-4" />
|
||||
<span className="hidden md:block">Библиография</span>
|
||||
</NavLink>
|
||||
</div>
|
||||
|
||||
{/* Авторизация */}
|
||||
<div className="flex items-center gap-2">
|
||||
{isAuthenticated ? (
|
||||
<>
|
||||
<NavLink
|
||||
to="/cabinet"
|
||||
className={({ isActive }) =>
|
||||
clsx(
|
||||
'flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm font-medium transition-colors',
|
||||
isActive
|
||||
? 'bg-brand-50 text-brand-700'
|
||||
: 'text-gray-600 hover:bg-gray-100'
|
||||
)
|
||||
}
|
||||
>
|
||||
<LayoutDashboard className="w-4 h-4" />
|
||||
<span className="hidden md:block">Кабинет</span>
|
||||
</NavLink>
|
||||
<div className="flex items-center gap-2 pl-2 border-l border-gray-100">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<div className="w-8 h-8 bg-brand-100 rounded-full flex items-center justify-center">
|
||||
<span className="text-xs font-semibold text-brand-700">
|
||||
{user?.name?.[0]?.toUpperCase() || 'U'}
|
||||
</span>
|
||||
</div>
|
||||
<span className="text-sm text-gray-600 hidden lg:block">{user?.name}</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={handleLogout}
|
||||
className="p-1.5 text-gray-400 hover:text-gray-600 hover:bg-gray-100 rounded-lg transition-colors"
|
||||
title="Выйти"
|
||||
>
|
||||
<LogOut className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Link
|
||||
to="/login"
|
||||
className="px-4 py-2 text-sm font-medium text-gray-600 hover:text-gray-900 transition-colors"
|
||||
>
|
||||
Войти
|
||||
</Link>
|
||||
<Link
|
||||
to="/register"
|
||||
className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors"
|
||||
>
|
||||
Регистрация
|
||||
</Link>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
{/* Основной контент */}
|
||||
<main className="max-w-6xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
{children}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
165
services/frontend/src/components/PlagiarismReport.tsx
Normal file
165
services/frontend/src/components/PlagiarismReport.tsx
Normal file
@@ -0,0 +1,165 @@
|
||||
import React from 'react';
|
||||
import { AlertTriangle, CheckCircle, Info } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import type { PlagiarismResultData } from '../types';
|
||||
|
||||
interface PlagiarismReportProps {
|
||||
data: PlagiarismResultData;
|
||||
}
|
||||
|
||||
function getSimilarityLevel(pct: number): {
|
||||
color: string;
|
||||
bgColor: string;
|
||||
borderColor: string;
|
||||
icon: typeof CheckCircle;
|
||||
label: string;
|
||||
} {
|
||||
if (pct > 30) {
|
||||
return {
|
||||
color: 'text-red-700',
|
||||
bgColor: 'bg-red-50',
|
||||
borderColor: 'border-red-200',
|
||||
icon: AlertTriangle,
|
||||
label: 'Высокий уровень схожести',
|
||||
};
|
||||
}
|
||||
if (pct > 10) {
|
||||
return {
|
||||
color: 'text-yellow-700',
|
||||
bgColor: 'bg-yellow-50',
|
||||
borderColor: 'border-yellow-200',
|
||||
icon: Info,
|
||||
label: 'Умеренный уровень схожести',
|
||||
};
|
||||
}
|
||||
return {
|
||||
color: 'text-green-700',
|
||||
bgColor: 'bg-green-50',
|
||||
borderColor: 'border-green-200',
|
||||
icon: CheckCircle,
|
||||
label: 'Низкий уровень схожести',
|
||||
};
|
||||
}
|
||||
|
||||
const METHOD_LABELS: Record<string, string> = {
|
||||
exact: 'Точное совпадение',
|
||||
fuzzy: 'Нечёткое совпадение',
|
||||
'semantic+llm': 'Семантика + LLM',
|
||||
};
|
||||
|
||||
export function PlagiarismReport({ data }: PlagiarismReportProps) {
|
||||
const level = getSimilarityLevel(data.overall_similarity);
|
||||
const Icon = level.icon;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Итоговый показатель */}
|
||||
<div className={clsx('p-6 rounded-xl border-2', level.bgColor, level.borderColor)}>
|
||||
<div className="flex items-center gap-4">
|
||||
<div className={clsx('text-5xl font-bold tabular-nums', level.color)}>
|
||||
{data.overall_similarity.toFixed(1)}%
|
||||
</div>
|
||||
<div>
|
||||
<div className={clsx('flex items-center gap-1.5 font-semibold text-base', level.color)}>
|
||||
<Icon className="w-5 h-5" />
|
||||
{level.label}
|
||||
</div>
|
||||
<p className="text-sm text-gray-600 mt-1">
|
||||
Проверено фрагментов: {data.total_fragments} · Выявлено совпадений: {data.flagged_fragments}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Прогресс-бар */}
|
||||
<div className="mt-4 bg-white/60 rounded-full h-3 overflow-hidden">
|
||||
<div
|
||||
className={clsx('h-full rounded-full transition-all', {
|
||||
'bg-red-500': data.overall_similarity > 30,
|
||||
'bg-yellow-400': data.overall_similarity > 10 && data.overall_similarity <= 30,
|
||||
'bg-green-500': data.overall_similarity <= 10,
|
||||
})}
|
||||
style={{ width: `${Math.min(data.overall_similarity, 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Методы обнаружения */}
|
||||
{data.by_method && (
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{[
|
||||
{ key: 'exact', label: 'Точные', count: data.by_method.exact },
|
||||
{ key: 'fuzzy', label: 'Нечёткие', count: data.by_method.fuzzy },
|
||||
{ key: 'semantic_llm', label: 'Семантика+LLM', count: data.by_method.semantic_llm },
|
||||
].map(({ key, label, count }) => (
|
||||
<div key={key} className="p-3 bg-gray-50 rounded-lg text-center">
|
||||
<div className="text-2xl font-bold text-gray-800">{count}</div>
|
||||
<div className="text-xs text-gray-500 mt-0.5">{label}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Список совпадений */}
|
||||
{data.matches.length > 0 && (
|
||||
<div>
|
||||
<h3 className="text-base font-semibold text-gray-900 mb-3">
|
||||
Обнаруженные совпадения ({data.matches.length})
|
||||
</h3>
|
||||
<div className="space-y-3">
|
||||
{data.matches.map((match, i) => (
|
||||
<div key={i} className="border border-gray-200 rounded-xl overflow-hidden">
|
||||
{/* Шапка совпадения */}
|
||||
<div className="flex items-center gap-3 px-4 py-2.5 bg-gray-50 border-b border-gray-200">
|
||||
<span className="text-sm font-medium text-gray-700 flex-1 truncate">
|
||||
{match.source_title}
|
||||
</span>
|
||||
<span className={clsx(
|
||||
'text-xs font-medium px-2 py-0.5 rounded-full',
|
||||
match.similarity > 70
|
||||
? 'bg-red-100 text-red-700'
|
||||
: match.similarity > 40
|
||||
? 'bg-yellow-100 text-yellow-700'
|
||||
: 'bg-gray-100 text-gray-600'
|
||||
)}>
|
||||
{match.similarity.toFixed(0)}%
|
||||
</span>
|
||||
<span className="text-xs text-gray-400 px-2 py-0.5 bg-white rounded-full border border-gray-200">
|
||||
{METHOD_LABELS[match.method] || match.method}
|
||||
</span>
|
||||
</div>
|
||||
{/* Фрагмент */}
|
||||
<div className="px-4 py-3">
|
||||
<p className="text-sm text-gray-700 italic leading-relaxed line-clamp-3">
|
||||
«{match.fragment}»
|
||||
</p>
|
||||
{match.reason && (
|
||||
<p className="text-xs text-gray-500 mt-2">
|
||||
{match.reason}
|
||||
</p>
|
||||
)}
|
||||
{match.source_url && (
|
||||
<a
|
||||
href={match.source_url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-xs text-brand-600 hover:underline mt-1.5 inline-block"
|
||||
>
|
||||
Открыть источник →
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{data.matches.length === 0 && (
|
||||
<div className="text-center py-8 text-gray-500">
|
||||
<CheckCircle className="w-12 h-12 text-green-400 mx-auto mb-3" />
|
||||
<p className="text-base font-medium">Совпадений не обнаружено</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
75
services/frontend/src/components/SearchBar.tsx
Normal file
75
services/frontend/src/components/SearchBar.tsx
Normal file
@@ -0,0 +1,75 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Search } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
|
||||
interface SearchBarProps {
|
||||
onSearch: (query: string) => void;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
size?: 'sm' | 'md' | 'lg';
|
||||
isLoading?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function SearchBar({
|
||||
onSearch,
|
||||
defaultValue = '',
|
||||
placeholder = 'Введите тему или запрос для поиска источников...',
|
||||
size = 'md',
|
||||
isLoading = false,
|
||||
className,
|
||||
}: SearchBarProps) {
|
||||
const [query, setQuery] = useState(defaultValue);
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
const trimmed = query.trim();
|
||||
if (trimmed.length < 3) return;
|
||||
onSearch(trimmed);
|
||||
};
|
||||
|
||||
return (
|
||||
<form onSubmit={handleSubmit} className={clsx('w-full', className)}>
|
||||
<div className="relative flex items-center">
|
||||
<Search
|
||||
className={clsx(
|
||||
'absolute left-4 text-gray-400 pointer-events-none',
|
||||
size === 'lg' ? 'w-6 h-6' : 'w-5 h-5'
|
||||
)}
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder={placeholder}
|
||||
className={clsx(
|
||||
'w-full bg-white border border-gray-200 rounded-xl shadow-sm',
|
||||
'placeholder:text-gray-400 text-gray-900',
|
||||
'focus:outline-none focus:ring-2 focus:ring-brand-500 focus:border-transparent',
|
||||
'transition-all duration-200',
|
||||
size === 'lg' && 'pl-14 pr-36 py-5 text-lg',
|
||||
size === 'md' && 'pl-12 pr-28 py-3 text-base',
|
||||
size === 'sm' && 'pl-10 pr-24 py-2 text-sm'
|
||||
)}
|
||||
minLength={3}
|
||||
maxLength={1000}
|
||||
required
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isLoading || query.trim().length < 3}
|
||||
className={clsx(
|
||||
'absolute right-2 bg-brand-600 text-white rounded-lg font-medium',
|
||||
'hover:bg-brand-700 disabled:opacity-50 disabled:cursor-not-allowed',
|
||||
'transition-colors duration-200',
|
||||
size === 'lg' && 'px-6 py-3 text-base',
|
||||
size === 'md' && 'px-5 py-2 text-sm',
|
||||
size === 'sm' && 'px-4 py-1.5 text-xs'
|
||||
)}
|
||||
>
|
||||
{isLoading ? 'Поиск...' : 'Найти'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
144
services/frontend/src/components/SourceCard.tsx
Normal file
144
services/frontend/src/components/SourceCard.tsx
Normal file
@@ -0,0 +1,144 @@
|
||||
import React from 'react';
|
||||
import { ExternalLink, BookmarkPlus, BookmarkCheck, Copy } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import toast from 'react-hot-toast';
|
||||
import { useBibliographyStore } from '../store/bibliography';
|
||||
import type { SearchSource } from '../types';
|
||||
|
||||
interface SourceCardProps {
|
||||
source: SearchSource;
|
||||
}
|
||||
|
||||
const SOURCE_DB_LABELS: Record<string, string> = {
|
||||
openalex: 'OpenAlex',
|
||||
cyberleninka: 'КиберЛенинка',
|
||||
arxiv: 'arXiv',
|
||||
wikipedia_ru: 'Wikipedia RU',
|
||||
wikipedia_en: 'Wikipedia EN',
|
||||
};
|
||||
|
||||
function getRelevanceColor(score: number): string {
|
||||
if (score >= 0.7) return 'text-green-700 bg-green-50 border-green-200';
|
||||
if (score >= 0.4) return 'text-yellow-700 bg-yellow-50 border-yellow-200';
|
||||
return 'text-gray-600 bg-gray-50 border-gray-200';
|
||||
}
|
||||
|
||||
export function SourceCard({ source }: SourceCardProps) {
|
||||
const { addSource, removeSource, hasSource } = useBibliographyStore();
|
||||
const inBibliography = hasSource(source.id);
|
||||
|
||||
const handleToggleBibliography = () => {
|
||||
if (inBibliography) {
|
||||
removeSource(source.id);
|
||||
toast.success('Удалено из библиографии');
|
||||
} else {
|
||||
addSource(source);
|
||||
toast.success('Добавлено в библиографию');
|
||||
}
|
||||
};
|
||||
|
||||
const handleCopyCitation = () => {
|
||||
navigator.clipboard.writeText(source.gost_citation).then(() => {
|
||||
toast.success('ГОСТ-цитата скопирована');
|
||||
});
|
||||
};
|
||||
|
||||
const authorsStr = source.authors
|
||||
.slice(0, 3)
|
||||
.map((a) => `${a.last_name} ${a.initials || ''}`.trim())
|
||||
.join(', ');
|
||||
|
||||
return (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5 hover:border-gray-200 hover:shadow-sm transition-all duration-200">
|
||||
{/* Заголовок и значки */}
|
||||
<div className="flex items-start justify-between gap-3 mb-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<h3 className="text-base font-semibold text-gray-900 leading-snug line-clamp-2 mb-1">
|
||||
{source.title}
|
||||
</h3>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
{authorsStr && (
|
||||
<span className="text-sm text-gray-500">{authorsStr}</span>
|
||||
)}
|
||||
{source.year && (
|
||||
<span className="text-sm text-gray-400">{source.year}</span>
|
||||
)}
|
||||
{source.journal && (
|
||||
<span className="text-sm text-gray-400 italic">{source.journal}</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Бейдж релевантности */}
|
||||
<div className={clsx(
|
||||
'flex-shrink-0 px-2 py-1 rounded-lg text-xs font-medium border',
|
||||
getRelevanceColor(source.relevance_score)
|
||||
)}>
|
||||
{(source.relevance_score * 100).toFixed(0)}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Аннотация */}
|
||||
{source.abstract && (
|
||||
<p className="text-sm text-gray-600 line-clamp-3 mb-3">
|
||||
{source.abstract}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* ГОСТ-цитата */}
|
||||
<div className="bg-gray-50 rounded-lg p-3 mb-3">
|
||||
<p className="text-xs text-gray-500 mb-1 font-medium">ГОСТ-цитата:</p>
|
||||
<p className="text-xs text-gray-700 leading-relaxed">{source.gost_citation}</p>
|
||||
</div>
|
||||
|
||||
{/* Действия */}
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
{/* Источник */}
|
||||
<span className="px-2 py-1 bg-gray-100 text-gray-500 text-xs rounded-md">
|
||||
{SOURCE_DB_LABELS[source.source_db] || source.source_db}
|
||||
</span>
|
||||
|
||||
<div className="ml-auto flex items-center gap-1.5">
|
||||
{/* Открыть источник */}
|
||||
{source.url && (
|
||||
<a
|
||||
href={source.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium text-gray-600 bg-white border border-gray-200 rounded-lg hover:bg-gray-50 transition-colors"
|
||||
>
|
||||
<ExternalLink className="w-3 h-3" />
|
||||
Открыть
|
||||
</a>
|
||||
)}
|
||||
|
||||
{/* Копировать цитату */}
|
||||
<button
|
||||
onClick={handleCopyCitation}
|
||||
className="inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium text-gray-600 bg-white border border-gray-200 rounded-lg hover:bg-gray-50 transition-colors"
|
||||
>
|
||||
<Copy className="w-3 h-3" />
|
||||
Цитата
|
||||
</button>
|
||||
|
||||
{/* В библиографию */}
|
||||
<button
|
||||
onClick={handleToggleBibliography}
|
||||
className={clsx(
|
||||
'inline-flex items-center gap-1 px-3 py-1.5 text-xs font-medium rounded-lg transition-colors',
|
||||
inBibliography
|
||||
? 'bg-brand-600 text-white hover:bg-brand-700'
|
||||
: 'bg-white text-brand-600 border border-brand-200 hover:bg-brand-50'
|
||||
)}
|
||||
>
|
||||
{inBibliography ? (
|
||||
<><BookmarkCheck className="w-3 h-3" />Добавлено</>
|
||||
) : (
|
||||
<><BookmarkPlus className="w-3 h-3" />В библиографию</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
55
services/frontend/src/components/StatusBadge.tsx
Normal file
55
services/frontend/src/components/StatusBadge.tsx
Normal file
@@ -0,0 +1,55 @@
|
||||
import React from 'react';
|
||||
import { clsx } from 'clsx';
|
||||
import type { TaskStatus } from '../types';
|
||||
|
||||
interface StatusBadgeProps {
|
||||
status: TaskStatus;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const STATUS_CONFIG: Record<TaskStatus, { label: string; className: string }> = {
|
||||
queued: {
|
||||
label: 'В очереди',
|
||||
className: 'bg-gray-100 text-gray-600 border-gray-200',
|
||||
},
|
||||
processing: {
|
||||
label: 'Выполняется',
|
||||
className: 'bg-blue-50 text-blue-700 border-blue-200 animate-pulse',
|
||||
},
|
||||
done: {
|
||||
label: 'Готово',
|
||||
className: 'bg-green-50 text-green-700 border-green-200',
|
||||
},
|
||||
failed: {
|
||||
label: 'Ошибка',
|
||||
className: 'bg-red-50 text-red-700 border-red-200',
|
||||
},
|
||||
};
|
||||
|
||||
export function StatusBadge({ status, className }: StatusBadgeProps) {
|
||||
const config = STATUS_CONFIG[status];
|
||||
|
||||
return (
|
||||
<span
|
||||
className={clsx(
|
||||
'inline-flex items-center gap-1.5 px-2.5 py-0.5 rounded-full text-xs font-medium border',
|
||||
config.className,
|
||||
className
|
||||
)}
|
||||
>
|
||||
{status === 'processing' && (
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-blue-500 animate-pulse" />
|
||||
)}
|
||||
{status === 'done' && (
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-green-500" />
|
||||
)}
|
||||
{status === 'failed' && (
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-red-500" />
|
||||
)}
|
||||
{status === 'queued' && (
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-gray-400" />
|
||||
)}
|
||||
{config.label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
103
services/frontend/src/components/TaskCard.tsx
Normal file
103
services/frontend/src/components/TaskCard.tsx
Normal file
@@ -0,0 +1,103 @@
|
||||
import React from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { formatDistanceToNow } from 'date-fns';
|
||||
import { ru } from 'date-fns/locale';
|
||||
import { Search, FileText, BookOpen, AlignLeft, ChevronRight } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import { StatusBadge } from './StatusBadge';
|
||||
import type { Task } from '../types';
|
||||
|
||||
interface TaskCardProps {
|
||||
task: Task;
|
||||
}
|
||||
|
||||
const TYPE_CONFIG = {
|
||||
search: {
|
||||
icon: Search,
|
||||
label: 'Поиск источников',
|
||||
color: 'text-blue-600',
|
||||
bg: 'bg-blue-50',
|
||||
},
|
||||
plagiarism: {
|
||||
icon: FileText,
|
||||
label: 'Проверка плагиата',
|
||||
color: 'text-purple-600',
|
||||
bg: 'bg-purple-50',
|
||||
},
|
||||
gost: {
|
||||
icon: BookOpen,
|
||||
label: 'ГОСТ библиография',
|
||||
color: 'text-emerald-600',
|
||||
bg: 'bg-emerald-50',
|
||||
},
|
||||
summarize: {
|
||||
icon: AlignLeft,
|
||||
label: 'Краткое изложение',
|
||||
color: 'text-orange-600',
|
||||
bg: 'bg-orange-50',
|
||||
},
|
||||
};
|
||||
|
||||
function getTaskSummary(task: Task): string {
|
||||
if (task.status === 'queued') {
|
||||
const pos = task.queue_position;
|
||||
return pos ? `Позиция в очереди: ${pos}` : 'Ожидает выполнения';
|
||||
}
|
||||
if (task.status === 'processing') return 'Выполняется...';
|
||||
if (task.status === 'failed') return task.error || 'Произошла ошибка';
|
||||
|
||||
const result = task.result as Record<string, unknown> | undefined;
|
||||
if (!result) return 'Результат готов';
|
||||
|
||||
if (task.type === 'search') {
|
||||
const total = result.total as number;
|
||||
return `Найдено ${total} источников`;
|
||||
}
|
||||
if (task.type === 'plagiarism') {
|
||||
const sim = result.overall_similarity as number;
|
||||
return `Схожесть: ${sim?.toFixed(1)}%`;
|
||||
}
|
||||
if (task.type === 'gost') {
|
||||
const count = (result.bibliography as unknown[])?.length;
|
||||
return `${count} записей в библиографии`;
|
||||
}
|
||||
return 'Результат готов';
|
||||
}
|
||||
|
||||
export function TaskCard({ task }: TaskCardProps) {
|
||||
const config = TYPE_CONFIG[task.type] || TYPE_CONFIG.search;
|
||||
const Icon = config.icon;
|
||||
const summary = getTaskSummary(task);
|
||||
|
||||
return (
|
||||
<Link
|
||||
to={`/tasks/${task.id}`}
|
||||
className="block group"
|
||||
>
|
||||
<div className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-100 hover:border-brand-200 hover:shadow-sm transition-all duration-200">
|
||||
{/* Иконка типа */}
|
||||
<div className={clsx('p-2.5 rounded-lg flex-shrink-0', config.bg)}>
|
||||
<Icon className={clsx('w-5 h-5', config.color)} />
|
||||
</div>
|
||||
|
||||
{/* Контент */}
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="text-sm font-medium text-gray-900">{config.label}</span>
|
||||
<StatusBadge status={task.status} />
|
||||
</div>
|
||||
<p className="text-sm text-gray-500 truncate">{summary}</p>
|
||||
<p className="text-xs text-gray-400 mt-0.5">
|
||||
{formatDistanceToNow(new Date(task.created_at), {
|
||||
addSuffix: true,
|
||||
locale: ru,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Стрелка */}
|
||||
<ChevronRight className="w-4 h-4 text-gray-300 group-hover:text-brand-500 flex-shrink-0 transition-colors" />
|
||||
</div>
|
||||
</Link>
|
||||
);
|
||||
}
|
||||
52
services/frontend/src/hooks/useTaskPolling.ts
Normal file
52
services/frontend/src/hooks/useTaskPolling.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import type { Task } from '../types';
|
||||
|
||||
/**
|
||||
* Хук для WebSocket подключения к задаче.
|
||||
* Обновляет кэш React Query при получении обновления статуса.
|
||||
*/
|
||||
export function useTaskWebSocket(taskId: string | undefined, enabled: boolean) {
|
||||
const queryClient = useQueryClient();
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!taskId || !enabled) return;
|
||||
|
||||
const wsUrl = `${window.location.protocol === 'https:' ? 'wss' : 'ws'}://${window.location.host}/ws/tasks/${taskId}`;
|
||||
const ws = new WebSocket(wsUrl);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data === 'pong') return;
|
||||
|
||||
// Обновить кэш задачи
|
||||
queryClient.setQueryData<Task>(['task', taskId], (old) => {
|
||||
if (!old) return old;
|
||||
return { ...old, ...data };
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn('Ошибка парсинга WebSocket сообщения:', e);
|
||||
}
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
console.warn(`WebSocket ошибка для задачи ${taskId}`);
|
||||
};
|
||||
|
||||
// Keepalive ping каждые 30 секунд
|
||||
const pingInterval = setInterval(() => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send('ping');
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
return () => {
|
||||
clearInterval(pingInterval);
|
||||
ws.close();
|
||||
wsRef.current = null;
|
||||
};
|
||||
}, [taskId, enabled, queryClient]);
|
||||
}
|
||||
10
services/frontend/src/index.css
Normal file
10
services/frontend/src/index.css
Normal file
@@ -0,0 +1,10 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
@layer base {
|
||||
html {
|
||||
font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
}
|
||||
59
services/frontend/src/main.tsx
Normal file
59
services/frontend/src/main.tsx
Normal file
@@ -0,0 +1,59 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import { BrowserRouter, Routes, Route } from 'react-router-dom';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import { Toaster } from 'react-hot-toast';
|
||||
|
||||
import { Layout } from './components/Layout';
|
||||
import { Home } from './pages/Home';
|
||||
import { Search } from './pages/Search';
|
||||
import { Check } from './pages/Check';
|
||||
import { Bibliography } from './pages/Bibliography';
|
||||
import { Cabinet } from './pages/Cabinet';
|
||||
import { Task } from './pages/Task';
|
||||
import { Pricing } from './pages/Pricing';
|
||||
import { Login } from './pages/Login';
|
||||
import { Register } from './pages/Register';
|
||||
|
||||
import './index.css';
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
staleTime: 30000,
|
||||
retry: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<BrowserRouter>
|
||||
<Layout>
|
||||
<Routes>
|
||||
<Route path="/" element={<Home />} />
|
||||
<Route path="/search" element={<Search />} />
|
||||
<Route path="/check" element={<Check />} />
|
||||
<Route path="/bibliography" element={<Bibliography />} />
|
||||
<Route path="/cabinet" element={<Cabinet />} />
|
||||
<Route path="/tasks/:taskId" element={<Task />} />
|
||||
<Route path="/pricing" element={<Pricing />} />
|
||||
<Route path="/login" element={<Login />} />
|
||||
<Route path="/register" element={<Register />} />
|
||||
</Routes>
|
||||
</Layout>
|
||||
</BrowserRouter>
|
||||
<Toaster
|
||||
position="top-right"
|
||||
toastOptions={{
|
||||
duration: 4000,
|
||||
style: {
|
||||
borderRadius: '12px',
|
||||
fontSize: '14px',
|
||||
},
|
||||
}}
|
||||
/>
|
||||
</QueryClientProvider>
|
||||
</React.StrictMode>
|
||||
);
|
||||
189
services/frontend/src/pages/Bibliography.tsx
Normal file
189
services/frontend/src/pages/Bibliography.tsx
Normal file
@@ -0,0 +1,189 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { BookOpen, Copy, Trash2, BookMarked } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { GostCitation } from '../components/GostCitation';
|
||||
import { useBibliographyStore } from '../store/bibliography';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import { api } from '../api/client';
|
||||
import type { GostResultData } from '../types';
|
||||
|
||||
export function Bibliography() {
|
||||
const { sources, removeSource, clearSources } = useBibliographyStore();
|
||||
const { isAuthenticated } = useAuthStore();
|
||||
const [style, setStyle] = useState<'7.1' | '7.0.5'>('7.1');
|
||||
const [formattedResult, setFormattedResult] = useState<GostResultData | null>(null);
|
||||
|
||||
const formatMutation = useMutation({
|
||||
mutationFn: async () => {
|
||||
// Создать задачу GOST форматирования
|
||||
const task = await api.post('/tasks/', {
|
||||
type: 'gost',
|
||||
input_data: {
|
||||
doc_ids: sources.map((s) => s.id),
|
||||
style,
|
||||
},
|
||||
});
|
||||
const taskId = task.data.id;
|
||||
|
||||
// Диспатчить задачу
|
||||
await api.post('/gost/format', {
|
||||
task_id: taskId,
|
||||
doc_ids: sources.map((s) => s.id),
|
||||
style,
|
||||
});
|
||||
|
||||
// Простое форматирование прямо на клиенте (используем ГОСТ-цитаты из источников)
|
||||
return {
|
||||
bibliography: sources.map((s, i) => ({
|
||||
number: i + 1,
|
||||
citation: s.gost_citation,
|
||||
doc_id: s.id,
|
||||
})),
|
||||
style,
|
||||
total: sources.length,
|
||||
} as GostResultData;
|
||||
},
|
||||
onSuccess: (data) => {
|
||||
setFormattedResult(data);
|
||||
toast.success('Библиография отформатирована');
|
||||
},
|
||||
onError: () => {
|
||||
// При ошибке используем локальные ГОСТ-цитаты
|
||||
setFormattedResult({
|
||||
bibliography: sources.map((s, i) => ({
|
||||
number: i + 1,
|
||||
citation: s.gost_citation,
|
||||
doc_id: s.id,
|
||||
})),
|
||||
style,
|
||||
total: sources.length,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const handleFormat = () => {
|
||||
if (sources.length === 0) {
|
||||
toast.error('Добавьте источники в библиографию');
|
||||
return;
|
||||
}
|
||||
formatMutation.mutate();
|
||||
};
|
||||
|
||||
const handleCopyAll = () => {
|
||||
if (!formattedResult) return;
|
||||
const text = formattedResult.bibliography
|
||||
.map((e) => `${e.number}. ${e.citation}`)
|
||||
.join('\n');
|
||||
navigator.clipboard.writeText(text).then(() => {
|
||||
toast.success('Список литературы скопирован');
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-3xl mx-auto space-y-6">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900 mb-2">Список литературы</h1>
|
||||
<p className="text-gray-500">
|
||||
Добавляйте источники из результатов поиска и форматируйте библиографию по ГОСТ.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Источники в списке */}
|
||||
{sources.length > 0 ? (
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<div className="flex items-center justify-between px-5 py-3 border-b border-gray-100">
|
||||
<span className="text-sm font-medium text-gray-700">
|
||||
Источников: {sources.length}
|
||||
</span>
|
||||
<button
|
||||
onClick={clearSources}
|
||||
className="text-xs text-red-500 hover:text-red-700 flex items-center gap-1"
|
||||
>
|
||||
<Trash2 className="w-3 h-3" />
|
||||
Очистить всё
|
||||
</button>
|
||||
</div>
|
||||
<div className="divide-y divide-gray-100">
|
||||
{sources.map((source) => (
|
||||
<div key={source.id} className="flex items-start gap-3 px-5 py-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-medium text-gray-800 truncate">{source.title}</p>
|
||||
<p className="text-xs text-gray-400 mt-0.5">
|
||||
{source.authors.slice(0, 2).map((a) => a.last_name).join(', ')}
|
||||
{source.year && ` · ${source.year}`}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => removeSource(source.id)}
|
||||
className="p-1 text-gray-300 hover:text-red-400 transition-colors flex-shrink-0"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="bg-gray-50 rounded-xl border border-dashed border-gray-200 p-10 text-center">
|
||||
<BookMarked className="w-10 h-10 text-gray-300 mx-auto mb-3" />
|
||||
<p className="text-gray-400 text-sm">
|
||||
Нажмите «В библиографию» на любом источнике из результатов поиска
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Настройки форматирования */}
|
||||
{sources.length > 0 && (
|
||||
<div className="flex items-center gap-4">
|
||||
<div>
|
||||
<label className="text-sm text-gray-500 mr-2">Стиль ГОСТ:</label>
|
||||
<select
|
||||
value={style}
|
||||
onChange={(e) => {
|
||||
setStyle(e.target.value as '7.1' | '7.0.5');
|
||||
setFormattedResult(null);
|
||||
}}
|
||||
className="text-sm border border-gray-200 rounded-lg px-3 py-1.5 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
>
|
||||
<option value="7.1">ГОСТ 7.1-2003 (полная)</option>
|
||||
<option value="7.0.5">ГОСТ Р 7.0.5-2008 (краткая)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onClick={handleFormat}
|
||||
disabled={formatMutation.isPending}
|
||||
className="flex items-center gap-2 px-5 py-2.5 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700 disabled:opacity-50 transition-colors ml-auto"
|
||||
>
|
||||
<BookOpen className="w-4 h-4" />
|
||||
{formatMutation.isPending ? 'Форматирование...' : 'Сформировать список'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Результат */}
|
||||
{formattedResult && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<div className="flex items-center justify-between px-5 py-3 border-b border-gray-100">
|
||||
<span className="text-sm font-medium text-gray-700">
|
||||
Список литературы · ГОСТ {style}
|
||||
</span>
|
||||
<button
|
||||
onClick={handleCopyAll}
|
||||
className="flex items-center gap-1.5 text-sm text-brand-600 hover:text-brand-700"
|
||||
>
|
||||
<Copy className="w-4 h-4" />
|
||||
Копировать всё
|
||||
</button>
|
||||
</div>
|
||||
<div className="px-2 py-2 divide-y divide-gray-50">
|
||||
{formattedResult.bibliography.map((entry) => (
|
||||
<GostCitation key={entry.doc_id} citation={entry.citation} number={entry.number} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
132
services/frontend/src/pages/Cabinet.tsx
Normal file
132
services/frontend/src/pages/Cabinet.tsx
Normal file
@@ -0,0 +1,132 @@
|
||||
import React from 'react';
|
||||
import { Navigate } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Crown, Search, Shield, BookOpen } from 'lucide-react';
|
||||
import { TaskCard } from '../components/TaskCard';
|
||||
import { tasksApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import { PLAN_LIMITS, type Task } from '../types';
|
||||
|
||||
const PLAN_BADGE_STYLES = {
|
||||
free: 'bg-gray-100 text-gray-600',
|
||||
student: 'bg-blue-100 text-blue-700',
|
||||
premium: 'bg-purple-100 text-purple-700',
|
||||
science: 'bg-amber-100 text-amber-700',
|
||||
};
|
||||
|
||||
export function Cabinet() {
|
||||
const { isAuthenticated, user } = useAuthStore();
|
||||
|
||||
if (!isAuthenticated) {
|
||||
return <Navigate to="/login" replace />;
|
||||
}
|
||||
|
||||
const { data: tasks, isLoading } = useQuery({
|
||||
queryKey: ['tasks'],
|
||||
queryFn: () => tasksApi.list().then((r) => r.data as Task[]),
|
||||
refetchInterval: 10000,
|
||||
});
|
||||
|
||||
const plan = user?.plan || 'free';
|
||||
const planInfo = PLAN_LIMITS[plan as keyof typeof PLAN_LIMITS];
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
{/* Профиль */}
|
||||
<div className="bg-white rounded-2xl border border-gray-100 p-6">
|
||||
<div className="flex items-start gap-4">
|
||||
<div className="w-16 h-16 bg-brand-100 rounded-2xl flex items-center justify-center flex-shrink-0">
|
||||
<span className="text-2xl font-bold text-brand-700">
|
||||
{user?.name?.[0]?.toUpperCase() || 'U'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<h2 className="text-xl font-semibold text-gray-900">{user?.name}</h2>
|
||||
<span className={`px-2 py-0.5 rounded-full text-xs font-medium ${PLAN_BADGE_STYLES[plan as keyof typeof PLAN_BADGE_STYLES]}`}>
|
||||
{planInfo.name}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-gray-500 text-sm">{user?.email}</p>
|
||||
</div>
|
||||
<button className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors">
|
||||
Обновить план
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Лимиты */}
|
||||
<div className="grid grid-cols-2 sm:grid-cols-4 gap-4">
|
||||
{[
|
||||
{
|
||||
icon: Search,
|
||||
label: 'Поисков/день',
|
||||
value: planInfo.search_per_day,
|
||||
color: 'text-blue-600',
|
||||
bg: 'bg-blue-50',
|
||||
},
|
||||
{
|
||||
icon: Shield,
|
||||
label: 'Проверок/мес',
|
||||
value: planInfo.plagiarism_per_month,
|
||||
color: 'text-purple-600',
|
||||
bg: 'bg-purple-50',
|
||||
},
|
||||
{
|
||||
icon: BookOpen,
|
||||
label: 'Изложений/мес',
|
||||
value: planInfo.summarize_per_month,
|
||||
color: 'text-emerald-600',
|
||||
bg: 'bg-emerald-50',
|
||||
},
|
||||
{
|
||||
icon: Crown,
|
||||
label: 'Одновременно',
|
||||
value: planInfo.concurrent,
|
||||
color: 'text-amber-600',
|
||||
bg: 'bg-amber-50',
|
||||
},
|
||||
].map(({ icon: Icon, label, value, color, bg }) => (
|
||||
<div key={label} className="bg-white rounded-xl border border-gray-100 p-4">
|
||||
<div className={`w-8 h-8 ${bg} rounded-lg flex items-center justify-center mb-2`}>
|
||||
<Icon className={`w-4 h-4 ${color}`} />
|
||||
</div>
|
||||
<div className="text-xl font-bold text-gray-900">
|
||||
{value === null ? '∞' : value}
|
||||
</div>
|
||||
<div className="text-xs text-gray-500">{label}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Задачи */}
|
||||
<div>
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">
|
||||
Мои задачи {tasks && `(${tasks.length})`}
|
||||
</h3>
|
||||
|
||||
{isLoading && (
|
||||
<div className="space-y-3">
|
||||
{[1, 2, 3].map((i) => (
|
||||
<div key={i} className="h-20 bg-gray-100 rounded-xl animate-pulse" />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tasks && tasks.length > 0 && (
|
||||
<div className="space-y-2">
|
||||
{tasks.map((task) => (
|
||||
<TaskCard key={task.id} task={task} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tasks && tasks.length === 0 && (
|
||||
<div className="text-center py-12 text-gray-400">
|
||||
<p>Задач пока нет. Начните с поиска источников!</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
137
services/frontend/src/pages/Check.tsx
Normal file
137
services/frontend/src/pages/Check.tsx
Normal file
@@ -0,0 +1,137 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { Shield, LayoutDashboard } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { DropZone } from '../components/DropZone';
|
||||
import { documentsApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import type { Task } from '../types';
|
||||
|
||||
export function Check() {
|
||||
const { isAuthenticated } = useAuthStore();
|
||||
const navigate = useNavigate();
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [submittedTask, setSubmittedTask] = useState<Task | null>(null);
|
||||
|
||||
const upload = useMutation({
|
||||
mutationFn: () => documentsApi.uploadForCheck(file!),
|
||||
onSuccess: (response) => {
|
||||
const task: Task = response.data;
|
||||
setSubmittedTask(task);
|
||||
toast.success('Файл загружен, проверка начата!');
|
||||
},
|
||||
onError: (error: any) => {
|
||||
const msg = error.response?.data?.detail || 'Ошибка загрузки';
|
||||
if (error.response?.status === 401) {
|
||||
toast.error('Войдите для проверки плагиата');
|
||||
navigate('/login');
|
||||
} else {
|
||||
toast.error(msg);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!file) return;
|
||||
if (!isAuthenticated) {
|
||||
toast.error('Необходима авторизация');
|
||||
navigate('/login');
|
||||
return;
|
||||
}
|
||||
upload.mutate();
|
||||
};
|
||||
|
||||
if (submittedTask) {
|
||||
return (
|
||||
<div className="max-w-xl mx-auto pt-8">
|
||||
<div className="bg-white rounded-2xl border border-gray-100 p-8 text-center">
|
||||
<div className="w-16 h-16 bg-green-50 rounded-2xl flex items-center justify-center mx-auto mb-4">
|
||||
<Shield className="w-8 h-8 text-green-500" />
|
||||
</div>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">Проверка запущена</h2>
|
||||
<p className="text-gray-500 mb-2">
|
||||
Файл <strong>{(submittedTask.input_data as any).filename}</strong> передан на проверку.
|
||||
</p>
|
||||
<p className="text-sm text-gray-400 mb-6">
|
||||
Вы получите email когда проверка завершится. Обычно это занимает 1-3 минуты.
|
||||
</p>
|
||||
<div className="flex flex-col sm:flex-row gap-3 justify-center">
|
||||
<Link
|
||||
to={`/tasks/${submittedTask.id}`}
|
||||
className="flex items-center justify-center gap-2 px-5 py-2.5 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700 transition-colors"
|
||||
>
|
||||
Следить за прогрессом
|
||||
</Link>
|
||||
<Link
|
||||
to="/cabinet"
|
||||
className="flex items-center justify-center gap-2 px-5 py-2.5 border border-gray-200 text-gray-600 rounded-lg text-sm font-medium hover:bg-gray-50 transition-colors"
|
||||
>
|
||||
<LayoutDashboard className="w-4 h-4" />
|
||||
Личный кабинет
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto">
|
||||
<div className="mb-8">
|
||||
<h1 className="text-2xl font-bold text-gray-900 mb-2">Проверка плагиата</h1>
|
||||
<p className="text-gray-500">
|
||||
Загрузите документ для проверки на 4 уровнях: точные совпадения, нечёткий поиск,
|
||||
семантика и LLM-анализ парафраза.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleSubmit} className="space-y-6">
|
||||
<DropZone
|
||||
onFile={setFile}
|
||||
file={file}
|
||||
onClear={() => setFile(null)}
|
||||
/>
|
||||
|
||||
{!isAuthenticated && (
|
||||
<div className="bg-amber-50 border border-amber-200 rounded-xl p-4 text-sm text-amber-700">
|
||||
<Link to="/login" className="underline font-medium">Войдите</Link> или{' '}
|
||||
<Link to="/register" className="underline font-medium">зарегистрируйтесь</Link>{' '}
|
||||
для проверки плагиата
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={!file || upload.isPending || !isAuthenticated}
|
||||
className="w-full flex items-center justify-center gap-2 py-3 bg-brand-600 text-white rounded-xl font-medium hover:bg-brand-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{upload.isPending ? (
|
||||
<>
|
||||
<Shield className="w-5 h-5 animate-pulse" />
|
||||
Загрузка...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Shield className="w-5 h-5" />
|
||||
Проверить на плагиат
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{/* Описание уровней */}
|
||||
<div className="mt-8 bg-gray-50 rounded-xl p-5">
|
||||
<h3 className="text-sm font-semibold text-gray-700 mb-3">4 уровня проверки:</h3>
|
||||
<div className="space-y-2 text-sm text-gray-500">
|
||||
<div>1. <strong>Winnowing + MinHash</strong> — точные и нечёткие совпадения (~мс)</div>
|
||||
<div>2. <strong>n-граммы + Jaccard</strong> — перестановки слов (~сек)</div>
|
||||
<div>3. <strong>FAISS GPU cosine</strong> — семантическая близость (~мс)</div>
|
||||
<div>4. <strong>Ollama Llama3</strong> — анализ парафраза (~2 сек)</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
98
services/frontend/src/pages/Home.tsx
Normal file
98
services/frontend/src/pages/Home.tsx
Normal file
@@ -0,0 +1,98 @@
|
||||
import React from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { Search, Shield, BookOpen, Zap } from 'lucide-react';
|
||||
import { SearchBar } from '../components/SearchBar';
|
||||
|
||||
const FEATURES = [
|
||||
{
|
||||
icon: Search,
|
||||
title: 'Семантический поиск',
|
||||
description: 'Поиск по миллионам статей через FAISS GPU + Elasticsearch BM25. Находит близкие по смыслу источники, а не только по ключевым словам.',
|
||||
color: 'text-blue-600',
|
||||
bg: 'bg-blue-50',
|
||||
},
|
||||
{
|
||||
icon: Shield,
|
||||
title: 'Проверка плагиата',
|
||||
description: '4 уровня проверки: Winnowing, MinHash, семантическое сравнение, LLM-анализ парафраза. Обнаружит даже перефразированный плагиат.',
|
||||
color: 'text-purple-600',
|
||||
bg: 'bg-purple-50',
|
||||
},
|
||||
{
|
||||
icon: BookOpen,
|
||||
title: 'ГОСТ-библиография',
|
||||
description: 'Автоматическое форматирование по ГОСТ 7.1-2003 и ГОСТ Р 7.0.5-2008. Правильный порядок и разделители.',
|
||||
color: 'text-emerald-600',
|
||||
bg: 'bg-emerald-50',
|
||||
},
|
||||
{
|
||||
icon: Zap,
|
||||
title: 'Асинхронно',
|
||||
description: 'Закройте браузер — получите email, когда результат готов. Очередь задач, real-time обновления через WebSocket.',
|
||||
color: 'text-orange-600',
|
||||
bg: 'bg-orange-50',
|
||||
},
|
||||
];
|
||||
|
||||
export function Home() {
|
||||
const navigate = useNavigate();
|
||||
|
||||
const handleSearch = (query: string) => {
|
||||
navigate(`/search?q=${encodeURIComponent(query)}`);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-16">
|
||||
{/* Hero */}
|
||||
<section className="text-center pt-12 pb-4">
|
||||
<h1 className="text-4xl sm:text-5xl font-bold text-gray-900 mb-4 leading-tight">
|
||||
Академический помощник
|
||||
</h1>
|
||||
<p className="text-lg text-gray-500 mb-10 max-w-2xl mx-auto">
|
||||
Поиск научных источников, проверка плагиата и ГОСТ-библиография — всё в одном месте.
|
||||
Введите тему и система найдёт релевантные источники автоматически.
|
||||
</p>
|
||||
|
||||
<div className="max-w-2xl mx-auto">
|
||||
<SearchBar
|
||||
onSearch={handleSearch}
|
||||
size="lg"
|
||||
placeholder="Введите тему исследования или научный вопрос..."
|
||||
/>
|
||||
</div>
|
||||
|
||||
<p className="text-sm text-gray-400 mt-4">
|
||||
Например: «нейронные сети в обработке естественного языка» или «квантовые вычисления алгоритмы»
|
||||
</p>
|
||||
</section>
|
||||
|
||||
{/* Фичи */}
|
||||
<section>
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-6">
|
||||
{FEATURES.map(({ icon: Icon, title, description, color, bg }) => (
|
||||
<div key={title} className="bg-white rounded-xl p-6 border border-gray-100 hover:shadow-md transition-shadow">
|
||||
<div className={`w-12 h-12 ${bg} rounded-xl flex items-center justify-center mb-4`}>
|
||||
<Icon className={`w-6 h-6 ${color}`} />
|
||||
</div>
|
||||
<h3 className="font-semibold text-gray-900 mb-2">{title}</h3>
|
||||
<p className="text-sm text-gray-500 leading-relaxed">{description}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* Источники */}
|
||||
<section className="bg-white rounded-2xl border border-gray-100 p-8">
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">Источники данных</h2>
|
||||
<p className="text-gray-500 text-sm mb-6">Поиск ведётся по открытым академическим базам данных</p>
|
||||
<div className="flex flex-wrap gap-3">
|
||||
{['OpenAlex', 'КиберЛенинка', 'arXiv', 'Wikipedia RU', 'Wikipedia EN'].map((name) => (
|
||||
<span key={name} className="px-4 py-2 bg-gray-50 text-gray-600 rounded-lg text-sm font-medium border border-gray-100">
|
||||
{name}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
90
services/frontend/src/pages/Login.tsx
Normal file
90
services/frontend/src/pages/Login.tsx
Normal file
@@ -0,0 +1,90 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Link, useNavigate } from 'react-router-dom';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { GraduationCap } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { authApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import type { TokenResponse } from '../types';
|
||||
|
||||
export function Login() {
|
||||
const navigate = useNavigate();
|
||||
const { setAuth } = useAuthStore();
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
|
||||
const login = useMutation({
|
||||
mutationFn: () => authApi.login({ email, password }),
|
||||
onSuccess: (response) => {
|
||||
const data: TokenResponse = response.data;
|
||||
setAuth(data.user, data.access_token);
|
||||
toast.success(`Добро пожаловать, ${data.user.name}!`);
|
||||
navigate('/cabinet');
|
||||
},
|
||||
onError: (error: any) => {
|
||||
const msg = error.response?.data?.detail || 'Ошибка входа';
|
||||
toast.error(msg);
|
||||
},
|
||||
});
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
login.mutate();
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-md mx-auto pt-8">
|
||||
<div className="bg-white rounded-2xl border border-gray-100 p-8">
|
||||
<div className="flex justify-center mb-6">
|
||||
<div className="p-3 bg-brand-600 rounded-xl">
|
||||
<GraduationCap className="w-7 h-7 text-white" />
|
||||
</div>
|
||||
</div>
|
||||
<h1 className="text-2xl font-bold text-gray-900 text-center mb-1">Вход</h1>
|
||||
<p className="text-gray-400 text-center text-sm mb-6">Войдите в свой аккаунт</p>
|
||||
|
||||
<form onSubmit={handleSubmit} className="space-y-4">
|
||||
<div>
|
||||
<label className="text-sm font-medium text-gray-700 block mb-1">Email</label>
|
||||
<input
|
||||
type="email"
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
required
|
||||
placeholder="ivan@example.com"
|
||||
className="w-full border border-gray-200 rounded-xl px-4 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="text-sm font-medium text-gray-700 block mb-1">Пароль</label>
|
||||
<input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
required
|
||||
minLength={8}
|
||||
placeholder="••••••••"
|
||||
className="w-full border border-gray-200 rounded-xl px-4 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={login.isPending}
|
||||
className="w-full py-2.5 bg-brand-600 text-white rounded-xl text-sm font-medium hover:bg-brand-700 disabled:opacity-50 transition-colors"
|
||||
>
|
||||
{login.isPending ? 'Входим...' : 'Войти'}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<p className="text-center text-sm text-gray-400 mt-6">
|
||||
Нет аккаунта?{' '}
|
||||
<Link to="/register" className="text-brand-600 hover:underline font-medium">
|
||||
Зарегистрироваться
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
122
services/frontend/src/pages/Pricing.tsx
Normal file
122
services/frontend/src/pages/Pricing.tsx
Normal file
@@ -0,0 +1,122 @@
|
||||
import React from 'react';
|
||||
import { Check } from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import { PLAN_LIMITS } from '../types';
|
||||
|
||||
const PLAN_ORDER = ['free', 'student', 'premium', 'science'] as const;
|
||||
|
||||
const PLAN_FEATURES = {
|
||||
free: [
|
||||
'10 поисков в день',
|
||||
'3 краткие изложения в месяц',
|
||||
'1 проверка плагиата в месяц',
|
||||
'1 задача одновременно',
|
||||
'ГОСТ библиография',
|
||||
],
|
||||
student: [
|
||||
'Безлимитный поиск',
|
||||
'30 кратких изложений в месяц',
|
||||
'10 проверок плагиата в месяц',
|
||||
'2 задачи одновременно',
|
||||
'ГОСТ библиография',
|
||||
'Email уведомления',
|
||||
],
|
||||
premium: [
|
||||
'Безлимитный поиск',
|
||||
'Безлимитные изложения',
|
||||
'50 проверок плагиата в месяц',
|
||||
'5 задач одновременно',
|
||||
'ГОСТ библиография',
|
||||
'Приоритетная очередь',
|
||||
],
|
||||
science: [
|
||||
'Безлимитный поиск',
|
||||
'Безлимитные изложения',
|
||||
'Безлимитные проверки плагиата',
|
||||
'10 задач одновременно',
|
||||
'ГОСТ библиография',
|
||||
'Максимальный приоритет',
|
||||
'API доступ',
|
||||
],
|
||||
};
|
||||
|
||||
const POPULAR_PLAN = 'student';
|
||||
|
||||
export function Pricing() {
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div className="text-center">
|
||||
<h1 className="text-3xl font-bold text-gray-900 mb-3">Тарифные планы</h1>
|
||||
<p className="text-gray-500 max-w-xl mx-auto">
|
||||
Начните бесплатно. Обновите план для расширенных возможностей.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-6">
|
||||
{PLAN_ORDER.map((planKey) => {
|
||||
const plan = PLAN_LIMITS[planKey];
|
||||
const features = PLAN_FEATURES[planKey];
|
||||
const isPopular = planKey === POPULAR_PLAN;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={planKey}
|
||||
className={clsx(
|
||||
'relative bg-white rounded-2xl border-2 p-6 flex flex-col',
|
||||
isPopular ? 'border-brand-500 shadow-lg shadow-brand-100' : 'border-gray-100'
|
||||
)}
|
||||
>
|
||||
{isPopular && (
|
||||
<div className="absolute -top-3 left-1/2 -translate-x-1/2">
|
||||
<span className="px-3 py-1 bg-brand-600 text-white text-xs font-medium rounded-full">
|
||||
Популярный
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mb-4">
|
||||
<h3 className="text-base font-semibold text-gray-900">{plan.name}</h3>
|
||||
<div className="mt-2">
|
||||
{plan.price === 0 ? (
|
||||
<span className="text-3xl font-bold text-gray-900">Бесплатно</span>
|
||||
) : (
|
||||
<>
|
||||
<span className="text-3xl font-bold text-gray-900">{plan.price}₽</span>
|
||||
<span className="text-gray-400 text-sm">/мес</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ul className="space-y-2.5 flex-1 mb-6">
|
||||
{features.map((feature) => (
|
||||
<li key={feature} className="flex items-start gap-2">
|
||||
<Check className="w-4 h-4 text-green-500 flex-shrink-0 mt-0.5" />
|
||||
<span className="text-sm text-gray-600">{feature}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
<button
|
||||
className={clsx(
|
||||
'w-full py-2.5 rounded-xl text-sm font-medium transition-colors',
|
||||
isPopular
|
||||
? 'bg-brand-600 text-white hover:bg-brand-700'
|
||||
: plan.price === 0
|
||||
? 'bg-gray-100 text-gray-700 hover:bg-gray-200'
|
||||
: 'border border-brand-200 text-brand-700 hover:bg-brand-50'
|
||||
)}
|
||||
>
|
||||
{plan.price === 0 ? 'Начать бесплатно' : 'Выбрать план'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div className="bg-gray-50 rounded-xl p-6 text-center text-sm text-gray-500">
|
||||
Оплата через российские системы. При вопросах пишите на noreply@jze9.ru
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
104
services/frontend/src/pages/Register.tsx
Normal file
104
services/frontend/src/pages/Register.tsx
Normal file
@@ -0,0 +1,104 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Link, useNavigate } from 'react-router-dom';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { GraduationCap } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { authApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import type { TokenResponse } from '../types';
|
||||
|
||||
export function Register() {
|
||||
const navigate = useNavigate();
|
||||
const { setAuth } = useAuthStore();
|
||||
const [name, setName] = useState('');
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
|
||||
const register = useMutation({
|
||||
mutationFn: () => authApi.register({ name, email, password }),
|
||||
onSuccess: (response) => {
|
||||
const data: TokenResponse = response.data;
|
||||
setAuth(data.user, data.access_token);
|
||||
toast.success('Аккаунт создан! Проверьте email для подтверждения.');
|
||||
navigate('/cabinet');
|
||||
},
|
||||
onError: (error: any) => {
|
||||
const msg = error.response?.data?.detail || 'Ошибка регистрации';
|
||||
toast.error(msg);
|
||||
},
|
||||
});
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
register.mutate();
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-md mx-auto pt-8">
|
||||
<div className="bg-white rounded-2xl border border-gray-100 p-8">
|
||||
<div className="flex justify-center mb-6">
|
||||
<div className="p-3 bg-brand-600 rounded-xl">
|
||||
<GraduationCap className="w-7 h-7 text-white" />
|
||||
</div>
|
||||
</div>
|
||||
<h1 className="text-2xl font-bold text-gray-900 text-center mb-1">Регистрация</h1>
|
||||
<p className="text-gray-400 text-center text-sm mb-6">Создайте аккаунт, это бесплатно</p>
|
||||
|
||||
<form onSubmit={handleSubmit} className="space-y-4">
|
||||
<div>
|
||||
<label className="text-sm font-medium text-gray-700 block mb-1">Имя</label>
|
||||
<input
|
||||
type="text"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
required
|
||||
minLength={2}
|
||||
placeholder="Иван Иванов"
|
||||
className="w-full border border-gray-200 rounded-xl px-4 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="text-sm font-medium text-gray-700 block mb-1">Email</label>
|
||||
<input
|
||||
type="email"
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
required
|
||||
placeholder="ivan@example.com"
|
||||
className="w-full border border-gray-200 rounded-xl px-4 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="text-sm font-medium text-gray-700 block mb-1">Пароль</label>
|
||||
<input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
required
|
||||
minLength={8}
|
||||
placeholder="Минимум 8 символов"
|
||||
className="w-full border border-gray-200 rounded-xl px-4 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={register.isPending}
|
||||
className="w-full py-2.5 bg-brand-600 text-white rounded-xl text-sm font-medium hover:bg-brand-700 disabled:opacity-50 transition-colors"
|
||||
>
|
||||
{register.isPending ? 'Создаём...' : 'Создать аккаунт'}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<p className="text-center text-sm text-gray-400 mt-6">
|
||||
Уже есть аккаунт?{' '}
|
||||
<Link to="/login" className="text-brand-600 hover:underline font-medium">
|
||||
Войти
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
201
services/frontend/src/pages/Search.tsx
Normal file
201
services/frontend/src/pages/Search.tsx
Normal file
@@ -0,0 +1,201 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { useNavigate, useSearchParams } from 'react-router-dom';
|
||||
import { useQuery, useMutation } from '@tanstack/react-query';
|
||||
import { Filter, Loader2 } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { SearchBar } from '../components/SearchBar';
|
||||
import { SourceCard } from '../components/SourceCard';
|
||||
import { StatusBadge } from '../components/StatusBadge';
|
||||
import { searchApi, tasksApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import { useTaskWebSocket } from '../hooks/useTaskPolling';
|
||||
import type { Task, SearchResultData } from '../types';
|
||||
|
||||
export function Search() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const navigate = useNavigate();
|
||||
const { isAuthenticated } = useAuthStore();
|
||||
|
||||
const query = searchParams.get('q') || '';
|
||||
const [taskId, setTaskId] = useState<string | null>(null);
|
||||
const [lang, setLang] = useState('');
|
||||
const [yearFrom, setYearFrom] = useState('');
|
||||
const [yearTo, setYearTo] = useState('');
|
||||
|
||||
// Создать задачу поиска
|
||||
const createSearch = useMutation({
|
||||
mutationFn: searchApi.create,
|
||||
onSuccess: (response) => {
|
||||
const task: Task = response.data;
|
||||
setTaskId(task.id);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
const msg = error.response?.data?.detail || 'Ошибка поиска';
|
||||
if (error.response?.status === 401) {
|
||||
toast.error('Войдите, чтобы выполнять поиск');
|
||||
navigate('/login');
|
||||
} else {
|
||||
toast.error(msg);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Поллинг задачи
|
||||
const {
|
||||
data: task,
|
||||
isLoading: isPolling,
|
||||
} = useQuery({
|
||||
queryKey: ['task', taskId],
|
||||
queryFn: () => tasksApi.get(taskId!).then((r) => r.data as Task),
|
||||
enabled: !!taskId,
|
||||
refetchInterval: (data) => {
|
||||
if (!data) return 3000;
|
||||
if (data.status === 'done' || data.status === 'failed') return false;
|
||||
return 3000;
|
||||
},
|
||||
});
|
||||
|
||||
// WebSocket для real-time обновлений
|
||||
useTaskWebSocket(
|
||||
taskId ?? undefined,
|
||||
!!taskId && task?.status !== 'done' && task?.status !== 'failed'
|
||||
);
|
||||
|
||||
// Запустить поиск при изменении query
|
||||
useEffect(() => {
|
||||
if (query && query.length >= 3 && isAuthenticated) {
|
||||
createSearch.mutate({
|
||||
query,
|
||||
lang: lang || undefined,
|
||||
year_from: yearFrom ? parseInt(yearFrom) : undefined,
|
||||
year_to: yearTo ? parseInt(yearTo) : undefined,
|
||||
});
|
||||
}
|
||||
}, [query]);
|
||||
|
||||
const handleSearch = (newQuery: string) => {
|
||||
setTaskId(null);
|
||||
setSearchParams({ q: newQuery });
|
||||
};
|
||||
|
||||
const result = task?.result as SearchResultData | undefined;
|
||||
const isLoading = createSearch.isPending || (!!taskId && task?.status === 'queued') || task?.status === 'processing';
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Поисковая строка */}
|
||||
<SearchBar
|
||||
onSearch={handleSearch}
|
||||
defaultValue={query}
|
||||
isLoading={isLoading}
|
||||
/>
|
||||
|
||||
<div className="flex gap-6">
|
||||
{/* Боковая панель фильтров */}
|
||||
<aside className="w-56 flex-shrink-0 hidden lg:block">
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-4 sticky top-24">
|
||||
<div className="flex items-center gap-2 mb-4">
|
||||
<Filter className="w-4 h-4 text-gray-400" />
|
||||
<span className="text-sm font-medium text-gray-700">Фильтры</span>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
<div>
|
||||
<label className="text-xs text-gray-500 mb-1 block">Язык</label>
|
||||
<select
|
||||
value={lang}
|
||||
onChange={(e) => setLang(e.target.value)}
|
||||
className="w-full text-sm border border-gray-200 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
>
|
||||
<option value="">Все</option>
|
||||
<option value="ru">Русский</option>
|
||||
<option value="en">English</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="text-xs text-gray-500 mb-1 block">Год с</label>
|
||||
<input
|
||||
type="number"
|
||||
value={yearFrom}
|
||||
onChange={(e) => setYearFrom(e.target.value)}
|
||||
placeholder="2000"
|
||||
min={1900}
|
||||
max={2100}
|
||||
className="w-full text-sm border border-gray-200 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="text-xs text-gray-500 mb-1 block">Год по</label>
|
||||
<input
|
||||
type="number"
|
||||
value={yearTo}
|
||||
onChange={(e) => setYearTo(e.target.value)}
|
||||
placeholder="2024"
|
||||
min={1900}
|
||||
max={2100}
|
||||
className="w-full text-sm border border-gray-200 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* Результаты */}
|
||||
<main className="flex-1 min-w-0">
|
||||
{/* Статус задачи */}
|
||||
{task && task.status !== 'done' && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-6 text-center">
|
||||
<div className="flex items-center justify-center gap-3 mb-2">
|
||||
<Loader2 className="w-5 h-5 text-brand-500 animate-spin" />
|
||||
<StatusBadge status={task.status} />
|
||||
</div>
|
||||
{task.queue_position && (
|
||||
<p className="text-sm text-gray-500">Позиция в очереди: {task.queue_position}</p>
|
||||
)}
|
||||
{task.eta_seconds && (
|
||||
<p className="text-sm text-gray-400">Ожидаемое время: ~{task.eta_seconds} сек</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Результаты поиска */}
|
||||
{task?.status === 'done' && result && (
|
||||
<div className="space-y-4">
|
||||
<p className="text-sm text-gray-500">
|
||||
Найдено: <strong>{result.total}</strong> источников
|
||||
</p>
|
||||
{result.sources.map((source) => (
|
||||
<SourceCard key={source.id} source={source} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Ошибка */}
|
||||
{task?.status === 'failed' && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-xl p-6 text-center">
|
||||
<p className="text-red-700 font-medium">Ошибка поиска</p>
|
||||
<p className="text-red-500 text-sm mt-1">{task.error}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Пустое состояние */}
|
||||
{!query && !task && (
|
||||
<div className="text-center py-16 text-gray-400">
|
||||
<p>Введите запрос для поиска источников</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Не авторизован */}
|
||||
{query && !isAuthenticated && (
|
||||
<div className="bg-brand-50 border border-brand-200 rounded-xl p-6 text-center">
|
||||
<p className="text-brand-700 font-medium mb-2">Необходима авторизация</p>
|
||||
<p className="text-brand-600 text-sm">Войдите или зарегистрируйтесь для выполнения поиска</p>
|
||||
</div>
|
||||
)}
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
141
services/frontend/src/pages/Task.tsx
Normal file
141
services/frontend/src/pages/Task.tsx
Normal file
@@ -0,0 +1,141 @@
|
||||
import React from 'react';
|
||||
import { useParams, Navigate } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Loader2, ArrowLeft } from 'lucide-react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { StatusBadge } from '../components/StatusBadge';
|
||||
import { SourceCard } from '../components/SourceCard';
|
||||
import { PlagiarismReport } from '../components/PlagiarismReport';
|
||||
import { GostCitation } from '../components/GostCitation';
|
||||
import { tasksApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import { useTaskWebSocket } from '../hooks/useTaskPolling';
|
||||
import type { Task as TaskType, SearchResultData, PlagiarismResultData, GostResultData } from '../types';
|
||||
|
||||
export function Task() {
|
||||
const { taskId } = useParams<{ taskId: string }>();
|
||||
const { isAuthenticated } = useAuthStore();
|
||||
|
||||
if (!isAuthenticated) return <Navigate to="/login" replace />;
|
||||
if (!taskId) return <Navigate to="/cabinet" replace />;
|
||||
|
||||
const { data: task, isLoading } = useQuery({
|
||||
queryKey: ['task', taskId],
|
||||
queryFn: () => tasksApi.get(taskId).then((r) => r.data as TaskType),
|
||||
refetchInterval: (data) => {
|
||||
if (!data) return 3000;
|
||||
return (data.status === 'done' || data.status === 'failed') ? false : 3000;
|
||||
},
|
||||
});
|
||||
|
||||
useTaskWebSocket(taskId, !!task && task.status !== 'done' && task.status !== 'failed');
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center py-16">
|
||||
<Loader2 className="w-8 h-8 text-brand-500 animate-spin" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!task) {
|
||||
return <Navigate to="/cabinet" replace />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Шапка */}
|
||||
<div className="flex items-center gap-3">
|
||||
<Link to="/cabinet" className="p-2 rounded-lg text-gray-400 hover:text-gray-600 hover:bg-gray-100 transition-colors">
|
||||
<ArrowLeft className="w-5 h-5" />
|
||||
</Link>
|
||||
<div>
|
||||
<h1 className="text-xl font-semibold text-gray-900 capitalize">
|
||||
{{
|
||||
search: 'Поиск источников',
|
||||
plagiarism: 'Проверка плагиата',
|
||||
gost: 'ГОСТ библиография',
|
||||
summarize: 'Краткое изложение',
|
||||
}[task.type] || task.type}
|
||||
</h1>
|
||||
<div className="flex items-center gap-2 mt-0.5">
|
||||
<StatusBadge status={task.status} />
|
||||
<span className="text-xs text-gray-400">{task.id}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* В процессе */}
|
||||
{(task.status === 'queued' || task.status === 'processing') && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-10 text-center">
|
||||
<Loader2 className="w-10 h-10 text-brand-400 animate-spin mx-auto mb-3" />
|
||||
<p className="text-base font-medium text-gray-700">
|
||||
{task.status === 'queued' ? 'Задача в очереди...' : 'Выполняется...'}
|
||||
</p>
|
||||
{task.queue_position && (
|
||||
<p className="text-sm text-gray-400 mt-1">Позиция: {task.queue_position}</p>
|
||||
)}
|
||||
<p className="text-sm text-gray-400 mt-1">Вы получите email когда задача завершится</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Ошибка */}
|
||||
{task.status === 'failed' && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-xl p-6">
|
||||
<p className="font-medium text-red-700 mb-1">Задача завершилась с ошибкой</p>
|
||||
<p className="text-sm text-red-500">{task.error}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Результаты поиска */}
|
||||
{task.status === 'done' && task.type === 'search' && task.result && (
|
||||
<div className="space-y-4">
|
||||
<p className="text-sm text-gray-500">
|
||||
Запрос: <strong>«{(task.input_data as any).query}»</strong>
|
||||
{' · '}{(task.result as SearchResultData).total} источников
|
||||
</p>
|
||||
{(task.result as SearchResultData).sources.map((source) => (
|
||||
<SourceCard key={source.id} source={source} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Отчёт о плагиате */}
|
||||
{task.status === 'done' && task.type === 'plagiarism' && task.result && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-6">
|
||||
<h2 className="text-base font-semibold text-gray-900 mb-4">
|
||||
Файл: {(task.input_data as any).filename}
|
||||
</h2>
|
||||
<PlagiarismReport data={task.result as PlagiarismResultData} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ГОСТ библиография */}
|
||||
{task.status === 'done' && task.type === 'gost' && task.result && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h2 className="text-base font-semibold text-gray-900">
|
||||
Список литературы (ГОСТ {(task.result as GostResultData).style}-2003)
|
||||
</h2>
|
||||
<button
|
||||
onClick={() => {
|
||||
const text = (task.result as GostResultData).bibliography
|
||||
.map((e) => `${e.number}. ${e.citation}`)
|
||||
.join('\n');
|
||||
navigator.clipboard.writeText(text);
|
||||
}}
|
||||
className="px-3 py-1.5 text-xs font-medium text-brand-600 border border-brand-200 rounded-lg hover:bg-brand-50 transition-colors"
|
||||
>
|
||||
Копировать всё
|
||||
</button>
|
||||
</div>
|
||||
<div className="divide-y divide-gray-100">
|
||||
{(task.result as GostResultData).bibliography.map((entry) => (
|
||||
<GostCitation key={entry.doc_id} citation={entry.citation} number={entry.number} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
44
services/frontend/src/store/auth.ts
Normal file
44
services/frontend/src/store/auth.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { create } from 'zustand';
|
||||
import { persist } from 'zustand/middleware';
|
||||
import type { User } from '../types';
|
||||
|
||||
interface AuthState {
|
||||
user: User | null;
|
||||
token: string | null;
|
||||
isAuthenticated: boolean;
|
||||
setAuth: (user: User, token: string) => void;
|
||||
updateUser: (user: Partial<User>) => void;
|
||||
logout: () => void;
|
||||
}
|
||||
|
||||
export const useAuthStore = create<AuthState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
user: null,
|
||||
token: null,
|
||||
isAuthenticated: false,
|
||||
|
||||
setAuth: (user, token) =>
|
||||
set({ user, token, isAuthenticated: true }),
|
||||
|
||||
updateUser: (updates) => {
|
||||
const current = get().user;
|
||||
if (current) {
|
||||
set({ user: { ...current, ...updates } });
|
||||
}
|
||||
},
|
||||
|
||||
logout: () =>
|
||||
set({ user: null, token: null, isAuthenticated: false }),
|
||||
}),
|
||||
{
|
||||
name: 'auth-storage',
|
||||
// Не сохранять методы в localStorage, только данные
|
||||
partialize: (state) => ({
|
||||
user: state.user,
|
||||
token: state.token,
|
||||
isAuthenticated: state.isAuthenticated,
|
||||
}),
|
||||
}
|
||||
)
|
||||
);
|
||||
39
services/frontend/src/store/bibliography.ts
Normal file
39
services/frontend/src/store/bibliography.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { create } from 'zustand';
|
||||
import { persist } from 'zustand/middleware';
|
||||
import type { SearchSource } from '../types';
|
||||
|
||||
interface BibliographyState {
|
||||
sources: SearchSource[];
|
||||
addSource: (source: SearchSource) => void;
|
||||
removeSource: (id: number) => void;
|
||||
clearSources: () => void;
|
||||
hasSource: (id: number) => boolean;
|
||||
}
|
||||
|
||||
export const useBibliographyStore = create<BibliographyState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
sources: [],
|
||||
|
||||
addSource: (source) => {
|
||||
const existing = get().sources.find((s) => s.id === source.id);
|
||||
if (!existing) {
|
||||
set((state) => ({ sources: [...state.sources, source] }));
|
||||
}
|
||||
},
|
||||
|
||||
removeSource: (id) => {
|
||||
set((state) => ({
|
||||
sources: state.sources.filter((s) => s.id !== id),
|
||||
}));
|
||||
},
|
||||
|
||||
clearSources: () => set({ sources: [] }),
|
||||
|
||||
hasSource: (id) => get().sources.some((s) => s.id === id),
|
||||
}),
|
||||
{
|
||||
name: 'bibliography-storage',
|
||||
}
|
||||
)
|
||||
);
|
||||
179
services/frontend/src/types/index.ts
Normal file
179
services/frontend/src/types/index.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
// ─── Типы данных для Академического помощника ─────────────────────────────────
|
||||
|
||||
export type TaskStatus = 'queued' | 'processing' | 'done' | 'failed';
|
||||
export type TaskType = 'search' | 'plagiarism' | 'summarize' | 'gost';
|
||||
export type UserPlan = 'free' | 'student' | 'premium' | 'science';
|
||||
|
||||
// ─── Пользователь ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface User {
|
||||
id: number;
|
||||
email: string;
|
||||
name: string;
|
||||
plan: UserPlan;
|
||||
is_verified: boolean;
|
||||
}
|
||||
|
||||
// ─── Источник в результатах поиска ────────────────────────────────────────────
|
||||
|
||||
export interface AuthorInfo {
|
||||
last_name: string;
|
||||
first_name?: string;
|
||||
initials?: string;
|
||||
}
|
||||
|
||||
export interface SearchSource {
|
||||
id: number;
|
||||
title: string;
|
||||
authors: AuthorInfo[];
|
||||
year: number | null;
|
||||
journal: string | null;
|
||||
abstract: string | null;
|
||||
url: string | null;
|
||||
doi: string | null;
|
||||
relevance_score: number;
|
||||
gost_citation: string;
|
||||
source_db: string;
|
||||
}
|
||||
|
||||
export interface SearchResultData {
|
||||
sources: SearchSource[];
|
||||
total: number;
|
||||
query: string;
|
||||
}
|
||||
|
||||
// ─── Отчёт о плагиате ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface PlagiarismMatch {
|
||||
fragment: string;
|
||||
position_start: number;
|
||||
position_end: number;
|
||||
similarity: number;
|
||||
method: 'exact' | 'fuzzy' | 'semantic+llm';
|
||||
confidence?: number;
|
||||
reason?: string;
|
||||
source_title: string;
|
||||
source_url: string | null;
|
||||
source_db: string;
|
||||
}
|
||||
|
||||
export interface PlagiarismResultData {
|
||||
overall_similarity: number;
|
||||
matches: PlagiarismMatch[];
|
||||
total_fragments: number;
|
||||
flagged_fragments: number;
|
||||
by_method?: {
|
||||
exact: number;
|
||||
fuzzy: number;
|
||||
semantic_llm: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Библиография ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface BibliographyEntry {
|
||||
number: number;
|
||||
citation: string;
|
||||
doc_id: number;
|
||||
}
|
||||
|
||||
export interface GostResultData {
|
||||
bibliography: BibliographyEntry[];
|
||||
style: string;
|
||||
total: number;
|
||||
}
|
||||
|
||||
// ─── Задача ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export type TaskResult = SearchResultData | PlagiarismResultData | GostResultData | null;
|
||||
|
||||
export interface Task {
|
||||
id: string;
|
||||
type: TaskType;
|
||||
status: TaskStatus;
|
||||
queue_position?: number;
|
||||
eta_seconds?: number;
|
||||
input_data: Record<string, unknown>;
|
||||
result?: TaskResult;
|
||||
error?: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// ─── API ответы ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface TokenResponse {
|
||||
access_token: string;
|
||||
token_type: string;
|
||||
user: User;
|
||||
}
|
||||
|
||||
export interface ApiError {
|
||||
detail: string;
|
||||
}
|
||||
|
||||
// ─── Запросы ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SearchRequest {
|
||||
query: string;
|
||||
lang?: string;
|
||||
year_from?: number;
|
||||
year_to?: number;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export interface RegisterRequest {
|
||||
email: string;
|
||||
password: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface LoginRequest {
|
||||
email: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
// ─── Лимиты тарифных планов ───────────────────────────────────────────────────
|
||||
|
||||
export interface PlanLimits {
|
||||
name: string;
|
||||
price: number;
|
||||
search_per_day: number | null;
|
||||
summarize_per_month: number | null;
|
||||
plagiarism_per_month: number | null;
|
||||
concurrent: number;
|
||||
}
|
||||
|
||||
export const PLAN_LIMITS: Record<UserPlan, PlanLimits> = {
|
||||
free: {
|
||||
name: 'Бесплатный',
|
||||
price: 0,
|
||||
search_per_day: 10,
|
||||
summarize_per_month: 3,
|
||||
plagiarism_per_month: 1,
|
||||
concurrent: 1,
|
||||
},
|
||||
student: {
|
||||
name: 'Студенческий',
|
||||
price: 199,
|
||||
search_per_day: null,
|
||||
summarize_per_month: 30,
|
||||
plagiarism_per_month: 10,
|
||||
concurrent: 2,
|
||||
},
|
||||
premium: {
|
||||
name: 'Премиум',
|
||||
price: 499,
|
||||
search_per_day: null,
|
||||
summarize_per_month: null,
|
||||
plagiarism_per_month: 50,
|
||||
concurrent: 5,
|
||||
},
|
||||
science: {
|
||||
name: 'Научный',
|
||||
price: 999,
|
||||
search_per_day: null,
|
||||
summarize_per_month: null,
|
||||
plagiarism_per_month: null,
|
||||
concurrent: 10,
|
||||
},
|
||||
};
|
||||
26
services/frontend/tailwind.config.js
Normal file
26
services/frontend/tailwind.config.js
Normal file
@@ -0,0 +1,26 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'],
|
||||
theme: {
|
||||
extend: {
|
||||
colors: {
|
||||
brand: {
|
||||
50: '#eff6ff',
|
||||
100: '#dbeafe',
|
||||
200: '#bfdbfe',
|
||||
300: '#93c5fd',
|
||||
400: '#60a5fa',
|
||||
500: '#3b82f6',
|
||||
600: '#2563eb',
|
||||
700: '#1d4ed8',
|
||||
800: '#1e40af',
|
||||
900: '#1e3a8a',
|
||||
},
|
||||
},
|
||||
animation: {
|
||||
'pulse-slow': 'pulse 3s cubic-bezier(0.4, 0, 0.6, 1) infinite',
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [],
|
||||
};
|
||||
25
services/frontend/tsconfig.json
Normal file
25
services/frontend/tsconfig.json
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [{ "path": "./tsconfig.node.json" }]
|
||||
}
|
||||
10
services/frontend/tsconfig.node.json
Normal file
10
services/frontend/tsconfig.node.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"skipLibCheck": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true
|
||||
},
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
23
services/frontend/vite.config.ts
Normal file
23
services/frontend/vite.config.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8000',
|
||||
changeOrigin: true,
|
||||
},
|
||||
'/ws': {
|
||||
target: 'ws://localhost:8000',
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
sourcemap: false,
|
||||
},
|
||||
});
|
||||
15
services/worker-gost/Dockerfile
Normal file
15
services/worker-gost/Dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["celery", "-A", "app.celery_app", "worker", "-Q", "queue.gost", "-c", "8", "-n", "gost@%h", "--loglevel=info"]
|
||||
0
services/worker-gost/app/__init__.py
Normal file
0
services/worker-gost/app/__init__.py
Normal file
27
services/worker-gost/app/celery_app.py
Normal file
27
services/worker-gost/app/celery_app.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""Celery приложение GOST воркера."""
|
||||
|
||||
from celery import Celery
|
||||
|
||||
from app.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"worker_gost",
|
||||
broker=settings.RABBITMQ_URL,
|
||||
backend=settings.REDIS_URL,
|
||||
include=["app.tasks.gost"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="Europe/Moscow",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
task_routes={
|
||||
"gost.*": {"queue": "queue.gost"},
|
||||
"notify.*": {"queue": "queue.notify"},
|
||||
},
|
||||
task_acks_late=True,
|
||||
result_expires=86400,
|
||||
)
|
||||
33
services/worker-gost/app/config.py
Normal file
33
services/worker-gost/app/config.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Конфигурация GOST воркера."""
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
POSTGRES_HOST: str = "postgres"
|
||||
POSTGRES_PORT: int = 5432
|
||||
POSTGRES_DB: str = "antiplagiator"
|
||||
POSTGRES_USER: str = "antiplagiator"
|
||||
POSTGRES_PASSWORD: str = "changeme"
|
||||
|
||||
REDIS_URL: str = "redis://redis:6379/0"
|
||||
RABBITMQ_URL: str = "amqp://guest:guest@rabbitmq:5672/"
|
||||
|
||||
ENVIRONMENT: str = "development"
|
||||
DEBUG: bool = False
|
||||
|
||||
@property
|
||||
def database_url_sync(self) -> str:
|
||||
return (
|
||||
f"postgresql+psycopg2://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}"
|
||||
f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
32
services/worker-gost/app/db.py
Normal file
32
services/worker-gost/app/db.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Подключение к PostgreSQL для gost-воркера."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
|
||||
engine = create_engine(
|
||||
settings.database_url_sync,
|
||||
pool_size=5,
|
||||
max_overflow=10,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db_session() -> Generator[Session, None, None]:
|
||||
"""Контекстный менеджер для сессии БД."""
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
0
services/worker-gost/app/formatters/__init__.py
Normal file
0
services/worker-gost/app/formatters/__init__.py
Normal file
57
services/worker-gost/app/formatters/gost_7_0_5.py
Normal file
57
services/worker-gost/app/formatters/gost_7_0_5.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""ГОСТ Р 7.0.5-2008 — Краткая библиографическая ссылка.
|
||||
|
||||
Используется для ссылок внутри текста: [Автор, год]
|
||||
"""
|
||||
|
||||
|
||||
class GOST705Formatter:
|
||||
"""Форматтер кратких библиографических ссылок по ГОСТ Р 7.0.5-2008."""
|
||||
|
||||
def format_short(self, doc: dict) -> str:
|
||||
"""
|
||||
Форматировать краткую ссылку вида [Автор, год].
|
||||
|
||||
Args:
|
||||
doc: Словарь с метаданными документа
|
||||
|
||||
Returns:
|
||||
Форматированная краткая ссылка, например [Иванов, 2023]
|
||||
"""
|
||||
authors = doc.get("authors", [])
|
||||
year = doc.get("year", "б. г.")
|
||||
|
||||
if authors:
|
||||
first_author = authors[0]
|
||||
last_name = first_author.get("last_name", "")
|
||||
if last_name:
|
||||
return f"[{last_name}, {year}]"
|
||||
|
||||
# Если нет авторов — используем первые слова названия
|
||||
title = doc.get("title", "")
|
||||
if title:
|
||||
short_title = " ".join(title.split()[:3])
|
||||
return f"[{short_title}..., {year}]"
|
||||
|
||||
return f"[{year}]"
|
||||
|
||||
def format_inline(self, doc: dict, page: str | None = None) -> str:
|
||||
"""
|
||||
Форматировать ссылку для включения в текст с опциональным номером страницы.
|
||||
|
||||
Args:
|
||||
doc: Словарь с метаданными
|
||||
page: Номер страницы (опционально)
|
||||
|
||||
Returns:
|
||||
Например: [Иванов, 2023, с. 15] или [Иванов, 2023]
|
||||
"""
|
||||
base = self.format_short(doc)
|
||||
if page:
|
||||
# Вставить номер страницы перед закрывающей скобкой
|
||||
return base[:-1] + f", с. {page}]"
|
||||
return base
|
||||
|
||||
|
||||
def format_short(doc: dict) -> str:
|
||||
"""Удобная функция для быстрого форматирования."""
|
||||
return GOST705Formatter().format_short(doc)
|
||||
267
services/worker-gost/app/formatters/gost_7_1.py
Normal file
267
services/worker-gost/app/formatters/gost_7_1.py
Normal file
@@ -0,0 +1,267 @@
|
||||
"""ГОСТ 7.1-2003 — Библиографическая запись. Библиографическое описание.
|
||||
|
||||
Полная запись для списка литературы.
|
||||
|
||||
Формат для статьи в журнале:
|
||||
Автор(ы). Название // Журнал. — Год. — Т. X, № X. — С. X–X.
|
||||
|
||||
Формат для книги:
|
||||
Автор(ы). Название. — Город : Издательство, Год. — X с.
|
||||
|
||||
Правила по ГОСТ 7.1-2003:
|
||||
- До 3 авторов: перечислить всех
|
||||
- 4+ авторов: первые 3 + "и др." (рус.) / "et al." (англ.)
|
||||
- Разделитель смысловых частей: " — "
|
||||
- Разделитель авторов: ", "
|
||||
- Место публикации через "/"
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def _format_author(author: dict) -> str:
|
||||
"""
|
||||
Форматировать одного автора.
|
||||
|
||||
Args:
|
||||
author: dict с полями last_name, first_name, initials (опционально)
|
||||
|
||||
Returns:
|
||||
Строка вида "Иванов И. И." или "Иванов И."
|
||||
"""
|
||||
last_name = author.get("last_name", "").strip()
|
||||
initials = author.get("initials", "").strip()
|
||||
first_name = author.get("first_name", "").strip()
|
||||
|
||||
if not last_name:
|
||||
return ""
|
||||
|
||||
if initials:
|
||||
# Нормализовать инициалы: обеспечить точки
|
||||
if not initials.endswith("."):
|
||||
initials += "."
|
||||
return f"{last_name} {initials}"
|
||||
elif first_name:
|
||||
# Извлечь инициалы из полного имени
|
||||
parts = first_name.split()
|
||||
inits = "".join(p[0].upper() + "." for p in parts if p)
|
||||
return f"{last_name} {inits}"
|
||||
else:
|
||||
return last_name
|
||||
|
||||
|
||||
def _format_authors(authors: list[dict], lang: str = "ru") -> str:
|
||||
"""
|
||||
Форматировать список авторов по правилам ГОСТ 7.1-2003.
|
||||
|
||||
Args:
|
||||
authors: Список словарей с авторами
|
||||
lang: Язык для "и др." / "et al."
|
||||
|
||||
Returns:
|
||||
Строка с отформатированными авторами
|
||||
"""
|
||||
if not authors:
|
||||
return ""
|
||||
|
||||
et_al = "и др." if lang == "ru" else "et al."
|
||||
formatted = [_format_author(a) for a in authors if a.get("last_name")]
|
||||
formatted = [f for f in formatted if f]
|
||||
|
||||
if not formatted:
|
||||
return ""
|
||||
|
||||
if len(formatted) <= 3:
|
||||
return ", ".join(formatted)
|
||||
else:
|
||||
return ", ".join(formatted[:3]) + f", {et_al}"
|
||||
|
||||
|
||||
class GOST71Formatter:
|
||||
"""Форматтер полных библиографических записей по ГОСТ 7.1-2003."""
|
||||
|
||||
def format_full(self, doc: dict) -> str:
|
||||
"""
|
||||
Форматировать полную библиографическую запись.
|
||||
|
||||
Args:
|
||||
doc: Словарь с полями документа
|
||||
|
||||
Returns:
|
||||
Отформатированная строка библиографической записи
|
||||
"""
|
||||
doc_type = self._detect_type(doc)
|
||||
|
||||
if doc_type == "article":
|
||||
return self._format_article(doc)
|
||||
elif doc_type == "book":
|
||||
return self._format_book(doc)
|
||||
elif doc_type == "web":
|
||||
return self._format_web(doc)
|
||||
else:
|
||||
return self._format_generic(doc)
|
||||
|
||||
def _detect_type(self, doc: dict) -> str:
|
||||
"""Определить тип документа."""
|
||||
if doc.get("journal"):
|
||||
return "article"
|
||||
if doc.get("url") and not doc.get("journal"):
|
||||
return "web"
|
||||
return "generic"
|
||||
|
||||
def _format_article(self, doc: dict) -> str:
|
||||
"""
|
||||
Форматировать статью в журнале.
|
||||
|
||||
Формат: Автор(ы). Название // Журнал. — Год. — Т. X, № X. — С. X–X.
|
||||
"""
|
||||
lang = doc.get("lang", "ru") or "ru"
|
||||
authors = doc.get("authors", [])
|
||||
author_str = _format_authors(authors, lang=lang)
|
||||
|
||||
title = (doc.get("title") or "").strip()
|
||||
journal = (doc.get("journal") or "").strip()
|
||||
year = doc.get("year", "")
|
||||
volume = doc.get("volume", "")
|
||||
issue = doc.get("issue", "")
|
||||
pages = doc.get("pages", "")
|
||||
doi = doc.get("doi", "")
|
||||
|
||||
parts = []
|
||||
|
||||
# Авторы и название
|
||||
if author_str:
|
||||
parts.append(f"{author_str}. {title}")
|
||||
else:
|
||||
parts.append(title)
|
||||
|
||||
# Место публикации
|
||||
location = f"// {journal}"
|
||||
if year:
|
||||
location += f". — {year}"
|
||||
if volume:
|
||||
location += f". — Т. {volume}"
|
||||
if issue:
|
||||
location += f", № {issue}"
|
||||
if pages:
|
||||
location += f". — С. {pages}"
|
||||
|
||||
parts.append(location)
|
||||
|
||||
result = " ".join(parts)
|
||||
|
||||
# DOI
|
||||
if doi:
|
||||
result += f". — DOI: {doi}"
|
||||
|
||||
return result.strip()
|
||||
|
||||
def _format_book(self, doc: dict) -> str:
|
||||
"""
|
||||
Форматировать книгу / монографию.
|
||||
|
||||
Формат: Автор(ы). Название. — Город : Издательство, Год. — X с.
|
||||
"""
|
||||
lang = doc.get("lang", "ru") or "ru"
|
||||
authors = doc.get("authors", [])
|
||||
author_str = _format_authors(authors, lang=lang)
|
||||
|
||||
title = (doc.get("title") or "").strip()
|
||||
year = doc.get("year", "б. г.")
|
||||
pages = doc.get("pages", "")
|
||||
|
||||
parts = []
|
||||
|
||||
if author_str:
|
||||
parts.append(f"{author_str}.")
|
||||
|
||||
parts.append(f"{title}.")
|
||||
parts.append(f"— {year}.")
|
||||
|
||||
if pages:
|
||||
# Предполагаем, что pages содержит количество страниц
|
||||
parts.append(f"— {pages} с.")
|
||||
|
||||
return " ".join(parts).strip()
|
||||
|
||||
def _format_web(self, doc: dict) -> str:
|
||||
"""
|
||||
Форматировать электронный ресурс.
|
||||
|
||||
Формат: Автор(ы). Название [Электронный ресурс]. — URL: ... (дата обращения: ...)
|
||||
"""
|
||||
lang = doc.get("lang", "ru") or "ru"
|
||||
authors = doc.get("authors", [])
|
||||
author_str = _format_authors(authors, lang=lang)
|
||||
|
||||
title = (doc.get("title") or "").strip()
|
||||
url = (doc.get("url") or "").strip()
|
||||
year = doc.get("year", "")
|
||||
|
||||
parts = []
|
||||
|
||||
if author_str:
|
||||
parts.append(f"{author_str}.")
|
||||
|
||||
# Для ГОСТ — обозначение электронного ресурса
|
||||
parts.append(f"{title} [Электронный ресурс].")
|
||||
|
||||
if year:
|
||||
parts.append(f"— {year}.")
|
||||
|
||||
if url:
|
||||
parts.append(f"— URL: {url}")
|
||||
|
||||
return " ".join(parts).strip()
|
||||
|
||||
def _format_generic(self, doc: dict) -> str:
|
||||
"""Базовый форматтер для неопределённых типов."""
|
||||
lang = doc.get("lang", "ru") or "ru"
|
||||
author_str = _format_authors(doc.get("authors", []), lang=lang)
|
||||
title = (doc.get("title") or "").strip()
|
||||
year = doc.get("year", "")
|
||||
|
||||
result = ""
|
||||
if author_str:
|
||||
result += f"{author_str}. "
|
||||
result += title
|
||||
if year:
|
||||
result += f". — {year}"
|
||||
|
||||
return result.strip()
|
||||
|
||||
|
||||
def _get_sort_key(doc: dict) -> str:
|
||||
"""
|
||||
Получить ключ сортировки для документа.
|
||||
|
||||
Кириллица идёт перед латиницей (для русских традиций):
|
||||
сначала русскоязычные источники, затем иностранные.
|
||||
|
||||
Args:
|
||||
doc: Словарь с метаданными
|
||||
|
||||
Returns:
|
||||
Строка для сортировки
|
||||
"""
|
||||
authors = doc.get("authors", [])
|
||||
if authors:
|
||||
last_name = authors[0].get("last_name", "")
|
||||
else:
|
||||
last_name = doc.get("title", "")
|
||||
|
||||
if not last_name:
|
||||
return "яяя" # В конец
|
||||
|
||||
# Кириллица < латиница (традиция: русские источники первыми)
|
||||
first_char = last_name[0].lower()
|
||||
is_latin = ord(first_char) < 256 and first_char.isalpha()
|
||||
|
||||
# Префикс для сортировки: 0 для кириллицы, 1 для латиницы
|
||||
prefix = "1" if is_latin else "0"
|
||||
return f"{prefix}{last_name.lower()}"
|
||||
|
||||
|
||||
def format_full(doc: dict) -> str:
|
||||
"""Удобная функция для быстрого форматирования."""
|
||||
return GOST71Formatter().format_full(doc)
|
||||
0
services/worker-gost/app/tasks/__init__.py
Normal file
0
services/worker-gost/app/tasks/__init__.py
Normal file
142
services/worker-gost/app/tasks/gost.py
Normal file
142
services/worker-gost/app/tasks/gost.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Celery задача форматирования библиографии по ГОСТ."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.celery_app import celery_app
|
||||
from app.db import db_session
|
||||
from app.formatters.gost_7_0_5 import GOST705Formatter
|
||||
from app.formatters.gost_7_1 import GOST71Formatter, _get_sort_key
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="gost.format_bibliography",
|
||||
bind=True,
|
||||
max_retries=3,
|
||||
default_retry_delay=30,
|
||||
)
|
||||
def format_bibliography(
|
||||
self,
|
||||
task_id: str,
|
||||
doc_ids: list[int],
|
||||
style: str = "7.1",
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Форматировать список литературы по ГОСТ для переданных doc_ids.
|
||||
|
||||
Args:
|
||||
task_id: ID задачи в PostgreSQL
|
||||
doc_ids: Список ID документов из PostgreSQL
|
||||
style: Стиль форматирования — "7.1" (полная) или "7.0.5" (краткая)
|
||||
|
||||
Returns:
|
||||
dict с отформатированной библиографией
|
||||
"""
|
||||
from app.models import Document, Task
|
||||
|
||||
logger.info(
|
||||
f"Форматирование библиографии для задачи {task_id!r}: "
|
||||
f"{len(doc_ids)} документов, стиль ГОСТ {style}"
|
||||
)
|
||||
|
||||
try:
|
||||
with db_session() as session:
|
||||
# Обновить статус
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.status = "processing"
|
||||
session.commit()
|
||||
|
||||
# Получить документы
|
||||
docs = session.execute(
|
||||
select(Document).where(Document.id.in_(doc_ids))
|
||||
).scalars().all()
|
||||
|
||||
if not docs:
|
||||
raise ValueError(f"Документы не найдены: {doc_ids}")
|
||||
|
||||
# Выбрать форматтер
|
||||
formatter = GOST71Formatter() if style == "7.1" else GOST705Formatter()
|
||||
|
||||
# Преобразовать ORM объекты в словари для форматирования
|
||||
docs_dicts = []
|
||||
for doc in docs:
|
||||
docs_dicts.append({
|
||||
"id": doc.id,
|
||||
"title": doc.title,
|
||||
"authors": doc.authors or [],
|
||||
"year": doc.year,
|
||||
"journal": doc.journal,
|
||||
"volume": doc.volume,
|
||||
"issue": doc.issue,
|
||||
"pages": doc.pages,
|
||||
"doi": doc.doi,
|
||||
"url": doc.url,
|
||||
"lang": doc.lang or "ru",
|
||||
"source": doc.source,
|
||||
})
|
||||
|
||||
# Сортировать: кириллица (рус. авторы) → латиница (иностр.)
|
||||
sorted_docs = sorted(docs_dicts, key=lambda d: _get_sort_key(d))
|
||||
|
||||
bibliography = []
|
||||
for i, doc_dict in enumerate(sorted_docs, 1):
|
||||
if style == "7.1":
|
||||
citation = formatter.format_full(doc_dict)
|
||||
else:
|
||||
citation = formatter.format_short(doc_dict)
|
||||
|
||||
bibliography.append({
|
||||
"number": i,
|
||||
"citation": citation,
|
||||
"doc_id": doc_dict["id"],
|
||||
})
|
||||
|
||||
result = {
|
||||
"bibliography": bibliography,
|
||||
"style": style,
|
||||
"total": len(bibliography),
|
||||
}
|
||||
|
||||
# Сохранить результат
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.result = result
|
||||
task.status = "done"
|
||||
task.queue_position = None
|
||||
session.commit()
|
||||
|
||||
logger.info(
|
||||
f"Библиография сформирована для задачи {task_id!r}: "
|
||||
f"{len(bibliography)} записей по ГОСТ {style}"
|
||||
)
|
||||
|
||||
# Уведомить пользователя
|
||||
celery_app.send_task(
|
||||
"notify.send_task_done",
|
||||
args=[task_id],
|
||||
queue="queue.notify",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"Ошибка при форматировании библиографии для задачи {task_id!r}: {exc}", exc_info=True)
|
||||
|
||||
try:
|
||||
from app.models import Task
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.status = "failed"
|
||||
task.error = str(exc)
|
||||
session.commit()
|
||||
except Exception as db_exc:
|
||||
logger.error(f"Не удалось обновить статус задачи: {db_exc}")
|
||||
|
||||
raise self.retry(exc=exc, countdown=30)
|
||||
5
services/worker-gost/requirements.txt
Normal file
5
services/worker-gost/requirements.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
celery==5.4.0
|
||||
redis==5.0.4
|
||||
sqlalchemy==2.0.30
|
||||
psycopg2-binary==2.9.9
|
||||
pydantic-settings==2.2.1
|
||||
30
services/worker-gpu/Dockerfile
Normal file
30
services/worker-gpu/Dockerfile
Normal file
@@ -0,0 +1,30 @@
|
||||
FROM nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
|
||||
|
||||
# Установить Python 3.11
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3.11 \
|
||||
python3.11-dev \
|
||||
python3-pip \
|
||||
python3.11-distutils \
|
||||
gcc \
|
||||
g++ \
|
||||
libpq-dev \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Сделать python3.11 дефолтным
|
||||
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
|
||||
update-alternatives --install /usr/bin/python python python3.11 1 && \
|
||||
python3.11 -m pip install --upgrade pip
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# Директория для FAISS индекса (монтируется как volume)
|
||||
RUN mkdir -p /data/index
|
||||
|
||||
CMD ["celery", "-A", "app.celery_app", "worker", "-Q", "queue.gpu", "-c", "1", "-n", "gpu@%h", "--loglevel=info"]
|
||||
0
services/worker-gpu/app/__init__.py
Normal file
0
services/worker-gpu/app/__init__.py
Normal file
38
services/worker-gpu/app/celery_app.py
Normal file
38
services/worker-gpu/app/celery_app.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""Celery приложение GPU воркера."""
|
||||
|
||||
from celery import Celery
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from app.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"worker_gpu",
|
||||
broker=settings.RABBITMQ_URL,
|
||||
backend=settings.REDIS_URL,
|
||||
include=["app.tasks.search", "app.tasks.plagiarism"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="Europe/Moscow",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
# Маршрутизация
|
||||
task_routes={
|
||||
"gpu.*": {"queue": "queue.gpu"},
|
||||
"index.*": {"queue": "queue.index"},
|
||||
"notify.*": {"queue": "queue.notify"},
|
||||
"gost.*": {"queue": "queue.gost"},
|
||||
},
|
||||
# Надёжность
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
# GPU воркер — только 1 процесс
|
||||
worker_concurrency=1,
|
||||
# Результаты хранить 24 часа
|
||||
result_expires=86400,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
64
services/worker-gpu/app/config.py
Normal file
64
services/worker-gpu/app/config.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""Конфигурация GPU воркера."""
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Настройки GPU воркера."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
# PostgreSQL
|
||||
POSTGRES_HOST: str = "postgres"
|
||||
POSTGRES_PORT: int = 5432
|
||||
POSTGRES_DB: str = "antiplagiator"
|
||||
POSTGRES_USER: str = "antiplagiator"
|
||||
POSTGRES_PASSWORD: str = "changeme"
|
||||
|
||||
# Redis
|
||||
REDIS_URL: str = "redis://redis:6379/0"
|
||||
|
||||
# RabbitMQ
|
||||
RABBITMQ_URL: str = "amqp://guest:guest@rabbitmq:5672/"
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: str = "minio:9000"
|
||||
MINIO_ACCESS_KEY: str = "minioadmin"
|
||||
MINIO_SECRET_KEY: str = "changeme"
|
||||
MINIO_BUCKET_DOCS: str = "documents"
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_URL: str = "http://elasticsearch:9200"
|
||||
|
||||
# Ollama
|
||||
OLLAMA_URL: str = "http://ollama:11434"
|
||||
|
||||
# FAISS / ML
|
||||
FAISS_INDEX_PATH: str = "/data/index/faiss.index"
|
||||
FAISS_ID_MAP_PATH: str = "/data/index/faiss_id_map.json"
|
||||
EMBED_MODEL: str = "paraphrase-multilingual-mpnet-base-v2"
|
||||
EMBED_DEVICE: str = "cuda"
|
||||
EMBED_BATCH_SIZE: int = 64
|
||||
EMBED_DIM: int = 768 # Размерность вектора paraphrase-multilingual-mpnet-base-v2
|
||||
FAISS_NLIST: int = 1024 # Количество кластеров для IVFFlat
|
||||
FAISS_NPROBE: int = 64 # Количество кластеров для поиска
|
||||
|
||||
# App
|
||||
APP_URL: str = "https://academic.jze9.ru"
|
||||
ENVIRONMENT: str = "development"
|
||||
DEBUG: bool = False
|
||||
|
||||
@property
|
||||
def database_url_sync(self) -> str:
|
||||
"""Синхронный URL для SQLAlchemy (psycopg2)."""
|
||||
return (
|
||||
f"postgresql+psycopg2://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}"
|
||||
f"@{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
50
services/worker-gpu/app/db.py
Normal file
50
services/worker-gpu/app/db.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""Синхронное подключение к PostgreSQL для Celery воркеров."""
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
import redis as redis_lib
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Синхронный движок для воркеров (psycopg2)
|
||||
engine = create_engine(
|
||||
settings.database_url_sync,
|
||||
pool_size=5,
|
||||
max_overflow=10,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db_session() -> Generator[Session, None, None]:
|
||||
"""Контекстный менеджер для сессии БД."""
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
# Redis клиент для кэширования результатов поиска
|
||||
_redis_client: redis_lib.Redis | None = None
|
||||
|
||||
|
||||
def get_redis() -> redis_lib.Redis:
|
||||
"""Получить или создать Redis клиент."""
|
||||
global _redis_client
|
||||
if _redis_client is None:
|
||||
_redis_client = redis_lib.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
return _redis_client
|
||||
151
services/worker-gpu/app/es_client.py
Normal file
151
services/worker-gpu/app/es_client.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Elasticsearch клиент для полнотекстового поиска (BM25)."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from elasticsearch import Elasticsearch, exceptions as es_exceptions
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_es_client: Elasticsearch | None = None
|
||||
|
||||
INDEX_NAME = "documents"
|
||||
|
||||
|
||||
def get_es_client() -> Elasticsearch:
|
||||
"""Получить или создать синглтон ES клиент."""
|
||||
global _es_client
|
||||
if _es_client is None:
|
||||
_es_client = Elasticsearch(
|
||||
settings.ELASTICSEARCH_URL,
|
||||
retry_on_timeout=True,
|
||||
max_retries=3,
|
||||
request_timeout=30,
|
||||
)
|
||||
return _es_client
|
||||
|
||||
|
||||
def search_fulltext(
|
||||
query: str,
|
||||
lang: str | None = None,
|
||||
year_from: int | None = None,
|
||||
year_to: int | None = None,
|
||||
category: str | None = None,
|
||||
size: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Полнотекстовый BM25 поиск в Elasticsearch.
|
||||
|
||||
Args:
|
||||
query: Поисковый запрос
|
||||
lang: Фильтр языка (ru, en, None = все)
|
||||
year_from: Фильтр года публикации (от)
|
||||
year_to: Фильтр года публикации (до)
|
||||
category: Фильтр тематической категории
|
||||
size: Максимальное количество результатов
|
||||
|
||||
Returns:
|
||||
Список словарей с полями doc_id и score
|
||||
"""
|
||||
es = get_es_client()
|
||||
|
||||
# Составить bool запрос
|
||||
must_clauses: list[dict] = [
|
||||
{
|
||||
"multi_match": {
|
||||
"query": query,
|
||||
"fields": ["title^3", "abstract^2", "authors"],
|
||||
"type": "best_fields",
|
||||
"operator": "or",
|
||||
"minimum_should_match": "30%",
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
filter_clauses: list[dict] = []
|
||||
|
||||
if lang:
|
||||
filter_clauses.append({"term": {"lang": lang}})
|
||||
|
||||
if year_from or year_to:
|
||||
range_filter: dict = {"range": {"year": {}}}
|
||||
if year_from:
|
||||
range_filter["range"]["year"]["gte"] = year_from
|
||||
if year_to:
|
||||
range_filter["range"]["year"]["lte"] = year_to
|
||||
filter_clauses.append(range_filter)
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must_clauses,
|
||||
"filter": filter_clauses,
|
||||
}
|
||||
},
|
||||
"size": size,
|
||||
"_source": ["doc_id"],
|
||||
}
|
||||
|
||||
try:
|
||||
response = es.search(index=INDEX_NAME, body=body)
|
||||
hits = response["hits"]["hits"]
|
||||
return [
|
||||
{
|
||||
"doc_id": hit["_source"]["doc_id"],
|
||||
"es_score": hit["_score"],
|
||||
"es_id": hit["_id"],
|
||||
}
|
||||
for hit in hits
|
||||
]
|
||||
except es_exceptions.ConnectionError as e:
|
||||
logger.error(f"Elasticsearch недоступен: {e}")
|
||||
return []
|
||||
except es_exceptions.NotFoundError:
|
||||
logger.warning(f"Индекс {INDEX_NAME!r} не найден в Elasticsearch")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка поиска в Elasticsearch: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def index_document(doc_id: int, doc_data: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Индексировать документ в Elasticsearch.
|
||||
|
||||
Args:
|
||||
doc_id: ID документа в PostgreSQL
|
||||
doc_data: Словарь с метаданными документа
|
||||
|
||||
Returns:
|
||||
True при успехе, False при ошибке
|
||||
"""
|
||||
es = get_es_client()
|
||||
|
||||
doc = {
|
||||
"doc_id": doc_id,
|
||||
"source": doc_data.get("source"),
|
||||
"title": doc_data.get("title", ""),
|
||||
"abstract": doc_data.get("abstract", ""),
|
||||
"authors": " ".join(
|
||||
f"{a.get('last_name', '')} {a.get('first_name', '')}"
|
||||
for a in doc_data.get("authors", [])
|
||||
),
|
||||
"year": doc_data.get("year"),
|
||||
"lang": doc_data.get("lang"),
|
||||
"journal": doc_data.get("journal"),
|
||||
"doi": doc_data.get("doi"),
|
||||
"url": doc_data.get("url"),
|
||||
}
|
||||
|
||||
try:
|
||||
es.index(
|
||||
index=INDEX_NAME,
|
||||
id=str(doc_id),
|
||||
document=doc,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка индексации документа {doc_id} в ES: {e}")
|
||||
return False
|
||||
210
services/worker-gpu/app/faiss_manager.py
Normal file
210
services/worker-gpu/app/faiss_manager.py
Normal file
@@ -0,0 +1,210 @@
|
||||
"""Singleton менеджер FAISS GPU индекса.
|
||||
|
||||
Использует IVFFlat (а не HNSW — не поддерживается на GPU).
|
||||
Поддерживает graceful degradation на CPU если GPU недоступна.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FAISSManager:
|
||||
"""Singleton для управления FAISS индексом на GPU."""
|
||||
|
||||
_index = None
|
||||
_id_map: dict[int, int] = {} # faiss_internal_id -> doc_id (PostgreSQL)
|
||||
_reverse_map: dict[int, int] = {} # doc_id -> faiss_internal_id
|
||||
_use_gpu: bool = False
|
||||
_is_trained: bool = False
|
||||
|
||||
@classmethod
|
||||
def load_or_create(cls) -> None:
|
||||
"""
|
||||
Загрузить индекс с диска или создать новый.
|
||||
|
||||
Пытается перенести индекс на GPU, при ошибке остаётся на CPU.
|
||||
"""
|
||||
import faiss
|
||||
|
||||
index_path = settings.FAISS_INDEX_PATH
|
||||
id_map_path = settings.FAISS_ID_MAP_PATH
|
||||
|
||||
# Загрузить ID маппинг
|
||||
if os.path.exists(id_map_path):
|
||||
with open(id_map_path, "r") as f:
|
||||
raw_map = json.load(f)
|
||||
cls._id_map = {int(k): int(v) for k, v in raw_map.items()}
|
||||
cls._reverse_map = {v: k for k, v in cls._id_map.items()}
|
||||
logger.info(f"ID маппинг загружен: {len(cls._id_map)} записей")
|
||||
|
||||
if os.path.exists(index_path):
|
||||
# Загрузить существующий индекс
|
||||
logger.info(f"Загрузка FAISS индекса из {index_path}")
|
||||
cpu_index = faiss.read_index(index_path)
|
||||
cls._is_trained = cpu_index.is_trained
|
||||
else:
|
||||
# Создать новый IVFFlat индекс
|
||||
logger.info("Создание нового FAISS IVFFlat индекса...")
|
||||
quantizer = faiss.IndexFlatIP(settings.EMBED_DIM)
|
||||
cpu_index = faiss.IndexIVFFlat(
|
||||
quantizer,
|
||||
settings.EMBED_DIM,
|
||||
settings.FAISS_NLIST,
|
||||
faiss.METRIC_INNER_PRODUCT,
|
||||
)
|
||||
# IVFFlat требует обучения перед использованием
|
||||
cls._is_trained = False
|
||||
|
||||
# Попытка перенести на GPU
|
||||
try:
|
||||
res = faiss.StandardGpuResources()
|
||||
cls._index = faiss.index_cpu_to_gpu(res, 0, cpu_index)
|
||||
cls._use_gpu = True
|
||||
logger.info("FAISS индекс размещён на GPU")
|
||||
except Exception as e:
|
||||
logger.warning(f"GPU недоступна: {e}. Используем CPU FAISS.")
|
||||
cls._index = cpu_index
|
||||
cls._use_gpu = False
|
||||
|
||||
if cls._is_trained:
|
||||
cls._index.nprobe = settings.FAISS_NPROBE
|
||||
|
||||
@classmethod
|
||||
def search(cls, query_vector: np.ndarray, k: int = 20) -> list[tuple[int, float]]:
|
||||
"""
|
||||
Поиск k ближайших векторов.
|
||||
|
||||
Args:
|
||||
query_vector: Нормализованный вектор запроса, форма (768,)
|
||||
k: Количество результатов
|
||||
|
||||
Returns:
|
||||
Список кортежей (doc_id, cosine_score), отсортированных по убыванию score
|
||||
"""
|
||||
if cls._index is None or not cls._is_trained:
|
||||
logger.warning("FAISS индекс не инициализирован или не обучен, пропускаем поиск")
|
||||
return []
|
||||
|
||||
try:
|
||||
query = query_vector.reshape(1, -1).astype(np.float32)
|
||||
distances, indices = cls._index.search(query, k)
|
||||
|
||||
results = []
|
||||
for idx, dist in zip(indices[0], distances[0]):
|
||||
if idx == -1:
|
||||
continue
|
||||
doc_id = cls._id_map.get(int(idx))
|
||||
if doc_id is not None:
|
||||
results.append((doc_id, float(dist)))
|
||||
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка поиска FAISS: {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def add_vectors(cls, vectors: np.ndarray, doc_ids: list[int]) -> None:
|
||||
"""
|
||||
Добавить векторы в индекс.
|
||||
|
||||
Если индекс не обучен и накопилось достаточно векторов — обучить его.
|
||||
|
||||
Args:
|
||||
vectors: numpy массив формы (N, 768)
|
||||
doc_ids: Список doc_id из PostgreSQL
|
||||
"""
|
||||
import faiss
|
||||
|
||||
if cls._index is None:
|
||||
cls.load_or_create()
|
||||
|
||||
vectors = vectors.astype(np.float32)
|
||||
|
||||
if not cls._is_trained:
|
||||
# Для IVFFlat нужно минимум nlist * 39 обучающих примеров
|
||||
min_train = settings.FAISS_NLIST * 39
|
||||
current_n = cls._index.ntotal if cls._index is not None else 0
|
||||
|
||||
if current_n + len(vectors) >= min_train:
|
||||
logger.info(f"Обучение IVFFlat индекса на {current_n + len(vectors)} векторах...")
|
||||
cls._index.train(vectors)
|
||||
cls._is_trained = True
|
||||
cls._index.nprobe = settings.FAISS_NPROBE
|
||||
logger.info("Обучение завершено")
|
||||
else:
|
||||
logger.info(
|
||||
f"Недостаточно векторов для обучения IVFFlat "
|
||||
f"({current_n + len(vectors)} < {min_train}). "
|
||||
"Используйте FlatIP до накопления достаточного количества документов."
|
||||
)
|
||||
# Временный flat индекс для малого количества документов
|
||||
if not hasattr(cls, '_flat_index') or cls._flat_index is None:
|
||||
cls._flat_index = faiss.IndexFlatIP(settings.EMBED_DIM)
|
||||
|
||||
# Добавить в flat индекс
|
||||
start_id = cls._flat_index.ntotal
|
||||
cls._flat_index.add(vectors)
|
||||
for i, doc_id in enumerate(doc_ids):
|
||||
internal_id = start_id + i
|
||||
cls._id_map[internal_id] = doc_id
|
||||
cls._reverse_map[doc_id] = internal_id
|
||||
cls._save_id_map()
|
||||
return
|
||||
|
||||
if cls._is_trained:
|
||||
start_id = cls._index.ntotal
|
||||
cls._index.add(vectors)
|
||||
for i, doc_id in enumerate(doc_ids):
|
||||
internal_id = start_id + i
|
||||
cls._id_map[internal_id] = doc_id
|
||||
cls._reverse_map[doc_id] = internal_id
|
||||
|
||||
cls._save_id_map()
|
||||
logger.info(f"Добавлено {len(doc_ids)} векторов в FAISS. Всего: {cls._index.ntotal}")
|
||||
|
||||
@classmethod
|
||||
def save(cls) -> None:
|
||||
"""Сохранить индекс на диск (CPU версия)."""
|
||||
if cls._index is None:
|
||||
return
|
||||
|
||||
import faiss
|
||||
|
||||
index_path = Path(settings.FAISS_INDEX_PATH)
|
||||
index_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Перенести на CPU перед сохранением
|
||||
if cls._use_gpu:
|
||||
cpu_index = faiss.index_gpu_to_cpu(cls._index)
|
||||
else:
|
||||
cpu_index = cls._index
|
||||
|
||||
faiss.write_index(cpu_index, str(index_path))
|
||||
cls._save_id_map()
|
||||
logger.info(f"FAISS индекс сохранён: {index_path} ({cpu_index.ntotal} векторов)")
|
||||
|
||||
@classmethod
|
||||
def _save_id_map(cls) -> None:
|
||||
"""Сохранить маппинг faiss_id -> doc_id на диск."""
|
||||
id_map_path = Path(settings.FAISS_ID_MAP_PATH)
|
||||
id_map_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(id_map_path, "w") as f:
|
||||
json.dump({str(k): v for k, v in cls._id_map.items()}, f)
|
||||
|
||||
@classmethod
|
||||
@property
|
||||
def total_vectors(cls) -> int:
|
||||
"""Количество векторов в индексе."""
|
||||
if cls._index is None:
|
||||
return 0
|
||||
return cls._index.ntotal
|
||||
81
services/worker-gpu/app/model_manager.py
Normal file
81
services/worker-gpu/app/model_manager.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Singleton менеджер sentence-transformers модели.
|
||||
|
||||
Модель загружается один раз при первом обращении и кэшируется в памяти GPU.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModelManager:
|
||||
"""Singleton для управления эмбеддинг-моделью."""
|
||||
|
||||
_instance = None
|
||||
_model = None
|
||||
_device: str = settings.EMBED_DEVICE
|
||||
|
||||
@classmethod
|
||||
def get_model(cls):
|
||||
"""Получить или загрузить модель sentence-transformers."""
|
||||
if cls._model is None:
|
||||
# Импорт здесь, чтобы не блокировать импорт модуля если torch не установлен
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
logger.info(
|
||||
f"Загрузка модели {settings.EMBED_MODEL!r} на устройство {settings.EMBED_DEVICE!r}..."
|
||||
)
|
||||
try:
|
||||
cls._model = SentenceTransformer(
|
||||
settings.EMBED_MODEL,
|
||||
device=settings.EMBED_DEVICE,
|
||||
)
|
||||
logger.info(
|
||||
f"Модель загружена. Размерность вектора: {cls._model.get_sentence_embedding_dimension()}"
|
||||
)
|
||||
except Exception as e:
|
||||
# Graceful degradation на CPU если CUDA недоступна
|
||||
logger.warning(f"Не удалось загрузить модель на CUDA: {e}. Переключение на CPU.")
|
||||
cls._device = "cpu"
|
||||
cls._model = SentenceTransformer(settings.EMBED_MODEL, device="cpu")
|
||||
|
||||
return cls._model
|
||||
|
||||
@classmethod
|
||||
def encode(cls, texts: list[str]) -> np.ndarray:
|
||||
"""
|
||||
Закодировать тексты в векторы.
|
||||
|
||||
Args:
|
||||
texts: Список текстов для кодирования
|
||||
|
||||
Returns:
|
||||
numpy массив формы (len(texts), 768), нормализованный для cosine similarity
|
||||
"""
|
||||
if not texts:
|
||||
return np.array([]).reshape(0, settings.EMBED_DIM)
|
||||
|
||||
model = cls.get_model()
|
||||
vectors = model.encode(
|
||||
texts,
|
||||
batch_size=settings.EMBED_BATCH_SIZE,
|
||||
normalize_embeddings=True, # Нормализация для cosine через inner product
|
||||
show_progress_bar=len(texts) > 100,
|
||||
convert_to_numpy=True,
|
||||
)
|
||||
return vectors.astype(np.float32)
|
||||
|
||||
@classmethod
|
||||
def encode_single(cls, text: str) -> np.ndarray:
|
||||
"""Закодировать один текст. Удобный метод."""
|
||||
return cls.encode([text])[0]
|
||||
|
||||
@classmethod
|
||||
def unload(cls) -> None:
|
||||
"""Выгрузить модель из памяти (для тестов/диагностики)."""
|
||||
cls._model = None
|
||||
logger.info("Модель выгружена из памяти")
|
||||
138
services/worker-gpu/app/ollama_client.py
Normal file
138
services/worker-gpu/app/ollama_client.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""Клиент для Ollama HTTP API — LLM анализ парафраза и суммаризация."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OllamaClient:
|
||||
"""HTTP клиент для Ollama LLM."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.base_url = settings.OLLAMA_URL
|
||||
self.model = "llama3:8b"
|
||||
self.timeout = 60.0 # секунд
|
||||
|
||||
def check_paraphrase(self, text_a: str, text_b: str) -> dict:
|
||||
"""
|
||||
Проверить является ли text_b парафразом text_a с помощью LLM.
|
||||
|
||||
Args:
|
||||
text_a: Исходный текст из базы данных (до 500 символов)
|
||||
text_b: Проверяемый фрагмент (до 500 символов)
|
||||
|
||||
Returns:
|
||||
dict с полями:
|
||||
- is_paraphrase: bool
|
||||
- confidence: float 0.0-1.0
|
||||
- reason: str — краткое объяснение
|
||||
"""
|
||||
prompt = f"""Ты эксперт по академическому плагиату. Определи, является ли Текст B парафразом Текста A.
|
||||
|
||||
Текст A (источник): {text_a[:500]}
|
||||
|
||||
Текст B (проверяемый): {text_b[:500]}
|
||||
|
||||
Критерии парафраза: передача тех же идей другими словами, перефразировка без ссылки на источник.
|
||||
|
||||
Ответь ТОЛЬКО валидным JSON без пояснений и markdown:
|
||||
{{"is_paraphrase": true/false, "confidence": 0.0-1.0, "reason": "краткое объяснение на русском"}}"""
|
||||
|
||||
try:
|
||||
response = httpx.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"format": "json",
|
||||
"options": {
|
||||
"temperature": 0.1, # Детерминированный вывод
|
||||
"num_predict": 200,
|
||||
},
|
||||
},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
llm_response = result.get("response", "{}")
|
||||
|
||||
# Парсим JSON из ответа
|
||||
parsed = json.loads(llm_response)
|
||||
return {
|
||||
"is_paraphrase": bool(parsed.get("is_paraphrase", False)),
|
||||
"confidence": float(parsed.get("confidence", 0.0)),
|
||||
"reason": str(parsed.get("reason", "")),
|
||||
}
|
||||
|
||||
except (httpx.TimeoutException, httpx.ConnectError) as e:
|
||||
logger.warning(f"Ollama недоступна: {e}")
|
||||
return {"is_paraphrase": False, "confidence": 0.0, "reason": "LLM недоступна"}
|
||||
except (json.JSONDecodeError, KeyError) as e:
|
||||
logger.warning(f"Ошибка парсинга ответа Ollama: {e}")
|
||||
return {"is_paraphrase": False, "confidence": 0.0, "reason": "Ошибка парсинга ответа"}
|
||||
except Exception as e:
|
||||
logger.error(f"Неожиданная ошибка при обращении к Ollama: {e}")
|
||||
return {"is_paraphrase": False, "confidence": 0.0, "reason": str(e)}
|
||||
|
||||
def summarize(self, title: str, abstract: str, lang: str = "ru") -> str:
|
||||
"""
|
||||
Сгенерировать краткое академическое изложение статьи.
|
||||
|
||||
Args:
|
||||
title: Название статьи
|
||||
abstract: Аннотация статьи
|
||||
lang: Язык ответа (ru/en)
|
||||
|
||||
Returns:
|
||||
Краткое изложение на указанном языке
|
||||
"""
|
||||
lang_instruction = "на русском языке" if lang == "ru" else "in English"
|
||||
|
||||
prompt = f"""Сделай краткое академическое изложение {lang_instruction}.
|
||||
|
||||
Название: {title}
|
||||
Аннотация: {abstract[:1000]}
|
||||
|
||||
Изложение должно:
|
||||
- Содержать 3-5 предложений
|
||||
- Передавать основную идею и результаты
|
||||
- Быть написано академическим стилем
|
||||
- НЕ копировать текст дословно
|
||||
|
||||
Ответ:"""
|
||||
|
||||
try:
|
||||
response = httpx.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3,
|
||||
"num_predict": 300,
|
||||
},
|
||||
},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json().get("response", "").strip()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка суммаризации через Ollama: {e}")
|
||||
return abstract[:500] if abstract else title
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Проверить доступность Ollama сервера."""
|
||||
try:
|
||||
response = httpx.get(f"{self.base_url}/api/tags", timeout=5.0)
|
||||
return response.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
0
services/worker-gpu/app/tasks/__init__.py
Normal file
0
services/worker-gpu/app/tasks/__init__.py
Normal file
262
services/worker-gpu/app/tasks/plagiarism.py
Normal file
262
services/worker-gpu/app/tasks/plagiarism.py
Normal file
@@ -0,0 +1,262 @@
|
||||
"""Celery задачи проверки плагиата (уровни 3 и 4) и построения эмбеддингов."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.celery_app import celery_app
|
||||
from app.db import db_session
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Пороговые значения
|
||||
FAISS_SIMILARITY_THRESHOLD = 0.75 # Минимальный cosine score для подозрительного совпадения
|
||||
LLM_CONFIDENCE_THRESHOLD = 0.7 # Минимальная уверенность LLM
|
||||
|
||||
|
||||
def _get_doc_text(doc_id: int) -> str | None:
|
||||
"""Получить текст документа из базы данных."""
|
||||
from app.models import Document
|
||||
with db_session() as session:
|
||||
doc = session.get(Document, doc_id)
|
||||
if doc is None:
|
||||
return None
|
||||
return doc.abstract or ""
|
||||
|
||||
|
||||
def _get_doc_meta(doc_id: int) -> dict | None:
|
||||
"""Получить метаданные документа из базы данных."""
|
||||
from app.models import Document
|
||||
with db_session() as session:
|
||||
doc = session.get(Document, doc_id)
|
||||
if doc is None:
|
||||
return None
|
||||
return {
|
||||
"id": doc.id,
|
||||
"title": doc.title,
|
||||
"url": doc.url,
|
||||
"source": doc.source,
|
||||
"authors": doc.authors,
|
||||
"year": doc.year,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="gpu.check_plagiarism",
|
||||
bind=True,
|
||||
max_retries=2,
|
||||
default_retry_delay=120,
|
||||
)
|
||||
def check_plagiarism(
|
||||
self,
|
||||
task_id: str,
|
||||
text: str,
|
||||
fragments: list[dict],
|
||||
level1_matches: list[dict] | None = None,
|
||||
level2_matches: list[dict] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Проверка плагиата уровни 3 (FAISS семантика) и 4 (Ollama LLM).
|
||||
|
||||
Принимает результаты уровней 1 и 2 от worker-indexer и дополняет их.
|
||||
|
||||
Args:
|
||||
task_id: ID задачи в PostgreSQL
|
||||
text: Полный текст документа
|
||||
fragments: Список фрагментов для проверки:
|
||||
[{"text": str, "start": int, "end": int}, ...]
|
||||
level1_matches: Совпадения уровня 1 (Winnowing)
|
||||
level2_matches: Совпадения уровня 2 (MinHash)
|
||||
|
||||
Returns:
|
||||
dict с результатами проверки
|
||||
"""
|
||||
from app.models import Task
|
||||
|
||||
logger.info(
|
||||
f"Проверка плагиата (ур. 3-4) для задачи {task_id!r}. "
|
||||
f"Фрагментов: {len(fragments)}"
|
||||
)
|
||||
|
||||
level1_matches = level1_matches or []
|
||||
level2_matches = level2_matches or []
|
||||
|
||||
# Обновить статус
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task is None:
|
||||
logger.error(f"Задача {task_id!r} не найдена")
|
||||
return {}
|
||||
task.status = "processing"
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
from app.faiss_manager import FAISSManager
|
||||
from app.model_manager import ModelManager
|
||||
from app.ollama_client import OllamaClient
|
||||
|
||||
ollama = OllamaClient()
|
||||
semantic_matches: list[dict] = []
|
||||
|
||||
for i, fragment in enumerate(fragments):
|
||||
frag_text = fragment.get("text", "")
|
||||
if len(frag_text.split()) < 10:
|
||||
# Пропустить слишком короткие фрагменты
|
||||
continue
|
||||
|
||||
# Уровень 3: Семантический поиск через FAISS
|
||||
frag_vec = ModelManager.encode_single(frag_text)
|
||||
faiss_results = FAISSManager.search(frag_vec, k=10)
|
||||
|
||||
for doc_id, score in faiss_results:
|
||||
if score < FAISS_SIMILARITY_THRESHOLD:
|
||||
continue
|
||||
|
||||
# Уровень 4: LLM анализ парафраза для подозрительных совпадений
|
||||
source_text = _get_doc_text(doc_id) or ""
|
||||
doc_meta = _get_doc_meta(doc_id)
|
||||
|
||||
if not doc_meta:
|
||||
continue
|
||||
|
||||
llm_result = {"is_paraphrase": False, "confidence": 0.0, "reason": ""}
|
||||
if source_text:
|
||||
llm_result = ollama.check_paraphrase(source_text, frag_text)
|
||||
|
||||
if llm_result.get("is_paraphrase") and llm_result.get("confidence", 0.0) >= LLM_CONFIDENCE_THRESHOLD:
|
||||
semantic_matches.append({
|
||||
"fragment": frag_text[:300],
|
||||
"position_start": fragment.get("start", 0),
|
||||
"position_end": fragment.get("end", len(frag_text)),
|
||||
"similarity": round(score * 100, 1),
|
||||
"method": "semantic+llm",
|
||||
"confidence": llm_result["confidence"],
|
||||
"reason": llm_result.get("reason", ""),
|
||||
"source_title": doc_meta["title"],
|
||||
"source_url": doc_meta["url"],
|
||||
"source_db": doc_meta["source"],
|
||||
})
|
||||
|
||||
if (i + 1) % 10 == 0:
|
||||
logger.info(f"Проверено фрагментов: {i + 1}/{len(fragments)}")
|
||||
|
||||
# Объединить все совпадения и дедуплицировать по source_title
|
||||
all_matches = level1_matches + level2_matches + semantic_matches
|
||||
seen_sources: set[str] = set()
|
||||
unique_matches = []
|
||||
for m in all_matches:
|
||||
key = f"{m.get('source_title', '')}:{m.get('position_start', '')}"
|
||||
if key not in seen_sources:
|
||||
seen_sources.add(key)
|
||||
unique_matches.append(m)
|
||||
|
||||
# Вычислить общий процент схожести
|
||||
total_frags = len(fragments)
|
||||
flagged_frags = len(unique_matches)
|
||||
overall_similarity = (flagged_frags / total_frags * 100) if total_frags > 0 else 0.0
|
||||
|
||||
result = {
|
||||
"overall_similarity": round(overall_similarity, 2),
|
||||
"matches": unique_matches,
|
||||
"total_fragments": total_frags,
|
||||
"flagged_fragments": flagged_frags,
|
||||
"by_method": {
|
||||
"exact": len(level1_matches),
|
||||
"fuzzy": len(level2_matches),
|
||||
"semantic_llm": len(semantic_matches),
|
||||
},
|
||||
}
|
||||
|
||||
# Сохранить результат
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.result = result
|
||||
task.status = "done"
|
||||
task.queue_position = None
|
||||
session.commit()
|
||||
|
||||
logger.info(
|
||||
f"Проверка плагиата завершена для задачи {task_id!r}. "
|
||||
f"Схожесть: {overall_similarity:.1f}%, совпадений: {flagged_frags}"
|
||||
)
|
||||
|
||||
# Уведомить пользователя
|
||||
celery_app.send_task(
|
||||
"notify.send_task_done",
|
||||
args=[task_id],
|
||||
queue="queue.notify",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"Ошибка проверки плагиата для задачи {task_id!r}: {exc}", exc_info=True)
|
||||
|
||||
try:
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.status = "failed"
|
||||
task.error = str(exc)
|
||||
session.commit()
|
||||
except Exception as db_exc:
|
||||
logger.error(f"Не удалось обновить статус задачи: {db_exc}")
|
||||
|
||||
raise self.retry(exc=exc, countdown=120)
|
||||
|
||||
|
||||
@celery_app.task(name="gpu.embed_documents")
|
||||
def embed_documents(doc_ids: list[int]) -> dict[str, Any]:
|
||||
"""
|
||||
Построить эмбеддинги для документов и добавить их в FAISS индекс.
|
||||
|
||||
Вызывается воркером-индексером после добавления новых документов.
|
||||
|
||||
Args:
|
||||
doc_ids: Список ID документов в PostgreSQL
|
||||
"""
|
||||
if not doc_ids:
|
||||
return {"status": "ok", "embedded": 0}
|
||||
|
||||
from app.models import Document
|
||||
from app.faiss_manager import FAISSManager
|
||||
from app.model_manager import ModelManager
|
||||
from sqlalchemy import select
|
||||
|
||||
logger.info(f"Построение эмбеддингов для {len(doc_ids)} документов...")
|
||||
|
||||
with db_session() as session:
|
||||
docs = session.execute(
|
||||
select(Document).where(Document.id.in_(doc_ids))
|
||||
).scalars().all()
|
||||
|
||||
if not docs:
|
||||
logger.warning(f"Документы не найдены: {doc_ids}")
|
||||
return {"status": "not_found", "embedded": 0}
|
||||
|
||||
# Формируем тексты: title + abstract
|
||||
texts = [
|
||||
f"{d.title}. {d.abstract or ''}".strip()
|
||||
for d in docs
|
||||
]
|
||||
ids = [d.id for d in docs]
|
||||
|
||||
import numpy as np
|
||||
vectors = ModelManager.encode(texts)
|
||||
|
||||
FAISSManager.add_vectors(vectors, ids)
|
||||
FAISSManager.save()
|
||||
|
||||
# Обновить faiss_id в PostgreSQL
|
||||
with db_session() as session:
|
||||
for doc_id in ids:
|
||||
doc = session.get(Document, doc_id)
|
||||
if doc and doc_id in FAISSManager._reverse_map:
|
||||
doc.faiss_id = FAISSManager._reverse_map[doc_id]
|
||||
session.commit()
|
||||
|
||||
logger.info(f"Встроено и проиндексировано {len(ids)} документов")
|
||||
return {"status": "ok", "embedded": len(ids)}
|
||||
281
services/worker-gpu/app/tasks/search.py
Normal file
281
services/worker-gpu/app/tasks/search.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""Celery задача семантического поиска источников."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.celery_app import celery_app
|
||||
from app.db import db_session, get_redis
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Время жизни кэша поиска (1 час)
|
||||
SEARCH_CACHE_TTL = 3600
|
||||
|
||||
# Минимальный cosine score для включения в результаты
|
||||
FAISS_MIN_SCORE = 0.3
|
||||
|
||||
# Вес BM25 и семантического скора при ранжировании
|
||||
WEIGHT_SEMANTIC = 0.6
|
||||
WEIGHT_BM25 = 0.4
|
||||
|
||||
|
||||
def _get_cache_key(query: str, lang: str | None, year_from: int | None, year_to: int | None) -> str:
|
||||
"""Получить ключ кэша для поискового запроса."""
|
||||
key_str = f"{query}:{lang}:{year_from}:{year_to}"
|
||||
return f"search:{hashlib.md5(key_str.encode()).hexdigest()}"
|
||||
|
||||
|
||||
def _merge_and_rank(
|
||||
faiss_results: list[tuple[int, float]],
|
||||
es_results: list[dict],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Объединить и ранжировать результаты FAISS и Elasticsearch.
|
||||
|
||||
Нормализует оба скора в диапазон [0, 1] и взвешенно суммирует.
|
||||
|
||||
Args:
|
||||
faiss_results: [(doc_id, cosine_score), ...] из FAISS
|
||||
es_results: [{"doc_id": int, "es_score": float}, ...] из ES
|
||||
|
||||
Returns:
|
||||
Список словарей с doc_id и combined_score, отсортированный по убыванию
|
||||
"""
|
||||
scores: dict[int, dict[str, float]] = {}
|
||||
|
||||
# Нормализация FAISS скоров
|
||||
if faiss_results:
|
||||
max_faiss = max(s for _, s in faiss_results) or 1.0
|
||||
for doc_id, score in faiss_results:
|
||||
scores[doc_id] = {"faiss": score / max_faiss, "es": 0.0}
|
||||
|
||||
# Нормализация BM25 скоров
|
||||
if es_results:
|
||||
max_es = max(r["es_score"] for r in es_results) or 1.0
|
||||
for r in es_results:
|
||||
doc_id = r["doc_id"]
|
||||
norm_score = r["es_score"] / max_es
|
||||
if doc_id in scores:
|
||||
scores[doc_id]["es"] = norm_score
|
||||
else:
|
||||
scores[doc_id] = {"faiss": 0.0, "es": norm_score}
|
||||
|
||||
# Комбинированный скор
|
||||
ranked = []
|
||||
for doc_id, s in scores.items():
|
||||
combined = WEIGHT_SEMANTIC * s["faiss"] + WEIGHT_BM25 * s["es"]
|
||||
ranked.append({
|
||||
"doc_id": doc_id,
|
||||
"combined_score": round(combined, 4),
|
||||
"faiss_score": round(s["faiss"], 4),
|
||||
"es_score": round(s["es"], 4),
|
||||
})
|
||||
|
||||
ranked.sort(key=lambda x: x["combined_score"], reverse=True)
|
||||
return ranked
|
||||
|
||||
|
||||
def _enrich_from_db(ranked: list[dict], session) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Обогатить результаты метаданными из PostgreSQL и добавить ГОСТ-цитату.
|
||||
|
||||
Args:
|
||||
ranked: Список с doc_id и скорами
|
||||
session: SQLAlchemy сессия
|
||||
|
||||
Returns:
|
||||
Список источников с полными метаданными
|
||||
"""
|
||||
from app.models import Document
|
||||
|
||||
if not ranked:
|
||||
return []
|
||||
|
||||
doc_ids = [r["doc_id"] for r in ranked]
|
||||
docs = session.execute(
|
||||
select(Document).where(Document.id.in_(doc_ids))
|
||||
).scalars().all()
|
||||
|
||||
doc_map = {d.id: d for d in docs}
|
||||
|
||||
results = []
|
||||
for r in ranked:
|
||||
doc = doc_map.get(r["doc_id"])
|
||||
if not doc:
|
||||
continue
|
||||
|
||||
# Простая ГОСТ-цитата (полная форматирует worker-gost)
|
||||
gost_citation = _format_gost_simple(doc)
|
||||
|
||||
results.append({
|
||||
"id": doc.id,
|
||||
"title": doc.title,
|
||||
"authors": doc.authors,
|
||||
"year": doc.year,
|
||||
"journal": doc.journal,
|
||||
"abstract": doc.abstract[:300] if doc.abstract else None,
|
||||
"url": doc.url,
|
||||
"doi": doc.doi,
|
||||
"relevance_score": r["combined_score"],
|
||||
"gost_citation": gost_citation,
|
||||
"source_db": doc.source,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _format_gost_simple(doc) -> str:
|
||||
"""Простое ГОСТ-форматирование (без полной логики worker-gost)."""
|
||||
authors = doc.authors or []
|
||||
author_str = ""
|
||||
if authors:
|
||||
first = authors[0]
|
||||
last_name = first.get("last_name", "")
|
||||
initials = first.get("initials", "")
|
||||
if len(authors) == 1:
|
||||
author_str = f"{last_name} {initials}"
|
||||
elif len(authors) <= 3:
|
||||
parts = [f"{a.get('last_name', '')} {a.get('initials', '')}" for a in authors]
|
||||
author_str = ", ".join(parts)
|
||||
else:
|
||||
author_str = f"{last_name} {initials} и др."
|
||||
|
||||
title = doc.title or ""
|
||||
year = str(doc.year) if doc.year else "б. г."
|
||||
journal = doc.journal or ""
|
||||
pages = doc.pages or ""
|
||||
|
||||
if journal:
|
||||
citation = f"{author_str}. {title} // {journal}. — {year}."
|
||||
if pages:
|
||||
citation += f" — С. {pages}."
|
||||
else:
|
||||
citation = f"{author_str}. {title}. — {year}."
|
||||
|
||||
return citation.strip()
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="gpu.search_semantic",
|
||||
bind=True,
|
||||
max_retries=3,
|
||||
default_retry_delay=60,
|
||||
)
|
||||
def search_semantic(
|
||||
self,
|
||||
task_id: str,
|
||||
query: str,
|
||||
lang: str | None = None,
|
||||
year_from: int | None = None,
|
||||
year_to: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Семантический поиск источников.
|
||||
|
||||
Алгоритм:
|
||||
1. Нормализация запроса
|
||||
2. Проверка Redis кэша
|
||||
3. Параллельно: FAISS GPU cosine + Elasticsearch BM25
|
||||
4. Merge + дедупликация + ранжирование
|
||||
5. Обогащение метаданными из PostgreSQL
|
||||
6. Сохранение результата в task.result
|
||||
7. Диспатч notify.send_task_done
|
||||
|
||||
Args:
|
||||
task_id: ID задачи в PostgreSQL
|
||||
query: Поисковый запрос
|
||||
lang: Фильтр языка
|
||||
year_from: Фильтр года (от)
|
||||
year_to: Фильтр года (до)
|
||||
"""
|
||||
from app.models import Task
|
||||
|
||||
logger.info(f"Начало поиска для задачи {task_id!r}, запрос: {query[:50]!r}")
|
||||
|
||||
# Обновить статус на processing
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task is None:
|
||||
logger.error(f"Задача {task_id!r} не найдена в БД")
|
||||
return {}
|
||||
task.status = "processing"
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
redis = get_redis()
|
||||
cache_key = _get_cache_key(query, lang, year_from, year_to)
|
||||
|
||||
# Проверить кэш
|
||||
cached = redis.get(cache_key)
|
||||
if cached:
|
||||
logger.info(f"Результат поиска получен из кэша: {cache_key}")
|
||||
results = json.loads(cached)
|
||||
else:
|
||||
# Нормализация запроса
|
||||
query_normalized = query.strip()
|
||||
|
||||
# Закодировать запрос в вектор
|
||||
from app.model_manager import ModelManager
|
||||
query_vec = ModelManager.encode_single(query_normalized)
|
||||
|
||||
# FAISS семантический поиск
|
||||
from app.faiss_manager import FAISSManager
|
||||
faiss_results = FAISSManager.search(query_vec, k=50)
|
||||
logger.info(f"FAISS: найдено {len(faiss_results)} результатов")
|
||||
|
||||
# Elasticsearch BM25 поиск
|
||||
from app.es_client import search_fulltext
|
||||
es_results = search_fulltext(query_normalized, lang=lang, year_from=year_from, year_to=year_to)
|
||||
logger.info(f"ES BM25: найдено {len(es_results)} результатов")
|
||||
|
||||
# Объединить и ранжировать
|
||||
ranked = _merge_and_rank(faiss_results, es_results)
|
||||
|
||||
# Обогатить метаданными
|
||||
with db_session() as session:
|
||||
results = _enrich_from_db(ranked[:30], session) # Топ-30 результатов
|
||||
|
||||
# Кэшировать на 1 час
|
||||
redis.setex(cache_key, SEARCH_CACHE_TTL, json.dumps(results, ensure_ascii=False, default=str))
|
||||
|
||||
# Сохранить результат в задачу
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.result = {"sources": results, "total": len(results), "query": query}
|
||||
task.status = "done"
|
||||
task.queue_position = None
|
||||
session.commit()
|
||||
|
||||
logger.info(f"Задача {task_id!r} выполнена. Найдено {len(results)} источников.")
|
||||
|
||||
# Уведомить пользователя
|
||||
celery_app.send_task(
|
||||
"notify.send_task_done",
|
||||
args=[task_id],
|
||||
queue="queue.notify",
|
||||
)
|
||||
|
||||
return {"task_id": task_id, "total": len(results)}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"Ошибка при выполнении поиска для задачи {task_id!r}: {exc}", exc_info=True)
|
||||
|
||||
# Обновить статус задачи на failed
|
||||
try:
|
||||
with db_session() as session:
|
||||
task = session.get(Task, task_id)
|
||||
if task:
|
||||
task.status = "failed"
|
||||
task.error = str(exc)
|
||||
session.commit()
|
||||
except Exception as db_exc:
|
||||
logger.error(f"Не удалось обновить статус задачи: {db_exc}")
|
||||
|
||||
# Повторить попытку
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
13
services/worker-gpu/requirements.txt
Normal file
13
services/worker-gpu/requirements.txt
Normal file
@@ -0,0 +1,13 @@
|
||||
celery==5.4.0
|
||||
redis==5.0.4
|
||||
sqlalchemy==2.0.30
|
||||
psycopg2-binary==2.9.9
|
||||
sentence-transformers==3.0.0
|
||||
faiss-gpu==1.7.4
|
||||
torch==2.3.0
|
||||
numpy==1.26.4
|
||||
httpx==0.27.0
|
||||
minio==7.2.7
|
||||
datasketch==1.6.5
|
||||
pydantic-settings==2.2.1
|
||||
elasticsearch==8.13.0
|
||||
20
services/worker-indexer/Dockerfile
Normal file
20
services/worker-indexer/Dockerfile
Normal file
@@ -0,0 +1,20 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Системные зависимости для PDF обработки
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
libpq-dev \
|
||||
libmupdf-dev \
|
||||
mupdf-tools \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["celery", "-A", "app.celery_app", "worker", "-Q", "queue.index", "-c", "4", "-n", "indexer@%h", "--loglevel=info"]
|
||||
0
services/worker-indexer/app/__init__.py
Normal file
0
services/worker-indexer/app/__init__.py
Normal file
0
services/worker-indexer/app/algorithms/__init__.py
Normal file
0
services/worker-indexer/app/algorithms/__init__.py
Normal file
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user