feat: админ-панель, лицензия, тестовый стек на распределённой инфраструктуре

Админ-панель (доступ: is_admin + секретный код сессии в /admin/<code>):
- разделы: дашборд (здоровье сервисов, статистика), клиенты, работы,
  база документов, хранилище MinIO, источники парсинга, отстойник работ
- backend: модели ParseSource/StagedWork/AdminSession, миграция 003,
  core/admin (авторизация), роутер api/admin
- frontend: AdminLayout + 7 вкладок, adminApi, вход из кабинета
- Celery index.run_parser (фоновый парсинг), автосбор проверенных работ
  в отстойник с ручным одобрением → индексация в базу

Исправления:
- openalex parser: тип article (не journal-article), убран is_oa-фильтр
- winnowing: приведение xxh64 к signed int64 (фикс bigint out of range)
- bcrypt пин 4.0.1 (совместимость с passlib 1.7.4)
- alembic: prepend_sys_path + asyncpg-драйвер
- frontend: контракт Task (public_id вместо id), react-dropzone

Инфраструктура:
- docker-compose.test.yml: api+frontend+воркеры локально, БД/кэш/хранилище/
  брокер/LLM — на удалённых серверах через .env
- .gitignore: исправлено ошибочное игнорирование кода services/*/app/models/
- LICENSE: проприетарная

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-05-30 20:45:53 +05:00
parent 9ee5ccdc3f
commit 9894fa9320
50 changed files with 5680 additions and 58 deletions

View File

@@ -4,6 +4,9 @@
# Путь к директории с миграциями
script_location = alembic
# Добавить корень проекта (.) в sys.path, чтобы импортировался пакет app
prepend_sys_path = .
# Формат имени файла миграции
file_template = %%(rev)s_%%(slug)s

View File

@@ -30,7 +30,7 @@ def get_url() -> str:
db = os.getenv("POSTGRES_DB", "antiplagiator")
user = os.getenv("POSTGRES_USER", "antiplagiator")
password = os.getenv("POSTGRES_PASSWORD", "changeme")
return f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{db}"
return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{db}"
def run_migrations_offline() -> None:

View File

@@ -0,0 +1,88 @@
"""Админ-панель: is_admin, источники парсинга, отстойник, сессии админа.
Revision ID: 003
Revises: 002
Create Date: 2026-05-30
"""
from alembic import op
import sqlalchemy as sa
revision = "003"
down_revision = "002"
branch_labels = None
depends_on = None
def upgrade() -> None:
# users.is_admin
op.add_column(
"users",
sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.false()),
)
# parse_sources
op.create_table(
"parse_sources",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("source_type", sa.String(50), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("query", sa.Text(), nullable=True),
sa.Column("lang", sa.String(10), nullable=True),
sa.Column("year_from", sa.Integer(), nullable=True),
sa.Column("year_to", sa.Integer(), nullable=True),
sa.Column("limit", sa.Integer(), nullable=False, server_default="1000"),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("last_status", sa.String(20), nullable=False, server_default="idle"),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("last_run_at", sa.DateTime(), nullable=True),
sa.Column("docs_added", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
)
op.create_index("ix_parse_sources_source_type", "parse_sources", ["source_type"])
# staged_works
op.create_table(
"staged_works",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=True),
sa.Column("task_id", sa.String(36), nullable=True),
sa.Column("filename", sa.Text(), nullable=True),
sa.Column("minio_key", sa.Text(), nullable=True),
sa.Column("text_key", sa.Text(), nullable=True),
sa.Column("title", sa.Text(), nullable=True),
sa.Column("authors", sa.JSON(), nullable=True),
sa.Column("year", sa.Integer(), nullable=True),
sa.Column("lang", sa.String(10), nullable=True),
sa.Column("word_count", sa.Integer(), nullable=True),
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
sa.Column("document_id", sa.Integer(), nullable=True),
sa.Column("reviewed_by", sa.Integer(), nullable=True),
sa.Column("reviewed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
)
op.create_index("ix_staged_works_user_id", "staged_works", ["user_id"])
op.create_index("ix_staged_works_task_id", "staged_works", ["task_id"])
op.create_index("ix_staged_works_status", "staged_works", ["status"])
op.create_index("ix_staged_works_created_at", "staged_works", ["created_at"])
# admin_sessions
op.create_table(
"admin_sessions",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=False),
sa.Column("code", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
sa.Column("expires_at", sa.DateTime(), nullable=False),
sa.Column("last_used_at", sa.DateTime(), nullable=True),
)
op.create_index("ix_admin_sessions_user_id", "admin_sessions", ["user_id"])
op.create_index("ix_admin_sessions_code", "admin_sessions", ["code"], unique=True)
def downgrade() -> None:
op.drop_table("admin_sessions")
op.drop_table("staged_works")
op.drop_index("ix_parse_sources_source_type", table_name="parse_sources")
op.drop_table("parse_sources")
op.drop_column("users", "is_admin")

View File

@@ -0,0 +1,571 @@
"""Админ-панель: дашборд, клиенты, работы, документы, хранилище, источники, отстойник.
Все эндпоинты требуют is_admin (get_admin_user). Эндпоинты под /admin/panel/{code}/*
дополнительно проверяют секретный код сессии (verify_admin_code).
"""
import logging
from datetime import datetime, timezone
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.core.admin import create_admin_session, get_admin_user
from app.core.celery_app import celery_app
from app.core.minio_client import get_minio
from app.core.redis_client import get_redis
from app.database import get_db
from app.models.admin import ParseSource, StagedWork
from app.models.document import Document, Fingerprint
from app.models.task import Task
from app.models.user import User
from app.schemas.admin import (
AdminDocumentResponse,
AdminSessionResponse,
AdminStats,
AdminTaskResponse,
AdminUserResponse,
AdminUserUpdate,
ParseSourceCreate,
ParseSourceResponse,
ParseSourceUpdate,
ServiceHealth,
StagedWorkDetail,
StagedWorkResponse,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin", tags=["admin"], dependencies=[Depends(get_admin_user)])
# ═══════════════════════════════════════════════════════════════════════════════
# СЕССИЯ ДОСТУПА
# ═══════════════════════════════════════════════════════════════════════════════
@router.post("/session", response_model=AdminSessionResponse)
async def open_admin_session(
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> AdminSessionResponse:
"""Сгенерировать секретный код доступа и ссылку /admin/<code>."""
code = await create_admin_session(admin, db)
from app.models.admin import AdminSession
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
sess = result.scalar_one()
return AdminSessionResponse(
code=code,
url=f"/admin/{code}",
expires_at=sess.expires_at,
)
@router.get("/session/verify/{code}")
async def verify_session(
code: str,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Проверить валидность кода (для фронта при загрузке /admin/<code>)."""
from app.models.admin import AdminSession
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
sess = result.scalar_one_or_none()
if sess is None or sess.user_id != admin.id:
return {"valid": False}
expires = sess.expires_at
if expires.tzinfo is not None:
expires = expires.replace(tzinfo=None)
return {"valid": expires >= datetime.utcnow()}
# ═══════════════════════════════════════════════════════════════════════════════
# ДАШБОРД / СЕРВИСЫ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/health", response_model=list[ServiceHealth])
async def health(db: AsyncSession = Depends(get_db)) -> list[ServiceHealth]:
"""Статус всех сервисов инфраструктуры."""
checks: list[ServiceHealth] = []
# PostgreSQL
try:
await db.execute(select(func.now()))
checks.append(ServiceHealth(name="PostgreSQL", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="PostgreSQL", ok=False, detail=str(e)[:200]))
# Redis
try:
r = get_redis()
await r.ping()
checks.append(ServiceHealth(name="Redis", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="Redis", ok=False, detail=str(e)[:200]))
# MinIO
try:
get_minio().bucket_exists(settings.MINIO_BUCKET_DOCS)
checks.append(ServiceHealth(name="MinIO", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="MinIO", ok=False, detail=str(e)[:200]))
# Elasticsearch
try:
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(f"{settings.ELASTICSEARCH_URL}/_cluster/health")
ok = resp.status_code == 200 and resp.json().get("status") != "red"
checks.append(ServiceHealth(name="Elasticsearch", ok=ok,
detail=resp.json().get("status")))
except Exception as e:
checks.append(ServiceHealth(name="Elasticsearch", ok=False, detail=str(e)[:200]))
# Ollama
try:
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(f"{settings.OLLAMA_URL}/api/version")
checks.append(ServiceHealth(name="Ollama", ok=resp.status_code == 200,
detail=resp.json().get("version")))
except Exception as e:
checks.append(ServiceHealth(name="Ollama", ok=False, detail=str(e)[:200]))
# RabbitMQ + воркеры через Celery ping
try:
insp = celery_app.control.inspect(timeout=3)
pong = insp.ping() or {}
workers = list(pong.keys())
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=bool(workers),
detail=f"воркеров онлайн: {len(workers)}"))
except Exception as e:
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=False, detail=str(e)[:200]))
return checks
@router.get("/stats", response_model=AdminStats)
async def stats(db: AsyncSession = Depends(get_db)) -> AdminStats:
"""Сводные счётчики для дашборда."""
users_total = (await db.execute(select(func.count()).select_from(User))).scalar_one()
rows = (await db.execute(select(Task.status, func.count()).group_by(Task.status))).all()
tasks_by_status = {s: c for s, c in rows}
docs_total = (await db.execute(select(func.count()).select_from(Document))).scalar_one()
rows = (await db.execute(select(Document.source, func.count()).group_by(Document.source))).all()
docs_by_source = {s: c for s, c in rows}
staging_pending = (
await db.execute(
select(func.count()).select_from(StagedWork).where(StagedWork.status == "pending")
)
).scalar_one()
# Хранилище MinIO
storage: dict = {}
try:
client = get_minio()
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
try:
if not client.bucket_exists(bucket):
continue
total = 0
count = 0
for obj in client.list_objects(bucket, recursive=True):
total += obj.size or 0
count += 1
storage[bucket] = {"objects": count, "bytes": total}
except Exception:
continue
except Exception as e:
storage = {"error": str(e)[:200]}
# Длины очередей через RabbitMQ management API (если доступно)
queues: dict = {}
try:
rmq_user = settings.RABBITMQ_URL.split("//")[1].split(":")[0]
rmq_pass = settings.RABBITMQ_URL.split(":")[2].split("@")[0]
rmq_host = settings.RABBITMQ_URL.split("@")[1].split(":")[0]
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(
f"http://{rmq_host}:15672/api/queues",
auth=(rmq_user, rmq_pass),
)
if resp.status_code == 200:
for q in resp.json():
name = q.get("name", "")
if name.startswith("queue."):
queues[name] = q.get("messages", 0)
except Exception as e:
queues = {"error": str(e)[:120]}
return AdminStats(
users_total=users_total,
tasks_by_status=tasks_by_status,
documents_total=docs_total,
documents_by_source=docs_by_source,
staging_pending=staging_pending,
storage=storage,
queues=queues,
)
# ═══════════════════════════════════════════════════════════════════════════════
# КЛИЕНТЫ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/users", response_model=list[AdminUserResponse])
async def list_users(
q: str | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminUserResponse]:
stmt = select(User)
if q:
like = f"%{q.lower()}%"
stmt = stmt.where(func.lower(User.email).like(like) | func.lower(User.name).like(like))
stmt = stmt.order_by(User.created_at.desc()).limit(limit).offset(offset)
users = (await db.execute(stmt)).scalars().all()
result = []
for u in users:
cnt = (
await db.execute(select(func.count()).select_from(Task).where(Task.user_id == u.id))
).scalar_one()
item = AdminUserResponse.model_validate(u)
item.tasks_count = cnt
result.append(item)
return result
@router.patch("/users/{user_id}", response_model=AdminUserResponse)
async def update_user(
user_id: int,
data: AdminUserUpdate,
db: AsyncSession = Depends(get_db),
) -> AdminUserResponse:
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
if user is None:
raise HTTPException(status_code=404, detail="Пользователь не найден")
if data.plan is not None:
user.plan = data.plan
if data.is_verified is not None:
user.is_verified = data.is_verified
if data.is_admin is not None:
user.is_admin = data.is_admin
await db.commit()
await db.refresh(user)
# Сбросить кэш пользователя
try:
await get_redis().delete(f"user:cache:{user_id}")
except Exception:
pass
return AdminUserResponse.model_validate(user)
@router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_user(user_id: int, db: AsyncSession = Depends(get_db)) -> None:
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
if user is None:
raise HTTPException(status_code=404, detail="Пользователь не найден")
await db.execute(delete(Task).where(Task.user_id == user_id))
await db.delete(user)
await db.commit()
# ═══════════════════════════════════════════════════════════════════════════════
# РАБОТЫ (задачи)
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/tasks", response_model=list[AdminTaskResponse])
async def list_tasks(
status_filter: str | None = Query(None, alias="status"),
type_filter: str | None = Query(None, alias="type"),
user_id: int | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminTaskResponse]:
stmt = select(Task)
if status_filter:
stmt = stmt.where(Task.status == status_filter)
if type_filter:
stmt = stmt.where(Task.type == type_filter)
if user_id:
stmt = stmt.where(Task.user_id == user_id)
stmt = stmt.order_by(Task.created_at.desc()).limit(limit).offset(offset)
tasks = (await db.execute(stmt)).scalars().all()
return [AdminTaskResponse.model_validate(t) for t in tasks]
@router.get("/tasks/{public_id}")
async def get_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
return {
"public_id": task.public_id,
"user_id": task.user_id,
"type": task.type,
"status": task.status,
"input_data": task.input_data,
"result": task.result,
"error": task.error,
"created_at": task.created_at,
}
@router.post("/tasks/{public_id}/requeue")
async def requeue_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
if task.type != "plagiarism":
raise HTTPException(status_code=400, detail="Перезапуск поддержан только для проверок плагиата")
minio_key = (task.input_data or {}).get("minio_key")
filename = (task.input_data or {}).get("filename")
if not minio_key:
raise HTTPException(status_code=400, detail="Нет minio_key для перезапуска")
task.status = "queued"
task.error = None
task.result = None
await db.commit()
celery_app.send_task(
"index.extract_and_check",
args=[task.id, minio_key, filename],
queue="queue.index",
)
return {"status": "requeued", "public_id": public_id}
@router.delete("/tasks/{public_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_task(public_id: str, db: AsyncSession = Depends(get_db)) -> None:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
await db.delete(task)
await db.commit()
# ═══════════════════════════════════════════════════════════════════════════════
# БАЗА ДОКУМЕНТОВ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/documents", response_model=list[AdminDocumentResponse])
async def list_documents(
q: str | None = None,
source: str | None = None,
year: int | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminDocumentResponse]:
stmt = select(Document)
if q:
stmt = stmt.where(func.lower(Document.title).like(f"%{q.lower()}%"))
if source:
stmt = stmt.where(Document.source == source)
if year:
stmt = stmt.where(Document.year == year)
stmt = stmt.order_by(Document.indexed_at.desc()).limit(limit).offset(offset)
docs = (await db.execute(stmt)).scalars().all()
return [AdminDocumentResponse.model_validate(d) for d in docs]
@router.delete("/documents/{doc_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_document(doc_id: int, db: AsyncSession = Depends(get_db)) -> None:
doc = (await db.execute(select(Document).where(Document.id == doc_id))).scalar_one_or_none()
if doc is None:
raise HTTPException(status_code=404, detail="Документ не найден")
await db.execute(delete(Fingerprint).where(Fingerprint.doc_id == doc_id))
await db.delete(doc)
await db.commit()
# Удалить из Elasticsearch (best-effort)
try:
async with httpx.AsyncClient(timeout=5) as c:
await c.delete(f"{settings.ELASTICSEARCH_URL}/documents/_doc/{doc_id}")
except Exception:
pass
# ═══════════════════════════════════════════════════════════════════════════════
# ХРАНИЛИЩЕ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/storage")
async def storage_info() -> dict:
client = get_minio()
result: dict = {"buckets": []}
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
try:
if not client.bucket_exists(bucket):
continue
total = 0
objs = []
for obj in client.list_objects(bucket, recursive=True):
total += obj.size or 0
if len(objs) < 100:
objs.append({"name": obj.object_name, "size": obj.size})
result["buckets"].append({
"name": bucket,
"objects_total": len(objs),
"bytes": total,
"sample": objs,
})
except Exception as e:
result["buckets"].append({"name": bucket, "error": str(e)[:120]})
return result
# ═══════════════════════════════════════════════════════════════════════════════
# ИСТОЧНИКИ ПАРСИНГА
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/sources", response_model=list[ParseSourceResponse])
async def list_sources(db: AsyncSession = Depends(get_db)) -> list[ParseSourceResponse]:
rows = (await db.execute(select(ParseSource).order_by(ParseSource.created_at.desc()))).scalars().all()
return [ParseSourceResponse.model_validate(s) for s in rows]
@router.post("/sources", response_model=ParseSourceResponse, status_code=201)
async def create_source(data: ParseSourceCreate, db: AsyncSession = Depends(get_db)) -> ParseSourceResponse:
if data.source_type not in ("openalex", "cyberleninka", "arxiv"):
raise HTTPException(status_code=400, detail="Недопустимый тип источника")
src = ParseSource(**data.model_dump())
db.add(src)
await db.commit()
await db.refresh(src)
return ParseSourceResponse.model_validate(src)
@router.patch("/sources/{source_id}", response_model=ParseSourceResponse)
async def update_source(
source_id: int, data: ParseSourceUpdate, db: AsyncSession = Depends(get_db)
) -> ParseSourceResponse:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
for k, v in data.model_dump(exclude_unset=True).items():
setattr(src, k, v)
await db.commit()
await db.refresh(src)
return ParseSourceResponse.model_validate(src)
@router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_source(source_id: int, db: AsyncSession = Depends(get_db)) -> None:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
await db.delete(src)
await db.commit()
@router.post("/sources/{source_id}/run")
async def run_source(source_id: int, db: AsyncSession = Depends(get_db)) -> dict:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
if src.last_status == "running":
raise HTTPException(status_code=409, detail="Источник уже парсится")
src.last_status = "running"
src.last_error = None
src.last_run_at = datetime.utcnow()
await db.commit()
celery_app.send_task("index.run_parser", args=[source_id], queue="queue.index")
return {"status": "started", "source_id": source_id}
# ═══════════════════════════════════════════════════════════════════════════════
# ОТСТОЙНИК
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/staging", response_model=list[StagedWorkResponse])
async def list_staging(
status_filter: str = Query("pending", alias="status"),
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[StagedWorkResponse]:
stmt = select(StagedWork)
if status_filter != "all":
stmt = stmt.where(StagedWork.status == status_filter)
stmt = stmt.order_by(StagedWork.created_at.desc()).limit(limit).offset(offset)
rows = (await db.execute(stmt)).scalars().all()
return [StagedWorkResponse.model_validate(s) for s in rows]
@router.get("/staging/{staged_id}", response_model=StagedWorkDetail)
async def get_staging(staged_id: int, db: AsyncSession = Depends(get_db)) -> StagedWorkDetail:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
detail = StagedWorkDetail.model_validate(sw)
# Превью текста из MinIO
if sw.text_key:
try:
obj = get_minio().get_object("staging", sw.text_key)
text = obj.read().decode("utf-8", errors="replace")
detail.text_preview = text[:5000]
except Exception as e:
detail.text_preview = f"(не удалось прочитать текст: {e})"
return detail
@router.post("/staging/{staged_id}/approve")
async def approve_staging(
staged_id: int,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
if sw.status != "pending":
raise HTTPException(status_code=409, detail=f"Работа уже {sw.status}")
# Прочитать извлечённый текст
full_text = ""
if sw.text_key:
try:
obj = get_minio().get_object("staging", sw.text_key)
full_text = obj.read().decode("utf-8", errors="replace")
except Exception as e:
raise HTTPException(status_code=500, detail=f"Не удалось прочитать текст: {e}")
# Добавить в базу документов через существующую задачу индексатора
doc_data = {
"source": "user_submission",
"ext_id": f"staged:{sw.id}",
"title": sw.title or sw.filename or f"Работа #{sw.id}",
"authors": sw.authors or [],
"year": sw.year,
"lang": sw.lang,
"abstract": full_text[:2000],
"full_text": full_text,
}
celery_app.send_task("index.add_document", args=[doc_data], queue="queue.index")
sw.status = "approved"
sw.reviewed_by = admin.id
sw.reviewed_at = datetime.utcnow()
await db.commit()
return {"status": "approved", "id": staged_id}
@router.post("/staging/{staged_id}/reject")
async def reject_staging(
staged_id: int,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
sw.status = "rejected"
sw.reviewed_by = admin.id
sw.reviewed_at = datetime.utcnow()
await db.commit()
return {"status": "rejected", "id": staged_id}

View File

@@ -0,0 +1,73 @@
"""Авторизация админ-панели: проверка роли is_admin и секретного кода сессии."""
import secrets
from datetime import datetime, timedelta, timezone
from fastapi import Depends, HTTPException, Path, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.database import get_db
from app.models.admin import AdminSession
from app.models.user import User
# Срок жизни секретного кода доступа к админке
ADMIN_SESSION_TTL_HOURS = 8
async def get_admin_user(current_user: User = Depends(get_current_user)) -> User:
"""Dependency: пользователь должен быть аутентифицирован И иметь is_admin."""
if not getattr(current_user, "is_admin", False):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Доступ только для администраторов",
)
return current_user
async def create_admin_session(user: User, db: AsyncSession) -> str:
"""Создать новый секретный код доступа к админке, вернуть код."""
code = secrets.token_urlsafe(32)
# Колонки TIMESTAMP WITHOUT TIME ZONE — храним naive UTC
session = AdminSession(
user_id=user.id,
code=code,
expires_at=datetime.utcnow() + timedelta(hours=ADMIN_SESSION_TTL_HOURS),
)
db.add(session)
await db.commit()
return code
async def verify_admin_code(
code: str = Path(..., description="Секретный код сессии из ссылки /admin/<code>"),
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> User:
"""Dependency: проверяет И роль администратора, И валидность кода из пути.
Двухфакторная защита: ссылка /admin/<code> бесполезна без JWT админа,
а JWT админа без верного кода — тоже.
"""
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
session = result.scalar_one_or_none()
if session is None or session.user_id != admin.id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Неверный код доступа к админке",
)
expires = session.expires_at
if expires.tzinfo is not None:
expires = expires.replace(tzinfo=None)
if expires < datetime.utcnow():
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Код доступа истёк, сгенерируйте новый",
)
session.last_used_at = datetime.utcnow()
await db.commit()
return admin

View File

@@ -92,6 +92,7 @@ async def _load_user(user_id: int, db: AsyncSession):
"name": user.name,
"plan": user.plan,
"is_verified": user.is_verified,
"is_admin": user.is_admin,
}
await r.setex(cache_key, _USER_CACHE_TTL, json.dumps(safe))

View File

@@ -8,7 +8,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api import auth, documents, reports, search, tasks
from app.api import admin, auth, documents, reports, search, tasks
from app.config import settings
from app.core.minio_client import ensure_bucket
from app.core.redis_client import close_pool
@@ -78,6 +78,7 @@ app.include_router(tasks.router, prefix="/api")
app.include_router(search.router, prefix="/api")
app.include_router(documents.router, prefix="/api")
app.include_router(reports.router, prefix="/api")
app.include_router(admin.router, prefix="/api")
# ─── WebSocket ────────────────────────────────────────────────────────────────

View File

@@ -0,0 +1,19 @@
"""Экспорт всех моделей для Alembic автоопределения."""
from app.models.admin import AdminSession, ParseSource, StagedWork
from app.models.document import Document, Fingerprint, UsageLog
from app.models.task import Task, TaskStatus, TaskType
from app.models.user import User
__all__ = [
"User",
"Task",
"TaskType",
"TaskStatus",
"Document",
"Fingerprint",
"UsageLog",
"ParseSource",
"StagedWork",
"AdminSession",
]

View File

@@ -0,0 +1,92 @@
"""Модели для админ-панели: источники парсинга, отстойник работ, сессии админа."""
from datetime import datetime
from sqlalchemy import JSON, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.database import Base
class ParseSource(Base):
"""Источник парсинга для наполнения базы документов.
Админ задаёт тип источника и параметры запроса; задача index.run_parser
запускает парсинг в фоне и наполняет базу.
"""
__tablename__ = "parse_sources"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
# Тип парсера: openalex / cyberleninka / arxiv
source_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
# Человекочитаемое имя
name: Mapped[str] = mapped_column(String(255), nullable=False)
query: Mapped[str | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
year_from: Mapped[int | None] = mapped_column(nullable=True)
year_to: Mapped[int | None] = mapped_column(nullable=True)
limit: Mapped[int] = mapped_column(default=1000)
enabled: Mapped[bool] = mapped_column(default=True)
# Статус последнего запуска: idle / running / done / error
last_status: Mapped[str] = mapped_column(String(20), default="idle")
last_error: Mapped[str | None] = mapped_column(nullable=True)
last_run_at: Mapped[datetime | None] = mapped_column(nullable=True)
docs_added: Mapped[int] = mapped_column(default=0)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
def __repr__(self) -> str:
return f"<ParseSource id={self.id} type={self.source_type!r} name={self.name!r}>"
class StagedWork(Base):
"""Отстойник: проверенная пользовательская работа, ожидающая решения админа.
После одобрения работа добавляется в базу документов (index.add_document),
после отклонения — помечается rejected.
"""
__tablename__ = "staged_works"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True, index=True)
# Внутренний UUID задачи проверки (Task.id)
task_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
filename: Mapped[str | None] = mapped_column(nullable=True)
# Ключ исходного файла и извлечённого текста в MinIO
minio_key: Mapped[str | None] = mapped_column(nullable=True)
text_key: Mapped[str | None] = mapped_column(nullable=True)
title: Mapped[str | None] = mapped_column(nullable=True)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
word_count: Mapped[int | None] = mapped_column(nullable=True)
# pending / approved / rejected
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
# ID документа в базе после одобрения
document_id: Mapped[int | None] = mapped_column(nullable=True)
reviewed_by: Mapped[int | None] = mapped_column(nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
def __repr__(self) -> str:
return f"<StagedWork id={self.id} status={self.status!r} title={(self.title or '')[:30]!r}>"
class AdminSession(Base):
"""Секретный код доступа к админке (часть ссылки /admin/<code>).
Доступ проверяется по двум факторам: JWT админа (is_admin) И валидный код.
"""
__tablename__ = "admin_sessions"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), nullable=False, index=True)
code: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
expires_at: Mapped[datetime] = mapped_column(nullable=False)
last_used_at: Mapped[datetime | None] = mapped_column(nullable=True)
def __repr__(self) -> str:
return f"<AdminSession id={self.id} user_id={self.user_id}>"

View File

@@ -0,0 +1,82 @@
"""Модели документов, fingerprints и логов использования."""
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import JSON, BigInteger, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.user import User
class Document(Base):
"""Документ в базе знаний системы (статья, книга, веб-страница)."""
__tablename__ = "documents"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
# Источник: openalex, cyberleninka, arxiv, wikipedia_ru, wikipedia_en
source: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
# Внешний ID (OpenAlex ID, arXiv ID и т.д.) — уникален в рамках источника
ext_id: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
doi: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
title: Mapped[str] = mapped_column(nullable=False)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True, index=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True, index=True)
journal: Mapped[str | None] = mapped_column(nullable=True)
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
abstract: Mapped[str | None] = mapped_column(nullable=True)
url: Mapped[str | None] = mapped_column(nullable=True)
# Ключ объекта в MinIO (полный текст)
minio_key: Mapped[str | None] = mapped_column(nullable=True)
# ID вектора в FAISS индексе
faiss_id: Mapped[int | None] = mapped_column(nullable=True, index=True)
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
# Отпечатки (Winnowing хэши)
fingerprints: Mapped[list["Fingerprint"]] = relationship(
back_populates="document",
lazy="select",
)
def __repr__(self) -> str:
return f"<Document id={self.id} source={self.source!r} title={self.title[:40]!r}>"
class Fingerprint(Base):
"""Winnowing fingerprint (хэш n-граммы) для быстрого поиска плагиата."""
__tablename__ = "fingerprints"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
doc_id: Mapped[int] = mapped_column(ForeignKey("documents.id"), index=True)
hash_value: Mapped[int] = mapped_column(BigInteger, index=True)
position: Mapped[int] = mapped_column() # Позиция в документе
document: Mapped["Document"] = relationship(back_populates="fingerprints")
def __repr__(self) -> str:
return f"<Fingerprint doc_id={self.doc_id} hash={self.hash_value}>"
class UsageLog(Base):
"""Лог использования для подсчёта лимитов по тарифу."""
__tablename__ = "usage_logs"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
# Действие: search, plagiarism, summarize, gost
action: Mapped[str] = mapped_column(String(50), nullable=False)
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
user: Mapped["User"] = relationship(back_populates="usage")
def __repr__(self) -> str:
return f"<UsageLog user_id={self.user_id} action={self.action!r}>"

View File

@@ -0,0 +1,75 @@
"""Модель задачи (асинхронная единица работы)."""
import secrets
from datetime import datetime
from enum import Enum
from typing import TYPE_CHECKING
from uuid import uuid4
from sqlalchemy import JSON, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.user import User
class TaskType(str, Enum):
"""Тип задачи."""
SEARCH = "search"
PLAGIARISM = "plagiarism"
SUMMARIZE = "summarize"
GOST = "gost"
class TaskStatus(str, Enum):
"""Статус задачи."""
QUEUED = "queued"
PROCESSING = "processing"
DONE = "done"
FAILED = "failed"
class Task(Base):
"""Задача пользователя, обрабатываемая воркерами Celery."""
__tablename__ = "tasks"
# Внутренний UUID — не светится в API/URL
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
# Публичный ID для URL: /tasks/{public_id}
# token_urlsafe(16) → 22 символа base64url, не угадываем, не связан с internal UUID
public_id: Mapped[str] = mapped_column(
String(32),
unique=True,
index=True,
default=lambda: secrets.token_urlsafe(16),
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
type: Mapped[str] = mapped_column(String(20), nullable=False)
status: Mapped[str] = mapped_column(String(20), default=TaskStatus.QUEUED)
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
# Входные данные: query, filename, doc_ids и т.д.
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
# Результат работы воркера
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(nullable=True)
queue_position: Mapped[int | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime | None] = mapped_column(
nullable=True,
onupdate=func.now(),
)
# Связи
user: Mapped["User"] = relationship(back_populates="tasks")
def __repr__(self) -> str:
return f"<Task id={self.id!r} type={self.type!r} status={self.status!r}>"

View File

@@ -0,0 +1,42 @@
"""Модель пользователя."""
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import func, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.task import Task
from app.models.document import UsageLog
class User(Base):
"""Пользователь системы."""
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
email: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
is_verified: Mapped[bool] = mapped_column(default=False)
# Администратор системы (доступ к админ-панели)
is_admin: Mapped[bool] = mapped_column(default=False)
# Тарифный план: free / student / premium / science
plan: Mapped[str] = mapped_column(String(20), default="free")
verification_token: Mapped[str | None] = mapped_column(String(255), nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
server_default=func.now(),
onupdate=func.now(),
)
# Связи
tasks: Mapped[list["Task"]] = relationship(back_populates="user", lazy="select")
usage: Mapped[list["UsageLog"]] = relationship(back_populates="user", lazy="select")
def __repr__(self) -> str:
return f"<User id={self.id} email={self.email!r} plan={self.plan!r}>"

View File

@@ -0,0 +1,141 @@
"""Pydantic-схемы для админ-панели."""
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
# ─── Сессия доступа ───────────────────────────────────────────────────────────
class AdminSessionResponse(BaseModel):
code: str
url: str
expires_at: datetime
# ─── Клиенты ──────────────────────────────────────────────────────────────────
class AdminUserResponse(BaseModel):
id: int
email: str
name: str
plan: str
is_verified: bool
is_admin: bool
created_at: datetime
tasks_count: int = 0
model_config = {"from_attributes": True}
class AdminUserUpdate(BaseModel):
plan: str | None = None
is_verified: bool | None = None
is_admin: bool | None = None
# ─── Работы ─────────────────────────────────────────────────────────────────
class AdminTaskResponse(BaseModel):
public_id: str
user_id: int
type: str
status: str
error: str | None = None
created_at: datetime
model_config = {"from_attributes": True}
# ─── Документы базы ───────────────────────────────────────────────────────────
class AdminDocumentResponse(BaseModel):
id: int
source: str
ext_id: str
title: str
authors: list = Field(default_factory=list)
year: int | None = None
lang: str | None = None
doi: str | None = None
url: str | None = None
indexed_at: datetime
model_config = {"from_attributes": True}
# ─── Источники парсинга ───────────────────────────────────────────────────────
class ParseSourceCreate(BaseModel):
source_type: str = Field(description="openalex / cyberleninka / arxiv")
name: str = Field(min_length=1, max_length=255)
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int = Field(default=1000, ge=1, le=100000)
enabled: bool = True
class ParseSourceUpdate(BaseModel):
name: str | None = None
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int | None = Field(default=None, ge=1, le=100000)
enabled: bool | None = None
class ParseSourceResponse(BaseModel):
id: int
source_type: str
name: str
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int
enabled: bool
last_status: str
last_error: str | None = None
last_run_at: datetime | None = None
docs_added: int
created_at: datetime
model_config = {"from_attributes": True}
# ─── Отстойник ────────────────────────────────────────────────────────────────
class StagedWorkResponse(BaseModel):
id: int
user_id: int | None = None
task_id: str | None = None
filename: str | None = None
title: str | None = None
authors: list = Field(default_factory=list)
year: int | None = None
lang: str | None = None
word_count: int | None = None
status: str
document_id: int | None = None
created_at: datetime
model_config = {"from_attributes": True}
class StagedWorkDetail(StagedWorkResponse):
text_preview: str | None = None
# ─── Дашборд ──────────────────────────────────────────────────────────────────
class ServiceHealth(BaseModel):
name: str
ok: bool
detail: str | None = None
class AdminStats(BaseModel):
users_total: int
tasks_by_status: dict[str, int]
documents_total: int
documents_by_source: dict[str, int]
staging_pending: int
storage: dict[str, Any]
queues: dict[str, Any]

View File

@@ -26,6 +26,7 @@ class UserInToken(BaseModel):
name: str
plan: str
is_verified: bool
is_admin: bool = False
model_config = {"from_attributes": True}

View File

@@ -10,6 +10,7 @@ pydantic==2.7.1
pydantic-settings==2.2.1
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
bcrypt==4.0.1 # passlib 1.7.4 несовместим с bcrypt>=4.1 (ошибка __about__ / 72-byte limit)
python-multipart==0.0.9
minio==7.2.7
httpx==0.27.0

View File

@@ -0,0 +1,13 @@
# Dev-образ фронтенда: Vite dev-server с hot reload
FROM node:20-slim
WORKDIR /app
# Сначала зависимости (кэш слоёв)
COPY package.json package-lock.json* ./
RUN npm install
COPY . .
EXPOSE 5173
CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0", "--port", "5173"]

2968
services/frontend/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -9,28 +9,28 @@
"preview": "vite preview"
},
"dependencies": {
"@tanstack/react-query": "^5.40.0",
"axios": "^1.7.2",
"clsx": "^2.1.1",
"date-fns": "^3.6.0",
"lucide-react": "^0.390.0",
"react": "^18.3.0",
"react-dom": "^18.3.0",
"react-router-dom": "^6.23.0",
"@tanstack/react-query": "^5.40.0",
"zustand": "^4.5.2",
"axios": "^1.7.2",
"react-dropzone": "^14.2.3",
"react-dropzone": "^14.4.1",
"react-hot-toast": "^2.4.1",
"lucide-react": "^0.390.0",
"clsx": "^2.1.1",
"react-router-dom": "^6.23.0",
"tailwind-merge": "^2.3.0",
"date-fns": "^3.6.0"
"zustand": "^4.5.2"
},
"devDependencies": {
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react": "^4.3.0",
"typescript": "^5.4.5",
"vite": "^5.2.12",
"tailwindcss": "^3.4.4",
"postcss": "^8.4.38",
"autoprefixer": "^10.4.19",
"eslint": "^9.4.0"
"eslint": "^9.4.0",
"postcss": "^8.4.38",
"tailwindcss": "^3.4.4",
"typescript": "^5.4.5",
"vite": "^5.2.12"
}
}

View File

@@ -78,3 +78,42 @@ export const reportsApi = {
get: (taskId: string) =>
api.get(`/reports/${taskId}`),
};
// ─── Админка ────────────────────────────────────────────────────────────────
export const adminApi = {
openSession: () => api.post('/admin/session'),
verifySession: (code: string) => api.get(`/admin/session/verify/${code}`),
health: () => api.get('/admin/health'),
stats: () => api.get('/admin/stats'),
users: (params?: { q?: string; limit?: number; offset?: number }) =>
api.get('/admin/users', { params }),
updateUser: (id: number, data: { plan?: string; is_verified?: boolean; is_admin?: boolean }) =>
api.patch(`/admin/users/${id}`, data),
deleteUser: (id: number) => api.delete(`/admin/users/${id}`),
tasks: (params?: { status?: string; type?: string; user_id?: number; limit?: number; offset?: number }) =>
api.get('/admin/tasks', { params }),
task: (publicId: string) => api.get(`/admin/tasks/${publicId}`),
requeueTask: (publicId: string) => api.post(`/admin/tasks/${publicId}/requeue`),
deleteTask: (publicId: string) => api.delete(`/admin/tasks/${publicId}`),
documents: (params?: { q?: string; source?: string; year?: number; limit?: number; offset?: number }) =>
api.get('/admin/documents', { params }),
deleteDocument: (id: number) => api.delete(`/admin/documents/${id}`),
storage: () => api.get('/admin/storage'),
sources: () => api.get('/admin/sources'),
createSource: (data: Record<string, unknown>) => api.post('/admin/sources', data),
updateSource: (id: number, data: Record<string, unknown>) => api.patch(`/admin/sources/${id}`, data),
deleteSource: (id: number) => api.delete(`/admin/sources/${id}`),
runSource: (id: number) => api.post(`/admin/sources/${id}/run`),
staging: (params?: { status?: string; limit?: number; offset?: number }) =>
api.get('/admin/staging', { params }),
stagingDetail: (id: number) => api.get(`/admin/staging/${id}`),
approveStaging: (id: number) => api.post(`/admin/staging/${id}/approve`),
rejectStaging: (id: number) => api.post(`/admin/staging/${id}/reject`),
};

View File

@@ -71,7 +71,7 @@ export function TaskCard({ task }: TaskCardProps) {
return (
<Link
to={`/tasks/${task.id}`}
to={`/tasks/${task.public_id}`}
className="block group"
>
<div className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-100 hover:border-brand-200 hover:shadow-sm transition-all duration-200">

View File

@@ -14,6 +14,14 @@ import { Task } from './pages/Task';
import { Pricing } from './pages/Pricing';
import { Login } from './pages/Login';
import { Register } from './pages/Register';
import { AdminLayout } from './pages/admin/AdminLayout';
import { Dashboard } from './pages/admin/Dashboard';
import { Users as AdminUsers } from './pages/admin/Users';
import { Works as AdminWorks } from './pages/admin/Works';
import { Documents as AdminDocuments } from './pages/admin/Documents';
import { Storage as AdminStorage } from './pages/admin/Storage';
import { Sources as AdminSources } from './pages/admin/Sources';
import { Staging as AdminStaging } from './pages/admin/Staging';
import './index.css';
@@ -26,23 +34,51 @@ const queryClient = new QueryClient({
},
});
// Маршруты пользовательской части (обёрнуты в общий Layout)
function PublicApp() {
return (
<Layout>
<Routes>
<Route path="/" element={<Home />} />
<Route path="/search" element={<Search />} />
<Route path="/check" element={<Check />} />
<Route path="/bibliography" element={<Bibliography />} />
<Route path="/cabinet" element={<Cabinet />} />
<Route path="/tasks/:taskId" element={<Task />} />
<Route path="/pricing" element={<Pricing />} />
<Route path="/login" element={<Login />} />
<Route path="/register" element={<Register />} />
</Routes>
</Layout>
);
}
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<QueryClientProvider client={queryClient}>
<BrowserRouter>
<Layout>
<Routes>
<Route path="/" element={<Home />} />
<Route path="/search" element={<Search />} />
<Route path="/check" element={<Check />} />
<Route path="/bibliography" element={<Bibliography />} />
<Route path="/cabinet" element={<Cabinet />} />
<Route path="/tasks/:taskId" element={<Task />} />
<Route path="/pricing" element={<Pricing />} />
<Route path="/login" element={<Login />} />
<Route path="/register" element={<Register />} />
</Routes>
</Layout>
<Routes>
{/* Админка — собственный layout, защита по is_admin + коду сессии */}
<Route
path="/admin/:code/*"
element={
<AdminLayout>
<Routes>
<Route path="dashboard" element={<Dashboard />} />
<Route path="users" element={<AdminUsers />} />
<Route path="works" element={<AdminWorks />} />
<Route path="documents" element={<AdminDocuments />} />
<Route path="storage" element={<AdminStorage />} />
<Route path="sources" element={<AdminSources />} />
<Route path="staging" element={<AdminStaging />} />
<Route path="*" element={<Dashboard />} />
</Routes>
</AdminLayout>
}
/>
{/* Всё остальное — пользовательская часть */}
<Route path="/*" element={<PublicApp />} />
</Routes>
</BrowserRouter>
<Toaster
position="top-right"

View File

@@ -1,9 +1,10 @@
import React from 'react';
import { Navigate } from 'react-router-dom';
import { Navigate, useNavigate } from 'react-router-dom';
import { useQuery } from '@tanstack/react-query';
import { Crown, Search, Shield, BookOpen } from 'lucide-react';
import { Crown, Search, Shield, BookOpen, ShieldAlert } from 'lucide-react';
import toast from 'react-hot-toast';
import { TaskCard } from '../components/TaskCard';
import { tasksApi } from '../api/client';
import { tasksApi, adminApi } from '../api/client';
import { useAuthStore } from '../store/auth';
import { PLAN_LIMITS, type Task } from '../types';
@@ -16,6 +17,16 @@ const PLAN_BADGE_STYLES = {
export function Cabinet() {
const { isAuthenticated, user } = useAuthStore();
const navigate = useNavigate();
const openAdmin = async () => {
try {
const { data } = await adminApi.openSession();
navigate(`${data.url}/dashboard`);
} catch {
toast.error('Не удалось открыть админ-панель');
}
};
if (!isAuthenticated) {
return <Navigate to="/login" replace />;
@@ -49,9 +60,20 @@ export function Cabinet() {
</div>
<p className="text-gray-500 text-sm">{user?.email}</p>
</div>
<button className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors">
Обновить план
</button>
<div className="flex items-center gap-2">
{user?.is_admin && (
<button
onClick={openAdmin}
className="flex items-center gap-1.5 px-4 py-2 text-sm font-medium bg-gray-900 text-white rounded-lg hover:bg-gray-800 transition-colors"
>
<ShieldAlert className="w-4 h-4" />
Админ-панель
</button>
)}
<button className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors">
Обновить план
</button>
</div>
</div>
</div>
@@ -116,7 +138,7 @@ export function Cabinet() {
{tasks && tasks.length > 0 && (
<div className="space-y-2">
{tasks.map((task) => (
<TaskCard key={task.id} task={task} />
<TaskCard key={task.public_id} task={task} />
))}
</div>
)}

View File

@@ -53,14 +53,14 @@ export function Check() {
</div>
<h2 className="text-xl font-semibold text-gray-900 mb-2">Проверка запущена</h2>
<p className="text-gray-500 mb-2">
Файл <strong>{(submittedTask.input_data as any).filename}</strong> передан на проверку.
Файл <strong>{file?.name ?? 'документ'}</strong> передан на проверку.
</p>
<p className="text-sm text-gray-400 mb-6">
Вы получите email когда проверка завершится. Обычно это занимает 1-3 минуты.
</p>
<div className="flex flex-col sm:flex-row gap-3 justify-center">
<Link
to={`/tasks/${submittedTask.id}`}
to={`/tasks/${submittedTask.public_id}`}
className="flex items-center justify-center gap-2 px-5 py-2.5 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700 transition-colors"
>
Следить за прогрессом

View File

@@ -27,7 +27,7 @@ export function Search() {
mutationFn: searchApi.create,
onSuccess: (response) => {
const task: Task = response.data;
setTaskId(task.id);
setTaskId(task.public_id);
},
onError: (error: any) => {
const msg = error.response?.data?.detail || 'Ошибка поиска';

View File

@@ -60,7 +60,7 @@ export function Task() {
</h1>
<div className="flex items-center gap-2 mt-0.5">
<StatusBadge status={task.status} />
<span className="text-xs text-gray-400">{task.id}</span>
<span className="text-xs text-gray-400">{task.public_id}</span>
</div>
</div>
</div>
@@ -91,8 +91,7 @@ export function Task() {
{task.status === 'done' && task.type === 'search' && task.result && (
<div className="space-y-4">
<p className="text-sm text-gray-500">
Запрос: <strong>«{(task.input_data as any).query}»</strong>
{' · '}{(task.result as SearchResultData).total} источников
Найдено {(task.result as SearchResultData).total} источников
</p>
{(task.result as SearchResultData).sources.map((source) => (
<SourceCard key={source.id} source={source} />
@@ -104,7 +103,7 @@ export function Task() {
{task.status === 'done' && task.type === 'plagiarism' && task.result && (
<div className="bg-white rounded-xl border border-gray-100 p-6">
<h2 className="text-base font-semibold text-gray-900 mb-4">
Файл: {(task.input_data as any).filename}
Результат проверки плагиата
</h2>
<PlagiarismReport data={task.result as PlagiarismResultData} />
</div>

View File

@@ -0,0 +1,102 @@
import React from 'react';
import { NavLink, useParams, useNavigate, Navigate } from 'react-router-dom';
import { useQuery } from '@tanstack/react-query';
import {
LayoutDashboard, Users, FileText, Database, HardDrive, Download, Inbox, LogOut, ShieldAlert,
} from 'lucide-react';
import { clsx } from 'clsx';
import { adminApi } from '../../api/client';
import { useAuthStore } from '../../store/auth';
const NAV = [
{ to: 'dashboard', label: 'Дашборд', icon: LayoutDashboard },
{ to: 'users', label: 'Клиенты', icon: Users },
{ to: 'works', label: 'Работы', icon: FileText },
{ to: 'documents', label: 'База', icon: Database },
{ to: 'storage', label: 'Хранилище', icon: HardDrive },
{ to: 'sources', label: 'Источники', icon: Download },
{ to: 'staging', label: 'Отстойник', icon: Inbox },
];
interface AdminLayoutProps {
children: React.ReactNode;
}
export function AdminLayout({ children }: AdminLayoutProps) {
const { code } = useParams<{ code: string }>();
const { isAuthenticated, user, logout } = useAuthStore();
const navigate = useNavigate();
// Доступ: только аутентифицированный админ
const { data: verify, isLoading } = useQuery({
queryKey: ['admin-verify', code],
queryFn: () => adminApi.verifySession(code!).then((r) => r.data),
enabled: !!code && isAuthenticated && !!user?.is_admin,
retry: false,
});
if (!isAuthenticated || !user?.is_admin) {
return <Navigate to="/login" replace />;
}
if (isLoading) {
return <div className="min-h-screen flex items-center justify-center text-gray-400">Проверка доступа</div>;
}
if (!verify?.valid) {
return (
<div className="min-h-screen flex flex-col items-center justify-center gap-4 bg-gray-50">
<ShieldAlert className="w-12 h-12 text-red-400" />
<p className="text-gray-600">Код доступа недействителен или истёк.</p>
<button
onClick={() => navigate('/cabinet')}
className="px-4 py-2 bg-brand-600 text-white rounded-lg text-sm"
>
В кабинет
</button>
</div>
);
}
return (
<div className="min-h-screen flex bg-gray-50">
{/* Сайдбар */}
<aside className="w-56 bg-gray-900 text-gray-300 flex flex-col fixed inset-y-0 left-0">
<div className="px-5 py-5 border-b border-gray-800 flex items-center gap-2">
<ShieldAlert className="w-5 h-5 text-amber-400" />
<span className="font-semibold text-white">Админ-панель</span>
</div>
<nav className="flex-1 px-3 py-4 space-y-1">
{NAV.map(({ to, label, icon: Icon }) => (
<NavLink
key={to}
to={`/admin/${code}/${to}`}
className={({ isActive }) =>
clsx(
'flex items-center gap-3 px-3 py-2.5 rounded-lg text-sm font-medium transition-colors',
isActive ? 'bg-brand-600 text-white' : 'hover:bg-gray-800 hover:text-white'
)
}
>
<Icon className="w-4 h-4" />
{label}
</NavLink>
))}
</nav>
<div className="px-3 py-4 border-t border-gray-800">
<div className="px-3 mb-2 text-xs text-gray-500">{user?.email}</div>
<button
onClick={() => { logout(); navigate('/'); }}
className="w-full flex items-center gap-3 px-3 py-2.5 rounded-lg text-sm hover:bg-gray-800 hover:text-white transition-colors"
>
<LogOut className="w-4 h-4" />
Выйти
</button>
</div>
</aside>
{/* Контент */}
<main className="flex-1 ml-56 p-8">{children}</main>
</div>
);
}

View File

@@ -0,0 +1,119 @@
import React from 'react';
import { useQuery } from '@tanstack/react-query';
import { CheckCircle2, XCircle, Users, FileText, Database, Inbox } from 'lucide-react';
import { adminApi } from '../../api/client';
interface Health { name: string; ok: boolean; detail?: string }
interface Stats {
users_total: number;
tasks_by_status: Record<string, number>;
documents_total: number;
documents_by_source: Record<string, number>;
staging_pending: number;
storage: Record<string, { objects: number; bytes: number } | unknown>;
queues: Record<string, number | string>;
}
function fmtBytes(b: number): string {
if (b < 1024) return `${b} Б`;
if (b < 1024 ** 2) return `${(b / 1024).toFixed(1)} КБ`;
if (b < 1024 ** 3) return `${(b / 1024 ** 2).toFixed(1)} МБ`;
return `${(b / 1024 ** 3).toFixed(2)} ГБ`;
}
export function Dashboard() {
const { data: health } = useQuery({
queryKey: ['admin-health'],
queryFn: () => adminApi.health().then((r) => r.data as Health[]),
refetchInterval: 10000,
});
const { data: stats } = useQuery({
queryKey: ['admin-stats'],
queryFn: () => adminApi.stats().then((r) => r.data as Stats),
refetchInterval: 10000,
});
const totalStorage = Object.values(stats?.storage || {}).reduce(
(acc: number, v) => acc + (typeof v === 'object' && v && 'bytes' in v ? (v as { bytes: number }).bytes : 0),
0
);
return (
<div className="space-y-6">
<h1 className="text-2xl font-bold text-gray-900">Дашборд</h1>
{/* Счётчики */}
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
<StatCard icon={Users} label="Клиентов" value={stats?.users_total ?? '—'} color="text-blue-600" />
<StatCard icon={FileText} label="Работ всего" value={Object.values(stats?.tasks_by_status || {}).reduce((a, b) => a + b, 0)} color="text-purple-600" />
<StatCard icon={Database} label="Документов в базе" value={stats?.documents_total ?? '—'} color="text-emerald-600" />
<StatCard icon={Inbox} label="В отстойнике" value={stats?.staging_pending ?? '—'} color="text-amber-600" />
</div>
<div className="grid lg:grid-cols-2 gap-6">
{/* Здоровье сервисов */}
<div className="bg-white rounded-xl border border-gray-100 p-5">
<h2 className="font-semibold text-gray-800 mb-4">Сервисы</h2>
<div className="space-y-2">
{health?.map((h) => (
<div key={h.name} className="flex items-center justify-between py-1.5">
<div className="flex items-center gap-2">
{h.ok ? <CheckCircle2 className="w-4 h-4 text-emerald-500" /> : <XCircle className="w-4 h-4 text-red-500" />}
<span className="text-sm text-gray-700">{h.name}</span>
</div>
<span className={`text-xs ${h.ok ? 'text-gray-400' : 'text-red-500'}`}>{h.detail || (h.ok ? 'OK' : 'недоступен')}</span>
</div>
))}
{!health && <div className="text-sm text-gray-400">Загрузка</div>}
</div>
</div>
{/* Работы по статусам + очереди */}
<div className="bg-white rounded-xl border border-gray-100 p-5 space-y-4">
<div>
<h2 className="font-semibold text-gray-800 mb-3">Работы по статусам</h2>
<div className="flex flex-wrap gap-2">
{Object.entries(stats?.tasks_by_status || {}).map(([s, c]) => (
<span key={s} className="px-3 py-1 bg-gray-100 rounded-lg text-sm text-gray-700">{s}: <b>{c}</b></span>
))}
{!Object.keys(stats?.tasks_by_status || {}).length && <span className="text-sm text-gray-400">нет данных</span>}
</div>
</div>
<div>
<h2 className="font-semibold text-gray-800 mb-3">Очереди</h2>
<div className="flex flex-wrap gap-2">
{Object.entries(stats?.queues || {}).map(([q, n]) => (
<span key={q} className="px-3 py-1 bg-gray-100 rounded-lg text-sm text-gray-700">{q}: <b>{String(n)}</b></span>
))}
</div>
</div>
<div>
<h2 className="font-semibold text-gray-800 mb-2">Хранилище</h2>
<p className="text-sm text-gray-600">Всего: <b>{fmtBytes(totalStorage)}</b></p>
</div>
</div>
</div>
{/* База по источникам */}
<div className="bg-white rounded-xl border border-gray-100 p-5">
<h2 className="font-semibold text-gray-800 mb-3">База документов по источникам</h2>
<div className="flex flex-wrap gap-2">
{Object.entries(stats?.documents_by_source || {}).map(([s, c]) => (
<span key={s} className="px-3 py-1 bg-emerald-50 text-emerald-700 rounded-lg text-sm">{s}: <b>{c}</b></span>
))}
{!Object.keys(stats?.documents_by_source || {}).length && <span className="text-sm text-gray-400">база пуста</span>}
</div>
</div>
</div>
);
}
function StatCard({ icon: Icon, label, value, color }: { icon: React.ElementType; label: string; value: React.ReactNode; color: string }) {
return (
<div className="bg-white rounded-xl border border-gray-100 p-5">
<Icon className={`w-6 h-6 ${color} mb-2`} />
<div className="text-2xl font-bold text-gray-900">{value}</div>
<div className="text-sm text-gray-500">{label}</div>
</div>
);
}

View File

@@ -0,0 +1,71 @@
import React, { useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { Trash2, Search } from 'lucide-react';
import toast from 'react-hot-toast';
import { adminApi } from '../../api/client';
interface Doc {
id: number; source: string; title: string; authors: Array<{ last_name?: string }>;
year: number | null; lang: string | null; url: string | null; indexed_at: string;
}
export function Documents() {
const qc = useQueryClient();
const [q, setQ] = useState('');
const [source, setSource] = useState('');
const { data: docs } = useQuery({
queryKey: ['admin-docs', q, source],
queryFn: () => adminApi.documents({ q: q || undefined, source: source || undefined }).then((r) => r.data as Doc[]),
});
const del = useMutation({
mutationFn: (id: number) => adminApi.deleteDocument(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-docs'] }); toast.success('Удалён из базы'); },
onError: () => toast.error('Ошибка'),
});
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">База документов</h1>
<div className="flex gap-3">
<div className="relative flex-1 max-w-sm">
<Search className="w-4 h-4 text-gray-400 absolute left-3 top-1/2 -translate-y-1/2" />
<input value={q} onChange={(e) => setQ(e.target.value)} placeholder="Поиск по названию"
className="w-full pl-9 pr-3 py-2 border border-gray-200 rounded-lg text-sm" />
</div>
<select value={source} onChange={(e) => setSource(e.target.value)}
className="border border-gray-200 rounded-lg px-3 py-2 text-sm">
<option value="">все источники</option>
<option value="openalex">openalex</option>
<option value="cyberleninka">cyberleninka</option>
<option value="arxiv">arxiv</option>
<option value="user_submission">работы пользователей</option>
</select>
</div>
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-gray-500 text-left">
<tr>
<th className="px-4 py-3">Название</th><th className="px-4 py-3">Источник</th>
<th className="px-4 py-3">Год</th><th className="px-4 py-3">Язык</th><th className="px-4 py-3"></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-50">
{docs?.map((d) => (
<tr key={d.id} className="hover:bg-gray-50">
<td className="px-4 py-3 text-gray-800 max-w-md truncate">{d.title}</td>
<td className="px-4 py-3"><span className="px-2 py-0.5 bg-emerald-50 text-emerald-700 rounded text-xs">{d.source}</span></td>
<td className="px-4 py-3 text-gray-600">{d.year || '—'}</td>
<td className="px-4 py-3 text-gray-600">{d.lang || '—'}</td>
<td className="px-4 py-3">
<button onClick={() => { if (confirm('Удалить документ из базы?')) del.mutate(d.id); }}
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
</td>
</tr>
))}
</tbody>
</table>
{!docs?.length && <div className="p-6 text-center text-gray-400 text-sm">База пуста</div>}
</div>
</div>
);
}

View File

@@ -0,0 +1,114 @@
import React, { useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { Play, Trash2, Plus } from 'lucide-react';
import toast from 'react-hot-toast';
import { adminApi } from '../../api/client';
interface Source {
id: number; source_type: string; name: string; query: string | null;
lang: string | null; year_from: number | null; year_to: number | null;
limit: number; enabled: boolean; last_status: string; last_error: string | null;
last_run_at: string | null; docs_added: number;
}
const STATUS_COLORS: Record<string, string> = {
idle: 'bg-gray-100 text-gray-600',
running: 'bg-blue-100 text-blue-700',
done: 'bg-emerald-100 text-emerald-700',
error: 'bg-red-100 text-red-700',
};
export function Sources() {
const qc = useQueryClient();
const [form, setForm] = useState({ source_type: 'openalex', name: '', query: '', lang: '', limit: 100 });
const { data: sources } = useQuery({
queryKey: ['admin-sources'],
queryFn: () => adminApi.sources().then((r) => r.data as Source[]),
refetchInterval: 5000,
});
const create = useMutation({
mutationFn: (data: Record<string, unknown>) => adminApi.createSource(data),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Источник добавлен'); setForm({ source_type: 'openalex', name: '', query: '', lang: '', limit: 100 }); },
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
});
const run = useMutation({
mutationFn: (id: number) => adminApi.runSource(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Парсинг запущен'); },
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
});
const del = useMutation({
mutationFn: (id: number) => adminApi.deleteSource(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Удалён'); },
});
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">Источники парсинга</h1>
{/* Форма добавления */}
<div className="bg-white rounded-xl border border-gray-100 p-5">
<h2 className="font-semibold text-gray-800 mb-3">Добавить источник</h2>
<div className="grid grid-cols-2 lg:grid-cols-5 gap-3">
<select value={form.source_type} onChange={(e) => setForm({ ...form, source_type: e.target.value })}
className="border border-gray-200 rounded-lg px-3 py-2 text-sm">
<option value="openalex">OpenAlex</option>
<option value="cyberleninka">КиберЛенинка</option>
<option value="arxiv">arXiv</option>
</select>
<input value={form.name} onChange={(e) => setForm({ ...form, name: e.target.value })} placeholder="Название"
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
<input value={form.query} onChange={(e) => setForm({ ...form, query: e.target.value })} placeholder="Запрос"
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
<input value={form.lang} onChange={(e) => setForm({ ...form, lang: e.target.value })} placeholder="Язык (ru/en)"
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
<input type="number" value={form.limit} onChange={(e) => setForm({ ...form, limit: Number(e.target.value) })} placeholder="Лимит"
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
</div>
<button
onClick={() => {
if (!form.name) { toast.error('Укажите название'); return; }
create.mutate({ ...form, lang: form.lang || null, query: form.query || null });
}}
className="mt-3 flex items-center gap-2 px-4 py-2 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700"
>
<Plus className="w-4 h-4" /> Добавить
</button>
</div>
{/* Список */}
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-gray-500 text-left">
<tr>
<th className="px-4 py-3">Название</th><th className="px-4 py-3">Тип</th>
<th className="px-4 py-3">Запрос</th><th className="px-4 py-3">Лимит</th>
<th className="px-4 py-3">Статус</th><th className="px-4 py-3">Добавлено</th><th className="px-4 py-3"></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-50">
{sources?.map((s) => (
<tr key={s.id} className="hover:bg-gray-50">
<td className="px-4 py-3 text-gray-800">{s.name}</td>
<td className="px-4 py-3 text-gray-600">{s.source_type}</td>
<td className="px-4 py-3 text-gray-500 max-w-[160px] truncate">{s.query || '—'}</td>
<td className="px-4 py-3 text-gray-600">{s.limit}</td>
<td className="px-4 py-3">
<span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[s.last_status] || 'bg-gray-100'}`} title={s.last_error || ''}>{s.last_status}</span>
</td>
<td className="px-4 py-3 text-gray-600">{s.docs_added}</td>
<td className="px-4 py-3 flex gap-1">
<button onClick={() => run.mutate(s.id)} disabled={s.last_status === 'running'} title="Запустить"
className="p-1.5 text-gray-400 hover:text-emerald-600 rounded disabled:opacity-40"><Play className="w-4 h-4" /></button>
<button onClick={() => { if (confirm('Удалить источник?')) del.mutate(s.id); }} title="Удалить"
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
</td>
</tr>
))}
</tbody>
</table>
{!sources?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет источников</div>}
</div>
</div>
);
}

View File

@@ -0,0 +1,119 @@
import React, { useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { Check, X, Eye } from 'lucide-react';
import toast from 'react-hot-toast';
import { adminApi } from '../../api/client';
interface Staged {
id: number; user_id: number | null; filename: string | null; title: string | null;
word_count: number | null; status: string; document_id: number | null; created_at: string;
}
interface StagedDetail extends Staged { text_preview: string | null }
const STATUS_COLORS: Record<string, string> = {
pending: 'bg-amber-100 text-amber-700',
approved: 'bg-emerald-100 text-emerald-700',
rejected: 'bg-red-100 text-red-700',
};
export function Staging() {
const qc = useQueryClient();
const [status, setStatus] = useState('pending');
const [preview, setPreview] = useState<StagedDetail | null>(null);
const { data: items } = useQuery({
queryKey: ['admin-staging', status],
queryFn: () => adminApi.staging({ status }).then((r) => r.data as Staged[]),
refetchInterval: 8000,
});
const approve = useMutation({
mutationFn: (id: number) => adminApi.approveStaging(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-staging'] }); setPreview(null); toast.success('Одобрено — добавляется в базу'); },
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
});
const reject = useMutation({
mutationFn: (id: number) => adminApi.rejectStaging(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-staging'] }); setPreview(null); toast.success('Отклонено'); },
});
const openPreview = async (id: number) => {
const { data } = await adminApi.stagingDetail(id);
setPreview(data as StagedDetail);
};
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">Отстойник проверенных работ</h1>
<p className="text-sm text-gray-500">Работы, прошедшие проверку. Одобрение добавляет их в базу источников для будущих сравнений.</p>
<div className="flex gap-2">
{['pending', 'approved', 'rejected', 'all'].map((s) => (
<button key={s} onClick={() => setStatus(s)}
className={`px-3 py-1.5 rounded-lg text-sm ${status === s ? 'bg-brand-600 text-white' : 'bg-white border border-gray-200 text-gray-600'}`}>
{s === 'all' ? 'все' : s}
</button>
))}
</div>
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-gray-500 text-left">
<tr>
<th className="px-4 py-3">Файл</th><th className="px-4 py-3">Клиент</th>
<th className="px-4 py-3">Слов</th><th className="px-4 py-3">Статус</th>
<th className="px-4 py-3">Дата</th><th className="px-4 py-3"></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-50">
{items?.map((s) => (
<tr key={s.id} className="hover:bg-gray-50">
<td className="px-4 py-3 text-gray-800 max-w-xs truncate">{s.filename || s.title || `#${s.id}`}</td>
<td className="px-4 py-3 text-gray-600">{s.user_id ? `#${s.user_id}` : '—'}</td>
<td className="px-4 py-3 text-gray-600">{s.word_count ?? '—'}</td>
<td className="px-4 py-3"><span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[s.status]}`}>{s.status}</span></td>
<td className="px-4 py-3 text-gray-400 text-xs">{new Date(s.created_at).toLocaleString('ru')}</td>
<td className="px-4 py-3 flex gap-1">
<button onClick={() => openPreview(s.id)} title="Просмотр"
className="p-1.5 text-gray-400 hover:text-brand-600 rounded"><Eye className="w-4 h-4" /></button>
{s.status === 'pending' && (
<>
<button onClick={() => approve.mutate(s.id)} title="Одобрить"
className="p-1.5 text-gray-400 hover:text-emerald-600 rounded"><Check className="w-4 h-4" /></button>
<button onClick={() => reject.mutate(s.id)} title="Отклонить"
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><X className="w-4 h-4" /></button>
</>
)}
</td>
</tr>
))}
</tbody>
</table>
{!items?.length && <div className="p-6 text-center text-gray-400 text-sm">Пусто</div>}
</div>
{/* Превью */}
{preview && (
<div className="fixed inset-0 bg-black/40 flex items-center justify-center z-50 p-6" onClick={() => setPreview(null)}>
<div className="bg-white rounded-xl max-w-2xl w-full max-h-[80vh] flex flex-col" onClick={(e) => e.stopPropagation()}>
<div className="px-5 py-4 border-b border-gray-100 flex items-center justify-between">
<h3 className="font-semibold text-gray-800 truncate">{preview.filename || preview.title}</h3>
<button onClick={() => setPreview(null)} className="text-gray-400 hover:text-gray-600"><X className="w-5 h-5" /></button>
</div>
<div className="p-5 overflow-auto text-sm text-gray-700 whitespace-pre-wrap flex-1">
{preview.text_preview || '(нет текста)'}
</div>
{preview.status === 'pending' && (
<div className="px-5 py-4 border-t border-gray-100 flex gap-2 justify-end">
<button onClick={() => reject.mutate(preview.id)}
className="px-4 py-2 border border-gray-200 text-gray-600 rounded-lg text-sm">Отклонить</button>
<button onClick={() => approve.mutate(preview.id)}
className="px-4 py-2 bg-emerald-600 text-white rounded-lg text-sm">Одобрить в базу</button>
</div>
)}
</div>
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,57 @@
import React from 'react';
import { useQuery } from '@tanstack/react-query';
import { HardDrive } from 'lucide-react';
import { adminApi } from '../../api/client';
interface Bucket {
name: string; objects_total?: number; bytes?: number;
sample?: Array<{ name: string; size: number }>; error?: string;
}
function fmtBytes(b: number): string {
if (b < 1024) return `${b} Б`;
if (b < 1024 ** 2) return `${(b / 1024).toFixed(1)} КБ`;
if (b < 1024 ** 3) return `${(b / 1024 ** 2).toFixed(1)} МБ`;
return `${(b / 1024 ** 3).toFixed(2)} ГБ`;
}
export function Storage() {
const { data } = useQuery({
queryKey: ['admin-storage'],
queryFn: () => adminApi.storage().then((r) => r.data as { buckets: Bucket[] }),
refetchInterval: 15000,
});
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">Хранилище (MinIO)</h1>
<div className="grid lg:grid-cols-3 gap-4">
{data?.buckets.map((b) => (
<div key={b.name} className="bg-white rounded-xl border border-gray-100 p-5">
<div className="flex items-center gap-2 mb-3">
<HardDrive className="w-5 h-5 text-brand-600" />
<span className="font-semibold text-gray-800">{b.name}</span>
</div>
{b.error ? (
<p className="text-sm text-red-500">{b.error}</p>
) : (
<>
<p className="text-sm text-gray-600">Объектов: <b>{b.objects_total}</b></p>
<p className="text-sm text-gray-600">Объём: <b>{fmtBytes(b.bytes || 0)}</b></p>
<div className="mt-3 max-h-40 overflow-auto space-y-1">
{b.sample?.map((o) => (
<div key={o.name} className="flex justify-between text-xs text-gray-400">
<span className="truncate max-w-[160px]">{o.name}</span>
<span>{fmtBytes(o.size)}</span>
</div>
))}
</div>
</>
)}
</div>
))}
{!data?.buckets.length && <div className="text-gray-400 text-sm">Нет бакетов</div>}
</div>
</div>
);
}

View File

@@ -0,0 +1,92 @@
import React, { useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { Trash2, Search } from 'lucide-react';
import toast from 'react-hot-toast';
import { adminApi } from '../../api/client';
interface AdminUser {
id: number; email: string; name: string; plan: string;
is_verified: boolean; is_admin: boolean; created_at: string; tasks_count: number;
}
const PLANS = ['free', 'student', 'premium', 'science'];
export function Users() {
const qc = useQueryClient();
const [q, setQ] = useState('');
const { data: users } = useQuery({
queryKey: ['admin-users', q],
queryFn: () => adminApi.users({ q: q || undefined }).then((r) => r.data as AdminUser[]),
});
const update = useMutation({
mutationFn: ({ id, data }: { id: number; data: Record<string, unknown> }) => adminApi.updateUser(id, data),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-users'] }); toast.success('Сохранено'); },
onError: () => toast.error('Ошибка'),
});
const del = useMutation({
mutationFn: (id: number) => adminApi.deleteUser(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-users'] }); toast.success('Удалён'); },
onError: () => toast.error('Ошибка удаления'),
});
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">Клиенты</h1>
<div className="relative max-w-sm">
<Search className="w-4 h-4 text-gray-400 absolute left-3 top-1/2 -translate-y-1/2" />
<input
value={q} onChange={(e) => setQ(e.target.value)} placeholder="Поиск по email/имени"
className="w-full pl-9 pr-3 py-2 border border-gray-200 rounded-lg text-sm"
/>
</div>
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-gray-500 text-left">
<tr>
<th className="px-4 py-3">Email</th><th className="px-4 py-3">Имя</th>
<th className="px-4 py-3">Тариф</th><th className="px-4 py-3">Работ</th>
<th className="px-4 py-3">Verified</th><th className="px-4 py-3">Admin</th>
<th className="px-4 py-3"></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-50">
{users?.map((u) => (
<tr key={u.id} className="hover:bg-gray-50">
<td className="px-4 py-3 text-gray-800">{u.email}</td>
<td className="px-4 py-3 text-gray-600">{u.name}</td>
<td className="px-4 py-3">
<select
value={u.plan}
onChange={(e) => update.mutate({ id: u.id, data: { plan: e.target.value } })}
className="border border-gray-200 rounded px-2 py-1 text-xs"
>
{PLANS.map((p) => <option key={p} value={p}>{p}</option>)}
</select>
</td>
<td className="px-4 py-3 text-gray-600">{u.tasks_count}</td>
<td className="px-4 py-3">
<input type="checkbox" checked={u.is_verified}
onChange={(e) => update.mutate({ id: u.id, data: { is_verified: e.target.checked } })} />
</td>
<td className="px-4 py-3">
<input type="checkbox" checked={u.is_admin}
onChange={(e) => update.mutate({ id: u.id, data: { is_admin: e.target.checked } })} />
</td>
<td className="px-4 py-3">
<button
onClick={() => { if (confirm(`Удалить ${u.email}?`)) del.mutate(u.id); }}
className="p-1.5 text-gray-400 hover:text-red-500 rounded"
>
<Trash2 className="w-4 h-4" />
</button>
</td>
</tr>
))}
</tbody>
</table>
{!users?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет клиентов</div>}
</div>
</div>
);
}

View File

@@ -0,0 +1,81 @@
import React, { useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { RefreshCw, Trash2 } from 'lucide-react';
import toast from 'react-hot-toast';
import { adminApi } from '../../api/client';
interface AdminTask {
public_id: string; user_id: number; type: string; status: string;
error: string | null; created_at: string;
}
const STATUS_COLORS: Record<string, string> = {
queued: 'bg-gray-100 text-gray-600',
processing: 'bg-blue-100 text-blue-700',
done: 'bg-emerald-100 text-emerald-700',
failed: 'bg-red-100 text-red-700',
};
export function Works() {
const qc = useQueryClient();
const [status, setStatus] = useState('');
const { data: tasks } = useQuery({
queryKey: ['admin-tasks', status],
queryFn: () => adminApi.tasks({ status: status || undefined }).then((r) => r.data as AdminTask[]),
refetchInterval: 8000,
});
const requeue = useMutation({
mutationFn: (id: string) => adminApi.requeueTask(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-tasks'] }); toast.success('Перезапущено'); },
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
});
const del = useMutation({
mutationFn: (id: string) => adminApi.deleteTask(id),
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-tasks'] }); toast.success('Удалено'); },
onError: () => toast.error('Ошибка'),
});
return (
<div className="space-y-5">
<h1 className="text-2xl font-bold text-gray-900">Работы</h1>
<div className="flex gap-2">
{['', 'queued', 'processing', 'done', 'failed'].map((s) => (
<button key={s} onClick={() => setStatus(s)}
className={`px-3 py-1.5 rounded-lg text-sm ${status === s ? 'bg-brand-600 text-white' : 'bg-white border border-gray-200 text-gray-600'}`}>
{s || 'все'}
</button>
))}
</div>
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-gray-50 text-gray-500 text-left">
<tr>
<th className="px-4 py-3">ID</th><th className="px-4 py-3">Клиент</th>
<th className="px-4 py-3">Тип</th><th className="px-4 py-3">Статус</th>
<th className="px-4 py-3">Создано</th><th className="px-4 py-3"></th>
</tr>
</thead>
<tbody className="divide-y divide-gray-50">
{tasks?.map((t) => (
<tr key={t.public_id} className="hover:bg-gray-50">
<td className="px-4 py-3 font-mono text-xs text-gray-500">{t.public_id.slice(0, 10)}</td>
<td className="px-4 py-3 text-gray-600">#{t.user_id}</td>
<td className="px-4 py-3 text-gray-600">{t.type}</td>
<td className="px-4 py-3"><span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[t.status] || 'bg-gray-100'}`}>{t.status}</span></td>
<td className="px-4 py-3 text-gray-400 text-xs">{new Date(t.created_at).toLocaleString('ru')}</td>
<td className="px-4 py-3 flex gap-1">
<button onClick={() => requeue.mutate(t.public_id)} title="Перезапустить"
className="p-1.5 text-gray-400 hover:text-brand-600 rounded"><RefreshCw className="w-4 h-4" /></button>
<button onClick={() => { if (confirm('Удалить работу?')) del.mutate(t.public_id); }} title="Удалить"
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
</td>
</tr>
))}
</tbody>
</table>
{!tasks?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет работ</div>}
</div>
</div>
);
}

View File

@@ -12,6 +12,7 @@ export interface User {
name: string;
plan: UserPlan;
is_verified: boolean;
is_admin?: boolean;
}
// ─── Источник в результатах поиска ────────────────────────────────────────────
@@ -88,12 +89,12 @@ export interface GostResultData {
export type TaskResult = SearchResultData | PlagiarismResultData | GostResultData | null;
export interface Task {
id: string;
// Бэкенд отдаёт public_id (а не внутренний id) и не возвращает input_data
public_id: string;
type: TaskType;
status: TaskStatus;
queue_position?: number;
eta_seconds?: number;
input_data: Record<string, unknown>;
result?: TaskResult;
error?: string;
created_at: string;

View File

@@ -1,17 +1,21 @@
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
const apiTarget = process.env.VITE_API_TARGET || 'http://localhost:8000';
const wsTarget = apiTarget.replace(/^http/, 'ws');
export default defineConfig({
plugins: [react()],
server: {
port: 5173,
host: true,
proxy: {
'/api': {
target: 'http://localhost:8000',
target: apiTarget,
changeOrigin: true,
},
'/ws': {
target: 'ws://localhost:8000',
target: wsTarget,
ws: true,
},
},

View File

@@ -0,0 +1,39 @@
"""Минимальные модели для gost-воркера."""
from datetime import datetime
from sqlalchemy import JSON, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class Task(Base):
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[int] = mapped_column(Integer)
type: Mapped[str] = mapped_column(String(20))
status: Mapped[str] = mapped_column(String(20))
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
class Document(Base):
__tablename__ = "documents"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source: Mapped[str] = mapped_column(String(50))
ext_id: Mapped[str] = mapped_column(String(255))
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
title: Mapped[str] = mapped_column(Text)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
url: Mapped[str | None] = mapped_column(Text, nullable=True)

View File

@@ -12,10 +12,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& rm -rf /var/lib/apt/lists/*
# Сделать python3.11 дефолтным
# Сделать python3.11 дефолтным и поставить pip именно для 3.11 (на CUDA-базе
# python3-pip ставит pip для системного python3.10 — он не виден из python3.11)
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
update-alternatives --install /usr/bin/python python python3.11 1 && \
python3.11 -m pip install --upgrade pip
update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
WORKDIR /app

View File

@@ -10,6 +10,7 @@ class Settings(BaseSettings):
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore", # игнорировать поля общего .env, не относящиеся к воркеру
)
# PostgreSQL

View File

@@ -0,0 +1,3 @@
from app.models.base import Base, Document, Task, User
__all__ = ["Base", "User", "Task", "Document"]

View File

@@ -0,0 +1,67 @@
"""Базовые модели данных для GPU воркера (минимальный набор для работы с БД)."""
from datetime import datetime
from uuid import uuid4
from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
"""Базовый класс SQLAlchemy моделей."""
pass
class User(Base):
"""Минимальная модель пользователя для GPU воркера."""
__tablename__ = "users"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
email: Mapped[str] = mapped_column(String(255))
name: Mapped[str] = mapped_column(String(255))
plan: Mapped[str] = mapped_column(String(20), default="free")
is_verified: Mapped[bool] = mapped_column(default=False)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
class Task(Base):
"""Модель задачи."""
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
type: Mapped[str] = mapped_column(String(20))
status: Mapped[str] = mapped_column(String(20), default="queued")
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
queue_position: Mapped[int | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime | None] = mapped_column(nullable=True)
class Document(Base):
"""Модель документа в базе знаний."""
__tablename__ = "documents"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source: Mapped[str] = mapped_column(String(50))
ext_id: Mapped[str] = mapped_column(String(255))
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
title: Mapped[str] = mapped_column(Text)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
url: Mapped[str | None] = mapped_column(Text, nullable=True)
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
faiss_id: Mapped[int | None] = mapped_column(nullable=True)
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())

View File

@@ -3,7 +3,7 @@ redis==5.0.4
sqlalchemy==2.0.30
psycopg2-binary==2.9.9
sentence-transformers==3.0.0
faiss-gpu==1.7.4
faiss-cpu==1.8.0 # faiss-gpu нет в pip для py3.11; индексы в коде CPU-типа, GPU занят эмбеддингами (torch) и LLM (Ollama)
torch==2.3.0
numpy==1.26.4
httpx==0.27.0

View File

@@ -31,9 +31,12 @@ def hash_ngram(ngram: str) -> int:
ngram: n-грамма для хэширования
Returns:
64-битный хэш
Знаковый 64-битный хэш (умещается в PostgreSQL BIGINT)
"""
return xxhash.xxh64(ngram.encode("utf-8")).intdigest()
# xxh64 даёт unsigned 64-bit (до 2^64-1), а PostgreSQL BIGINT — signed
# (макс 2^63-1). Приводим к диапазону signed int64.
unsigned = xxhash.xxh64(ngram.encode("utf-8")).intdigest()
return unsigned - (1 << 64) if unsigned >= (1 << 63) else unsigned
def winnow(text: str, k: int = 5, window: int = 4) -> set[int]:

View File

@@ -10,6 +10,7 @@ class Settings(BaseSettings):
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore",
)
# PostgreSQL
@@ -30,6 +31,7 @@ class Settings(BaseSettings):
MINIO_ACCESS_KEY: str = "minioadmin"
MINIO_SECRET_KEY: str = "changeme"
MINIO_BUCKET_DOCS: str = "documents"
MINIO_BUCKET_STAGING: str = "staging"
# Elasticsearch
ELASTICSEARCH_URL: str = "http://elasticsearch:9200"

View File

@@ -0,0 +1,103 @@
"""Минимальные модели для индексер-воркера."""
from datetime import datetime
from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class User(Base):
__tablename__ = "users"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
email: Mapped[str] = mapped_column(String(255))
name: Mapped[str] = mapped_column(String(255))
plan: Mapped[str] = mapped_column(String(20), default="free")
class Task(Base):
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
type: Mapped[str] = mapped_column(String(20))
status: Mapped[str] = mapped_column(String(20), default="queued")
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
queue_position: Mapped[int | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime | None] = mapped_column(nullable=True)
class Document(Base):
__tablename__ = "documents"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source: Mapped[str] = mapped_column(String(50))
ext_id: Mapped[str] = mapped_column(String(255), unique=True)
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
title: Mapped[str] = mapped_column(Text)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
url: Mapped[str | None] = mapped_column(Text, nullable=True)
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
faiss_id: Mapped[int | None] = mapped_column(nullable=True)
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
class Fingerprint(Base):
__tablename__ = "fingerprints"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
doc_id: Mapped[int] = mapped_column(ForeignKey("documents.id"))
hash_value: Mapped[int] = mapped_column(BigInteger)
position: Mapped[int] = mapped_column()
class ParseSource(Base):
__tablename__ = "parse_sources"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_type: Mapped[str] = mapped_column(String(50))
name: Mapped[str] = mapped_column(String(255))
query: Mapped[str | None] = mapped_column(Text, nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
year_from: Mapped[int | None] = mapped_column(nullable=True)
year_to: Mapped[int | None] = mapped_column(nullable=True)
limit: Mapped[int] = mapped_column(default=1000)
enabled: Mapped[bool] = mapped_column(default=True)
last_status: Mapped[str] = mapped_column(String(20), default="idle")
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
last_run_at: Mapped[datetime | None] = mapped_column(nullable=True)
docs_added: Mapped[int] = mapped_column(default=0)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
class StagedWork(Base):
__tablename__ = "staged_works"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
user_id: Mapped[int | None] = mapped_column(nullable=True)
task_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
filename: Mapped[str | None] = mapped_column(Text, nullable=True)
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
text_key: Mapped[str | None] = mapped_column(Text, nullable=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
word_count: Mapped[int | None] = mapped_column(nullable=True)
status: Mapped[str] = mapped_column(String(20), default="pending")
document_id: Mapped[int | None] = mapped_column(nullable=True)
reviewed_by: Mapped[int | None] = mapped_column(nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
__all__ = ["Base", "User", "Task", "Document", "Fingerprint", "ParseSource", "StagedWork"]

View File

@@ -123,7 +123,13 @@ def extract_and_check(
if not text.strip():
raise ValueError("Не удалось извлечь текст из документа")
logger.info(f"Текст извлечён: {len(text)} символов, {len(text.split())} слов")
word_count = len(text.split())
logger.info(f"Текст извлечён: {len(text)} символов, {word_count} слов")
# ──── Автосбор в отстойник (буфер для пополнения базы) ────────────────
# Сохраняем извлечённый текст и метаданные для последующего ручного
# одобрения админом. Не дублируем в базу источников до решения.
_stage_work(task_id, minio_key, filename, text, word_count)
# Разбить на фрагменты
fragments = _split_into_fragments(
@@ -327,3 +333,145 @@ def add_document(doc_data: dict[str, Any]) -> dict[str, Any]:
)
return {"status": "indexed", "doc_id": doc_id}
def _stage_work(
task_id: str,
minio_key: str,
filename: str,
text: str,
word_count: int,
) -> None:
"""Сохранить проверенную работу в отстойник (StagedWork) + текст в MinIO.
Идемпотентно по task_id: повторный вызов (например при requeue) не дублирует.
Ошибки логируются, но не валят основную проверку плагиата.
"""
try:
from app.models import StagedWork
# Сохранить извлечённый текст в bucket staging
text_key = f"text/{task_id}.txt"
minio = get_minio()
bucket = settings.MINIO_BUCKET_STAGING
if not minio.bucket_exists(bucket):
minio.make_bucket(bucket)
data = text.encode("utf-8")
minio.put_object(
bucket, text_key, io.BytesIO(data), length=len(data),
content_type="text/plain; charset=utf-8",
)
with db_session() as session:
existing = session.execute(
select(StagedWork).where(StagedWork.task_id == task_id)
).scalar_one_or_none()
if existing:
return
# Достать user_id из задачи
from app.models import Task
task = session.get(Task, task_id)
user_id = task.user_id if task else None
session.add(StagedWork(
user_id=user_id,
task_id=task_id,
filename=filename,
minio_key=minio_key,
text_key=text_key,
title=Path(filename).stem if filename else None,
word_count=word_count,
status="pending",
))
session.commit()
logger.info(f"Работа задачи {task_id!r} добавлена в отстойник")
except Exception as e:
logger.warning(f"Не удалось добавить работу {task_id!r} в отстойник: {e}")
@celery_app.task(name="index.run_parser")
def run_parser(source_id: int) -> dict[str, Any]:
"""Запустить парсинг источника и наполнить базу документов.
Переиспользует парсеры из scripts/parsers (BaseParser.run) и
задачу add_document для каждого полученного документа.
"""
import sys
from datetime import datetime, timezone
from app.models import ParseSource
# Парсеры лежат в /parsers (скопированы в образ)
if "/parsers" not in sys.path:
sys.path.insert(0, "/parsers")
with db_session() as session:
src = session.get(ParseSource, source_id)
if src is None:
return {"status": "error", "reason": "источник не найден"}
cfg = {
"source_type": src.source_type,
"query": src.query,
"lang": src.lang,
"year_from": src.year_from,
"year_to": src.year_to,
"limit": src.limit,
}
src.last_status = "running"
session.commit()
added = 0
error_msg = None
try:
# Выбрать парсер по типу и собрать совместимые с его fetch() аргументы
stype = cfg["source_type"]
if stype == "openalex":
from openalex import OpenAlexParser as P
fetch_kwargs = {
"query": cfg.get("query") or "",
"limit": cfg["limit"],
"lang": cfg.get("lang"),
"year_from": cfg.get("year_from"),
"year_to": cfg.get("year_to"),
}
elif stype == "cyberleninka":
from cyberleninka import CyberLeninkaParser as P
fetch_kwargs = {"query": cfg.get("query") or "", "limit": cfg["limit"]}
elif stype == "arxiv":
from arxiv import ArxivParser as P
fetch_kwargs = {
"query": cfg.get("query") or "",
"limit": cfg["limit"],
"year_from": cfg.get("year_from"),
}
else:
raise ValueError(f"неизвестный тип источника: {stype}")
parser = P()
# fetch+transform без записи в JSONL — работаем in-memory
raw_docs = parser.fetch(**fetch_kwargs)
for raw in raw_docs:
try:
doc = parser.transform(raw)
if not (doc and doc.get("title") and doc.get("ext_id")):
continue
result = add_document(doc)
if result.get("status") == "indexed":
added += 1
except Exception as e:
logger.warning(f"run_parser: ошибка документа: {e}")
except Exception as e:
error_msg = str(e)[:500]
logger.error(f"run_parser source={source_id} ошибка: {e}", exc_info=True)
with db_session() as session:
src = session.get(ParseSource, source_id)
if src:
src.last_status = "error" if error_msg else "done"
src.last_error = error_msg
src.docs_added = (src.docs_added or 0) + added
src.last_run_at = datetime.now(timezone.utc)
session.commit()
return {"status": "error" if error_msg else "done", "added": added, "error": error_msg}

View File

@@ -0,0 +1,30 @@
"""Минимальные модели для notifier-воркера."""
from datetime import datetime
from sqlalchemy import JSON, ForeignKey, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class User(Base):
__tablename__ = "users"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
email: Mapped[str] = mapped_column(String(255))
name: Mapped[str] = mapped_column(String(255))
plan: Mapped[str] = mapped_column(String(20), default="free")
is_verified: Mapped[bool] = mapped_column(default=False)
class Task(Base):
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
type: Mapped[str] = mapped_column(String(20))
status: Mapped[str] = mapped_column(String(20))
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())