feat: админ-панель, лицензия, тестовый стек на распределённой инфраструктуре
Админ-панель (доступ: is_admin + секретный код сессии в /admin/<code>): - разделы: дашборд (здоровье сервисов, статистика), клиенты, работы, база документов, хранилище MinIO, источники парсинга, отстойник работ - backend: модели ParseSource/StagedWork/AdminSession, миграция 003, core/admin (авторизация), роутер api/admin - frontend: AdminLayout + 7 вкладок, adminApi, вход из кабинета - Celery index.run_parser (фоновый парсинг), автосбор проверенных работ в отстойник с ручным одобрением → индексация в базу Исправления: - openalex parser: тип article (не journal-article), убран is_oa-фильтр - winnowing: приведение xxh64 к signed int64 (фикс bigint out of range) - bcrypt пин 4.0.1 (совместимость с passlib 1.7.4) - alembic: prepend_sys_path + asyncpg-драйвер - frontend: контракт Task (public_id вместо id), react-dropzone Инфраструктура: - docker-compose.test.yml: api+frontend+воркеры локально, БД/кэш/хранилище/ брокер/LLM — на удалённых серверах через .env - .gitignore: исправлено ошибочное игнорирование кода services/*/app/models/ - LICENSE: проприетарная Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,9 @@
|
||||
# Путь к директории с миграциями
|
||||
script_location = alembic
|
||||
|
||||
# Добавить корень проекта (.) в sys.path, чтобы импортировался пакет app
|
||||
prepend_sys_path = .
|
||||
|
||||
# Формат имени файла миграции
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ def get_url() -> str:
|
||||
db = os.getenv("POSTGRES_DB", "antiplagiator")
|
||||
user = os.getenv("POSTGRES_USER", "antiplagiator")
|
||||
password = os.getenv("POSTGRES_PASSWORD", "changeme")
|
||||
return f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{db}"
|
||||
return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{db}"
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
|
||||
88
services/api/alembic/versions/003_admin.py
Normal file
88
services/api/alembic/versions/003_admin.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Админ-панель: is_admin, источники парсинга, отстойник, сессии админа.
|
||||
|
||||
Revision ID: 003
|
||||
Revises: 002
|
||||
Create Date: 2026-05-30
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "003"
|
||||
down_revision = "002"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# users.is_admin
|
||||
op.add_column(
|
||||
"users",
|
||||
sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
# parse_sources
|
||||
op.create_table(
|
||||
"parse_sources",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("source_type", sa.String(50), nullable=False),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("query", sa.Text(), nullable=True),
|
||||
sa.Column("lang", sa.String(10), nullable=True),
|
||||
sa.Column("year_from", sa.Integer(), nullable=True),
|
||||
sa.Column("year_to", sa.Integer(), nullable=True),
|
||||
sa.Column("limit", sa.Integer(), nullable=False, server_default="1000"),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("last_status", sa.String(20), nullable=False, server_default="idle"),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("last_run_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("docs_added", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_parse_sources_source_type", "parse_sources", ["source_type"])
|
||||
|
||||
# staged_works
|
||||
op.create_table(
|
||||
"staged_works",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=True),
|
||||
sa.Column("task_id", sa.String(36), nullable=True),
|
||||
sa.Column("filename", sa.Text(), nullable=True),
|
||||
sa.Column("minio_key", sa.Text(), nullable=True),
|
||||
sa.Column("text_key", sa.Text(), nullable=True),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("authors", sa.JSON(), nullable=True),
|
||||
sa.Column("year", sa.Integer(), nullable=True),
|
||||
sa.Column("lang", sa.String(10), nullable=True),
|
||||
sa.Column("word_count", sa.Integer(), nullable=True),
|
||||
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
|
||||
sa.Column("document_id", sa.Integer(), nullable=True),
|
||||
sa.Column("reviewed_by", sa.Integer(), nullable=True),
|
||||
sa.Column("reviewed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_staged_works_user_id", "staged_works", ["user_id"])
|
||||
op.create_index("ix_staged_works_task_id", "staged_works", ["task_id"])
|
||||
op.create_index("ix_staged_works_status", "staged_works", ["status"])
|
||||
op.create_index("ix_staged_works_created_at", "staged_works", ["created_at"])
|
||||
|
||||
# admin_sessions
|
||||
op.create_table(
|
||||
"admin_sessions",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=False),
|
||||
sa.Column("code", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
|
||||
sa.Column("expires_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("last_used_at", sa.DateTime(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_admin_sessions_user_id", "admin_sessions", ["user_id"])
|
||||
op.create_index("ix_admin_sessions_code", "admin_sessions", ["code"], unique=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("admin_sessions")
|
||||
op.drop_table("staged_works")
|
||||
op.drop_index("ix_parse_sources_source_type", table_name="parse_sources")
|
||||
op.drop_table("parse_sources")
|
||||
op.drop_column("users", "is_admin")
|
||||
571
services/api/app/api/admin.py
Normal file
571
services/api/app/api/admin.py
Normal file
@@ -0,0 +1,571 @@
|
||||
"""Админ-панель: дашборд, клиенты, работы, документы, хранилище, источники, отстойник.
|
||||
|
||||
Все эндпоинты требуют is_admin (get_admin_user). Эндпоинты под /admin/panel/{code}/*
|
||||
дополнительно проверяют секретный код сессии (verify_admin_code).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.core.admin import create_admin_session, get_admin_user
|
||||
from app.core.celery_app import celery_app
|
||||
from app.core.minio_client import get_minio
|
||||
from app.core.redis_client import get_redis
|
||||
from app.database import get_db
|
||||
from app.models.admin import ParseSource, StagedWork
|
||||
from app.models.document import Document, Fingerprint
|
||||
from app.models.task import Task
|
||||
from app.models.user import User
|
||||
from app.schemas.admin import (
|
||||
AdminDocumentResponse,
|
||||
AdminSessionResponse,
|
||||
AdminStats,
|
||||
AdminTaskResponse,
|
||||
AdminUserResponse,
|
||||
AdminUserUpdate,
|
||||
ParseSourceCreate,
|
||||
ParseSourceResponse,
|
||||
ParseSourceUpdate,
|
||||
ServiceHealth,
|
||||
StagedWorkDetail,
|
||||
StagedWorkResponse,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"], dependencies=[Depends(get_admin_user)])
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# СЕССИЯ ДОСТУПА
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.post("/session", response_model=AdminSessionResponse)
|
||||
async def open_admin_session(
|
||||
admin: User = Depends(get_admin_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> AdminSessionResponse:
|
||||
"""Сгенерировать секретный код доступа и ссылку /admin/<code>."""
|
||||
code = await create_admin_session(admin, db)
|
||||
from app.models.admin import AdminSession
|
||||
|
||||
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
|
||||
sess = result.scalar_one()
|
||||
return AdminSessionResponse(
|
||||
code=code,
|
||||
url=f"/admin/{code}",
|
||||
expires_at=sess.expires_at,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/session/verify/{code}")
|
||||
async def verify_session(
|
||||
code: str,
|
||||
admin: User = Depends(get_admin_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Проверить валидность кода (для фронта при загрузке /admin/<code>)."""
|
||||
from app.models.admin import AdminSession
|
||||
|
||||
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
|
||||
sess = result.scalar_one_or_none()
|
||||
if sess is None or sess.user_id != admin.id:
|
||||
return {"valid": False}
|
||||
expires = sess.expires_at
|
||||
if expires.tzinfo is not None:
|
||||
expires = expires.replace(tzinfo=None)
|
||||
return {"valid": expires >= datetime.utcnow()}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# ДАШБОРД / СЕРВИСЫ
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/health", response_model=list[ServiceHealth])
|
||||
async def health(db: AsyncSession = Depends(get_db)) -> list[ServiceHealth]:
|
||||
"""Статус всех сервисов инфраструктуры."""
|
||||
checks: list[ServiceHealth] = []
|
||||
|
||||
# PostgreSQL
|
||||
try:
|
||||
await db.execute(select(func.now()))
|
||||
checks.append(ServiceHealth(name="PostgreSQL", ok=True))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="PostgreSQL", ok=False, detail=str(e)[:200]))
|
||||
|
||||
# Redis
|
||||
try:
|
||||
r = get_redis()
|
||||
await r.ping()
|
||||
checks.append(ServiceHealth(name="Redis", ok=True))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="Redis", ok=False, detail=str(e)[:200]))
|
||||
|
||||
# MinIO
|
||||
try:
|
||||
get_minio().bucket_exists(settings.MINIO_BUCKET_DOCS)
|
||||
checks.append(ServiceHealth(name="MinIO", ok=True))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="MinIO", ok=False, detail=str(e)[:200]))
|
||||
|
||||
# Elasticsearch
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as c:
|
||||
resp = await c.get(f"{settings.ELASTICSEARCH_URL}/_cluster/health")
|
||||
ok = resp.status_code == 200 and resp.json().get("status") != "red"
|
||||
checks.append(ServiceHealth(name="Elasticsearch", ok=ok,
|
||||
detail=resp.json().get("status")))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="Elasticsearch", ok=False, detail=str(e)[:200]))
|
||||
|
||||
# Ollama
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as c:
|
||||
resp = await c.get(f"{settings.OLLAMA_URL}/api/version")
|
||||
checks.append(ServiceHealth(name="Ollama", ok=resp.status_code == 200,
|
||||
detail=resp.json().get("version")))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="Ollama", ok=False, detail=str(e)[:200]))
|
||||
|
||||
# RabbitMQ + воркеры через Celery ping
|
||||
try:
|
||||
insp = celery_app.control.inspect(timeout=3)
|
||||
pong = insp.ping() or {}
|
||||
workers = list(pong.keys())
|
||||
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=bool(workers),
|
||||
detail=f"воркеров онлайн: {len(workers)}"))
|
||||
except Exception as e:
|
||||
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=False, detail=str(e)[:200]))
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
@router.get("/stats", response_model=AdminStats)
|
||||
async def stats(db: AsyncSession = Depends(get_db)) -> AdminStats:
|
||||
"""Сводные счётчики для дашборда."""
|
||||
users_total = (await db.execute(select(func.count()).select_from(User))).scalar_one()
|
||||
|
||||
rows = (await db.execute(select(Task.status, func.count()).group_by(Task.status))).all()
|
||||
tasks_by_status = {s: c for s, c in rows}
|
||||
|
||||
docs_total = (await db.execute(select(func.count()).select_from(Document))).scalar_one()
|
||||
rows = (await db.execute(select(Document.source, func.count()).group_by(Document.source))).all()
|
||||
docs_by_source = {s: c for s, c in rows}
|
||||
|
||||
staging_pending = (
|
||||
await db.execute(
|
||||
select(func.count()).select_from(StagedWork).where(StagedWork.status == "pending")
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
# Хранилище MinIO
|
||||
storage: dict = {}
|
||||
try:
|
||||
client = get_minio()
|
||||
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
|
||||
try:
|
||||
if not client.bucket_exists(bucket):
|
||||
continue
|
||||
total = 0
|
||||
count = 0
|
||||
for obj in client.list_objects(bucket, recursive=True):
|
||||
total += obj.size or 0
|
||||
count += 1
|
||||
storage[bucket] = {"objects": count, "bytes": total}
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
storage = {"error": str(e)[:200]}
|
||||
|
||||
# Длины очередей через RabbitMQ management API (если доступно)
|
||||
queues: dict = {}
|
||||
try:
|
||||
rmq_user = settings.RABBITMQ_URL.split("//")[1].split(":")[0]
|
||||
rmq_pass = settings.RABBITMQ_URL.split(":")[2].split("@")[0]
|
||||
rmq_host = settings.RABBITMQ_URL.split("@")[1].split(":")[0]
|
||||
async with httpx.AsyncClient(timeout=5) as c:
|
||||
resp = await c.get(
|
||||
f"http://{rmq_host}:15672/api/queues",
|
||||
auth=(rmq_user, rmq_pass),
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
for q in resp.json():
|
||||
name = q.get("name", "")
|
||||
if name.startswith("queue."):
|
||||
queues[name] = q.get("messages", 0)
|
||||
except Exception as e:
|
||||
queues = {"error": str(e)[:120]}
|
||||
|
||||
return AdminStats(
|
||||
users_total=users_total,
|
||||
tasks_by_status=tasks_by_status,
|
||||
documents_total=docs_total,
|
||||
documents_by_source=docs_by_source,
|
||||
staging_pending=staging_pending,
|
||||
storage=storage,
|
||||
queues=queues,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# КЛИЕНТЫ
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/users", response_model=list[AdminUserResponse])
|
||||
async def list_users(
|
||||
q: str | None = None,
|
||||
limit: int = Query(50, le=200),
|
||||
offset: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[AdminUserResponse]:
|
||||
stmt = select(User)
|
||||
if q:
|
||||
like = f"%{q.lower()}%"
|
||||
stmt = stmt.where(func.lower(User.email).like(like) | func.lower(User.name).like(like))
|
||||
stmt = stmt.order_by(User.created_at.desc()).limit(limit).offset(offset)
|
||||
users = (await db.execute(stmt)).scalars().all()
|
||||
|
||||
result = []
|
||||
for u in users:
|
||||
cnt = (
|
||||
await db.execute(select(func.count()).select_from(Task).where(Task.user_id == u.id))
|
||||
).scalar_one()
|
||||
item = AdminUserResponse.model_validate(u)
|
||||
item.tasks_count = cnt
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
@router.patch("/users/{user_id}", response_model=AdminUserResponse)
|
||||
async def update_user(
|
||||
user_id: int,
|
||||
data: AdminUserUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> AdminUserResponse:
|
||||
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="Пользователь не найден")
|
||||
if data.plan is not None:
|
||||
user.plan = data.plan
|
||||
if data.is_verified is not None:
|
||||
user.is_verified = data.is_verified
|
||||
if data.is_admin is not None:
|
||||
user.is_admin = data.is_admin
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
# Сбросить кэш пользователя
|
||||
try:
|
||||
await get_redis().delete(f"user:cache:{user_id}")
|
||||
except Exception:
|
||||
pass
|
||||
return AdminUserResponse.model_validate(user)
|
||||
|
||||
|
||||
@router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_user(user_id: int, db: AsyncSession = Depends(get_db)) -> None:
|
||||
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="Пользователь не найден")
|
||||
await db.execute(delete(Task).where(Task.user_id == user_id))
|
||||
await db.delete(user)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# РАБОТЫ (задачи)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/tasks", response_model=list[AdminTaskResponse])
|
||||
async def list_tasks(
|
||||
status_filter: str | None = Query(None, alias="status"),
|
||||
type_filter: str | None = Query(None, alias="type"),
|
||||
user_id: int | None = None,
|
||||
limit: int = Query(50, le=200),
|
||||
offset: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[AdminTaskResponse]:
|
||||
stmt = select(Task)
|
||||
if status_filter:
|
||||
stmt = stmt.where(Task.status == status_filter)
|
||||
if type_filter:
|
||||
stmt = stmt.where(Task.type == type_filter)
|
||||
if user_id:
|
||||
stmt = stmt.where(Task.user_id == user_id)
|
||||
stmt = stmt.order_by(Task.created_at.desc()).limit(limit).offset(offset)
|
||||
tasks = (await db.execute(stmt)).scalars().all()
|
||||
return [AdminTaskResponse.model_validate(t) for t in tasks]
|
||||
|
||||
|
||||
@router.get("/tasks/{public_id}")
|
||||
async def get_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
|
||||
task = (
|
||||
await db.execute(select(Task).where(Task.public_id == public_id))
|
||||
).scalar_one_or_none()
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="Задача не найдена")
|
||||
return {
|
||||
"public_id": task.public_id,
|
||||
"user_id": task.user_id,
|
||||
"type": task.type,
|
||||
"status": task.status,
|
||||
"input_data": task.input_data,
|
||||
"result": task.result,
|
||||
"error": task.error,
|
||||
"created_at": task.created_at,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/{public_id}/requeue")
|
||||
async def requeue_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
|
||||
task = (
|
||||
await db.execute(select(Task).where(Task.public_id == public_id))
|
||||
).scalar_one_or_none()
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="Задача не найдена")
|
||||
if task.type != "plagiarism":
|
||||
raise HTTPException(status_code=400, detail="Перезапуск поддержан только для проверок плагиата")
|
||||
minio_key = (task.input_data or {}).get("minio_key")
|
||||
filename = (task.input_data or {}).get("filename")
|
||||
if not minio_key:
|
||||
raise HTTPException(status_code=400, detail="Нет minio_key для перезапуска")
|
||||
task.status = "queued"
|
||||
task.error = None
|
||||
task.result = None
|
||||
await db.commit()
|
||||
celery_app.send_task(
|
||||
"index.extract_and_check",
|
||||
args=[task.id, minio_key, filename],
|
||||
queue="queue.index",
|
||||
)
|
||||
return {"status": "requeued", "public_id": public_id}
|
||||
|
||||
|
||||
@router.delete("/tasks/{public_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_task(public_id: str, db: AsyncSession = Depends(get_db)) -> None:
|
||||
task = (
|
||||
await db.execute(select(Task).where(Task.public_id == public_id))
|
||||
).scalar_one_or_none()
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="Задача не найдена")
|
||||
await db.delete(task)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# БАЗА ДОКУМЕНТОВ
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/documents", response_model=list[AdminDocumentResponse])
|
||||
async def list_documents(
|
||||
q: str | None = None,
|
||||
source: str | None = None,
|
||||
year: int | None = None,
|
||||
limit: int = Query(50, le=200),
|
||||
offset: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[AdminDocumentResponse]:
|
||||
stmt = select(Document)
|
||||
if q:
|
||||
stmt = stmt.where(func.lower(Document.title).like(f"%{q.lower()}%"))
|
||||
if source:
|
||||
stmt = stmt.where(Document.source == source)
|
||||
if year:
|
||||
stmt = stmt.where(Document.year == year)
|
||||
stmt = stmt.order_by(Document.indexed_at.desc()).limit(limit).offset(offset)
|
||||
docs = (await db.execute(stmt)).scalars().all()
|
||||
return [AdminDocumentResponse.model_validate(d) for d in docs]
|
||||
|
||||
|
||||
@router.delete("/documents/{doc_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_document(doc_id: int, db: AsyncSession = Depends(get_db)) -> None:
|
||||
doc = (await db.execute(select(Document).where(Document.id == doc_id))).scalar_one_or_none()
|
||||
if doc is None:
|
||||
raise HTTPException(status_code=404, detail="Документ не найден")
|
||||
await db.execute(delete(Fingerprint).where(Fingerprint.doc_id == doc_id))
|
||||
await db.delete(doc)
|
||||
await db.commit()
|
||||
# Удалить из Elasticsearch (best-effort)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as c:
|
||||
await c.delete(f"{settings.ELASTICSEARCH_URL}/documents/_doc/{doc_id}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# ХРАНИЛИЩЕ
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/storage")
|
||||
async def storage_info() -> dict:
|
||||
client = get_minio()
|
||||
result: dict = {"buckets": []}
|
||||
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
|
||||
try:
|
||||
if not client.bucket_exists(bucket):
|
||||
continue
|
||||
total = 0
|
||||
objs = []
|
||||
for obj in client.list_objects(bucket, recursive=True):
|
||||
total += obj.size or 0
|
||||
if len(objs) < 100:
|
||||
objs.append({"name": obj.object_name, "size": obj.size})
|
||||
result["buckets"].append({
|
||||
"name": bucket,
|
||||
"objects_total": len(objs),
|
||||
"bytes": total,
|
||||
"sample": objs,
|
||||
})
|
||||
except Exception as e:
|
||||
result["buckets"].append({"name": bucket, "error": str(e)[:120]})
|
||||
return result
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# ИСТОЧНИКИ ПАРСИНГА
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/sources", response_model=list[ParseSourceResponse])
|
||||
async def list_sources(db: AsyncSession = Depends(get_db)) -> list[ParseSourceResponse]:
|
||||
rows = (await db.execute(select(ParseSource).order_by(ParseSource.created_at.desc()))).scalars().all()
|
||||
return [ParseSourceResponse.model_validate(s) for s in rows]
|
||||
|
||||
|
||||
@router.post("/sources", response_model=ParseSourceResponse, status_code=201)
|
||||
async def create_source(data: ParseSourceCreate, db: AsyncSession = Depends(get_db)) -> ParseSourceResponse:
|
||||
if data.source_type not in ("openalex", "cyberleninka", "arxiv"):
|
||||
raise HTTPException(status_code=400, detail="Недопустимый тип источника")
|
||||
src = ParseSource(**data.model_dump())
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
await db.refresh(src)
|
||||
return ParseSourceResponse.model_validate(src)
|
||||
|
||||
|
||||
@router.patch("/sources/{source_id}", response_model=ParseSourceResponse)
|
||||
async def update_source(
|
||||
source_id: int, data: ParseSourceUpdate, db: AsyncSession = Depends(get_db)
|
||||
) -> ParseSourceResponse:
|
||||
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Источник не найден")
|
||||
for k, v in data.model_dump(exclude_unset=True).items():
|
||||
setattr(src, k, v)
|
||||
await db.commit()
|
||||
await db.refresh(src)
|
||||
return ParseSourceResponse.model_validate(src)
|
||||
|
||||
|
||||
@router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_source(source_id: int, db: AsyncSession = Depends(get_db)) -> None:
|
||||
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Источник не найден")
|
||||
await db.delete(src)
|
||||
await db.commit()
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/run")
|
||||
async def run_source(source_id: int, db: AsyncSession = Depends(get_db)) -> dict:
|
||||
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Источник не найден")
|
||||
if src.last_status == "running":
|
||||
raise HTTPException(status_code=409, detail="Источник уже парсится")
|
||||
src.last_status = "running"
|
||||
src.last_error = None
|
||||
src.last_run_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
celery_app.send_task("index.run_parser", args=[source_id], queue="queue.index")
|
||||
return {"status": "started", "source_id": source_id}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# ОТСТОЙНИК
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
@router.get("/staging", response_model=list[StagedWorkResponse])
|
||||
async def list_staging(
|
||||
status_filter: str = Query("pending", alias="status"),
|
||||
limit: int = Query(50, le=200),
|
||||
offset: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[StagedWorkResponse]:
|
||||
stmt = select(StagedWork)
|
||||
if status_filter != "all":
|
||||
stmt = stmt.where(StagedWork.status == status_filter)
|
||||
stmt = stmt.order_by(StagedWork.created_at.desc()).limit(limit).offset(offset)
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
return [StagedWorkResponse.model_validate(s) for s in rows]
|
||||
|
||||
|
||||
@router.get("/staging/{staged_id}", response_model=StagedWorkDetail)
|
||||
async def get_staging(staged_id: int, db: AsyncSession = Depends(get_db)) -> StagedWorkDetail:
|
||||
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
|
||||
if sw is None:
|
||||
raise HTTPException(status_code=404, detail="Работа не найдена")
|
||||
detail = StagedWorkDetail.model_validate(sw)
|
||||
# Превью текста из MinIO
|
||||
if sw.text_key:
|
||||
try:
|
||||
obj = get_minio().get_object("staging", sw.text_key)
|
||||
text = obj.read().decode("utf-8", errors="replace")
|
||||
detail.text_preview = text[:5000]
|
||||
except Exception as e:
|
||||
detail.text_preview = f"(не удалось прочитать текст: {e})"
|
||||
return detail
|
||||
|
||||
|
||||
@router.post("/staging/{staged_id}/approve")
|
||||
async def approve_staging(
|
||||
staged_id: int,
|
||||
admin: User = Depends(get_admin_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
|
||||
if sw is None:
|
||||
raise HTTPException(status_code=404, detail="Работа не найдена")
|
||||
if sw.status != "pending":
|
||||
raise HTTPException(status_code=409, detail=f"Работа уже {sw.status}")
|
||||
|
||||
# Прочитать извлечённый текст
|
||||
full_text = ""
|
||||
if sw.text_key:
|
||||
try:
|
||||
obj = get_minio().get_object("staging", sw.text_key)
|
||||
full_text = obj.read().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Не удалось прочитать текст: {e}")
|
||||
|
||||
# Добавить в базу документов через существующую задачу индексатора
|
||||
doc_data = {
|
||||
"source": "user_submission",
|
||||
"ext_id": f"staged:{sw.id}",
|
||||
"title": sw.title or sw.filename or f"Работа #{sw.id}",
|
||||
"authors": sw.authors or [],
|
||||
"year": sw.year,
|
||||
"lang": sw.lang,
|
||||
"abstract": full_text[:2000],
|
||||
"full_text": full_text,
|
||||
}
|
||||
celery_app.send_task("index.add_document", args=[doc_data], queue="queue.index")
|
||||
|
||||
sw.status = "approved"
|
||||
sw.reviewed_by = admin.id
|
||||
sw.reviewed_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
return {"status": "approved", "id": staged_id}
|
||||
|
||||
|
||||
@router.post("/staging/{staged_id}/reject")
|
||||
async def reject_staging(
|
||||
staged_id: int,
|
||||
admin: User = Depends(get_admin_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
|
||||
if sw is None:
|
||||
raise HTTPException(status_code=404, detail="Работа не найдена")
|
||||
sw.status = "rejected"
|
||||
sw.reviewed_by = admin.id
|
||||
sw.reviewed_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
return {"status": "rejected", "id": staged_id}
|
||||
73
services/api/app/core/admin.py
Normal file
73
services/api/app/core/admin.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""Авторизация админ-панели: проверка роли is_admin и секретного кода сессии."""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import Depends, HTTPException, Path, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.admin import AdminSession
|
||||
from app.models.user import User
|
||||
|
||||
# Срок жизни секретного кода доступа к админке
|
||||
ADMIN_SESSION_TTL_HOURS = 8
|
||||
|
||||
|
||||
async def get_admin_user(current_user: User = Depends(get_current_user)) -> User:
|
||||
"""Dependency: пользователь должен быть аутентифицирован И иметь is_admin."""
|
||||
if not getattr(current_user, "is_admin", False):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Доступ только для администраторов",
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
async def create_admin_session(user: User, db: AsyncSession) -> str:
|
||||
"""Создать новый секретный код доступа к админке, вернуть код."""
|
||||
code = secrets.token_urlsafe(32)
|
||||
# Колонки TIMESTAMP WITHOUT TIME ZONE — храним naive UTC
|
||||
session = AdminSession(
|
||||
user_id=user.id,
|
||||
code=code,
|
||||
expires_at=datetime.utcnow() + timedelta(hours=ADMIN_SESSION_TTL_HOURS),
|
||||
)
|
||||
db.add(session)
|
||||
await db.commit()
|
||||
return code
|
||||
|
||||
|
||||
async def verify_admin_code(
|
||||
code: str = Path(..., description="Секретный код сессии из ссылки /admin/<code>"),
|
||||
admin: User = Depends(get_admin_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""Dependency: проверяет И роль администратора, И валидность кода из пути.
|
||||
|
||||
Двухфакторная защита: ссылка /admin/<code> бесполезна без JWT админа,
|
||||
а JWT админа без верного кода — тоже.
|
||||
"""
|
||||
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
|
||||
session = result.scalar_one_or_none()
|
||||
|
||||
if session is None or session.user_id != admin.id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Неверный код доступа к админке",
|
||||
)
|
||||
|
||||
expires = session.expires_at
|
||||
if expires.tzinfo is not None:
|
||||
expires = expires.replace(tzinfo=None)
|
||||
if expires < datetime.utcnow():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Код доступа истёк, сгенерируйте новый",
|
||||
)
|
||||
|
||||
session.last_used_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
return admin
|
||||
@@ -92,6 +92,7 @@ async def _load_user(user_id: int, db: AsyncSession):
|
||||
"name": user.name,
|
||||
"plan": user.plan,
|
||||
"is_verified": user.is_verified,
|
||||
"is_admin": user.is_admin,
|
||||
}
|
||||
await r.setex(cache_key, _USER_CACHE_TTL, json.dumps(safe))
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api import auth, documents, reports, search, tasks
|
||||
from app.api import admin, auth, documents, reports, search, tasks
|
||||
from app.config import settings
|
||||
from app.core.minio_client import ensure_bucket
|
||||
from app.core.redis_client import close_pool
|
||||
@@ -78,6 +78,7 @@ app.include_router(tasks.router, prefix="/api")
|
||||
app.include_router(search.router, prefix="/api")
|
||||
app.include_router(documents.router, prefix="/api")
|
||||
app.include_router(reports.router, prefix="/api")
|
||||
app.include_router(admin.router, prefix="/api")
|
||||
|
||||
|
||||
# ─── WebSocket ────────────────────────────────────────────────────────────────
|
||||
|
||||
19
services/api/app/models/__init__.py
Normal file
19
services/api/app/models/__init__.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""Экспорт всех моделей для Alembic автоопределения."""
|
||||
|
||||
from app.models.admin import AdminSession, ParseSource, StagedWork
|
||||
from app.models.document import Document, Fingerprint, UsageLog
|
||||
from app.models.task import Task, TaskStatus, TaskType
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = [
|
||||
"User",
|
||||
"Task",
|
||||
"TaskType",
|
||||
"TaskStatus",
|
||||
"Document",
|
||||
"Fingerprint",
|
||||
"UsageLog",
|
||||
"ParseSource",
|
||||
"StagedWork",
|
||||
"AdminSession",
|
||||
]
|
||||
92
services/api/app/models/admin.py
Normal file
92
services/api/app/models/admin.py
Normal file
@@ -0,0 +1,92 @@
|
||||
"""Модели для админ-панели: источники парсинга, отстойник работ, сессии админа."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class ParseSource(Base):
|
||||
"""Источник парсинга для наполнения базы документов.
|
||||
|
||||
Админ задаёт тип источника и параметры запроса; задача index.run_parser
|
||||
запускает парсинг в фоне и наполняет базу.
|
||||
"""
|
||||
|
||||
__tablename__ = "parse_sources"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
# Тип парсера: openalex / cyberleninka / arxiv
|
||||
source_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
# Человекочитаемое имя
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
query: Mapped[str | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
year_from: Mapped[int | None] = mapped_column(nullable=True)
|
||||
year_to: Mapped[int | None] = mapped_column(nullable=True)
|
||||
limit: Mapped[int] = mapped_column(default=1000)
|
||||
enabled: Mapped[bool] = mapped_column(default=True)
|
||||
# Статус последнего запуска: idle / running / done / error
|
||||
last_status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
last_error: Mapped[str | None] = mapped_column(nullable=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
docs_added: Mapped[int] = mapped_column(default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<ParseSource id={self.id} type={self.source_type!r} name={self.name!r}>"
|
||||
|
||||
|
||||
class StagedWork(Base):
|
||||
"""Отстойник: проверенная пользовательская работа, ожидающая решения админа.
|
||||
|
||||
После одобрения работа добавляется в базу документов (index.add_document),
|
||||
после отклонения — помечается rejected.
|
||||
"""
|
||||
|
||||
__tablename__ = "staged_works"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True, index=True)
|
||||
# Внутренний UUID задачи проверки (Task.id)
|
||||
task_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
filename: Mapped[str | None] = mapped_column(nullable=True)
|
||||
# Ключ исходного файла и извлечённого текста в MinIO
|
||||
minio_key: Mapped[str | None] = mapped_column(nullable=True)
|
||||
text_key: Mapped[str | None] = mapped_column(nullable=True)
|
||||
title: Mapped[str | None] = mapped_column(nullable=True)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
word_count: Mapped[int | None] = mapped_column(nullable=True)
|
||||
# pending / approved / rejected
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
|
||||
# ID документа в базе после одобрения
|
||||
document_id: Mapped[int | None] = mapped_column(nullable=True)
|
||||
reviewed_by: Mapped[int | None] = mapped_column(nullable=True)
|
||||
reviewed_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<StagedWork id={self.id} status={self.status!r} title={(self.title or '')[:30]!r}>"
|
||||
|
||||
|
||||
class AdminSession(Base):
|
||||
"""Секретный код доступа к админке (часть ссылки /admin/<code>).
|
||||
|
||||
Доступ проверяется по двум факторам: JWT админа (is_admin) И валидный код.
|
||||
"""
|
||||
|
||||
__tablename__ = "admin_sessions"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), nullable=False, index=True)
|
||||
code: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
expires_at: Mapped[datetime] = mapped_column(nullable=False)
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<AdminSession id={self.id} user_id={self.user_id}>"
|
||||
82
services/api/app/models/document.py
Normal file
82
services/api/app/models/document.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Модели документов, fingerprints и логов использования."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
class Document(Base):
|
||||
"""Документ в базе знаний системы (статья, книга, веб-страница)."""
|
||||
|
||||
__tablename__ = "documents"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
# Источник: openalex, cyberleninka, arxiv, wikipedia_ru, wikipedia_en
|
||||
source: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
# Внешний ID (OpenAlex ID, arXiv ID и т.д.) — уникален в рамках источника
|
||||
ext_id: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
|
||||
doi: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(nullable=False)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True, index=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True, index=True)
|
||||
journal: Mapped[str | None] = mapped_column(nullable=True)
|
||||
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
abstract: Mapped[str | None] = mapped_column(nullable=True)
|
||||
url: Mapped[str | None] = mapped_column(nullable=True)
|
||||
# Ключ объекта в MinIO (полный текст)
|
||||
minio_key: Mapped[str | None] = mapped_column(nullable=True)
|
||||
# ID вектора в FAISS индексе
|
||||
faiss_id: Mapped[int | None] = mapped_column(nullable=True, index=True)
|
||||
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
# Отпечатки (Winnowing хэши)
|
||||
fingerprints: Mapped[list["Fingerprint"]] = relationship(
|
||||
back_populates="document",
|
||||
lazy="select",
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Document id={self.id} source={self.source!r} title={self.title[:40]!r}>"
|
||||
|
||||
|
||||
class Fingerprint(Base):
|
||||
"""Winnowing fingerprint (хэш n-граммы) для быстрого поиска плагиата."""
|
||||
|
||||
__tablename__ = "fingerprints"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
doc_id: Mapped[int] = mapped_column(ForeignKey("documents.id"), index=True)
|
||||
hash_value: Mapped[int] = mapped_column(BigInteger, index=True)
|
||||
position: Mapped[int] = mapped_column() # Позиция в документе
|
||||
|
||||
document: Mapped["Document"] = relationship(back_populates="fingerprints")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Fingerprint doc_id={self.doc_id} hash={self.hash_value}>"
|
||||
|
||||
|
||||
class UsageLog(Base):
|
||||
"""Лог использования для подсчёта лимитов по тарифу."""
|
||||
|
||||
__tablename__ = "usage_logs"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
|
||||
# Действие: search, plagiarism, summarize, gost
|
||||
action: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
|
||||
|
||||
user: Mapped["User"] = relationship(back_populates="usage")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<UsageLog user_id={self.user_id} action={self.action!r}>"
|
||||
75
services/api/app/models/task.py
Normal file
75
services/api/app/models/task.py
Normal file
@@ -0,0 +1,75 @@
|
||||
"""Модель задачи (асинхронная единица работы)."""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
class TaskType(str, Enum):
|
||||
"""Тип задачи."""
|
||||
|
||||
SEARCH = "search"
|
||||
PLAGIARISM = "plagiarism"
|
||||
SUMMARIZE = "summarize"
|
||||
GOST = "gost"
|
||||
|
||||
|
||||
class TaskStatus(str, Enum):
|
||||
"""Статус задачи."""
|
||||
|
||||
QUEUED = "queued"
|
||||
PROCESSING = "processing"
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class Task(Base):
|
||||
"""Задача пользователя, обрабатываемая воркерами Celery."""
|
||||
|
||||
__tablename__ = "tasks"
|
||||
|
||||
# Внутренний UUID — не светится в API/URL
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
primary_key=True,
|
||||
default=lambda: str(uuid4()),
|
||||
)
|
||||
# Публичный ID для URL: /tasks/{public_id}
|
||||
# token_urlsafe(16) → 22 символа base64url, не угадываем, не связан с internal UUID
|
||||
public_id: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
unique=True,
|
||||
index=True,
|
||||
default=lambda: secrets.token_urlsafe(16),
|
||||
)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
|
||||
type: Mapped[str] = mapped_column(String(20), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(20), default=TaskStatus.QUEUED)
|
||||
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
# Входные данные: query, filename, doc_ids и т.д.
|
||||
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
# Результат работы воркера
|
||||
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(nullable=True)
|
||||
queue_position: Mapped[int | None] = mapped_column(nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
updated_at: Mapped[datetime | None] = mapped_column(
|
||||
nullable=True,
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
# Связи
|
||||
user: Mapped["User"] = relationship(back_populates="tasks")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Task id={self.id!r} type={self.type!r} status={self.status!r}>"
|
||||
42
services/api/app/models/user.py
Normal file
42
services/api/app/models/user.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Модель пользователя."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import func, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.task import Task
|
||||
from app.models.document import UsageLog
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Пользователь системы."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
email: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
|
||||
hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
is_verified: Mapped[bool] = mapped_column(default=False)
|
||||
# Администратор системы (доступ к админ-панели)
|
||||
is_admin: Mapped[bool] = mapped_column(default=False)
|
||||
# Тарифный план: free / student / premium / science
|
||||
plan: Mapped[str] = mapped_column(String(20), default="free")
|
||||
verification_token: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
# Связи
|
||||
tasks: Mapped[list["Task"]] = relationship(back_populates="user", lazy="select")
|
||||
usage: Mapped[list["UsageLog"]] = relationship(back_populates="user", lazy="select")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<User id={self.id} email={self.email!r} plan={self.plan!r}>"
|
||||
141
services/api/app/schemas/admin.py
Normal file
141
services/api/app/schemas/admin.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""Pydantic-схемы для админ-панели."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ─── Сессия доступа ───────────────────────────────────────────────────────────
|
||||
class AdminSessionResponse(BaseModel):
|
||||
code: str
|
||||
url: str
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
# ─── Клиенты ──────────────────────────────────────────────────────────────────
|
||||
class AdminUserResponse(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
name: str
|
||||
plan: str
|
||||
is_verified: bool
|
||||
is_admin: bool
|
||||
created_at: datetime
|
||||
tasks_count: int = 0
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class AdminUserUpdate(BaseModel):
|
||||
plan: str | None = None
|
||||
is_verified: bool | None = None
|
||||
is_admin: bool | None = None
|
||||
|
||||
|
||||
# ─── Работы ─────────────────────────────────────────────────────────────────
|
||||
class AdminTaskResponse(BaseModel):
|
||||
public_id: str
|
||||
user_id: int
|
||||
type: str
|
||||
status: str
|
||||
error: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ─── Документы базы ───────────────────────────────────────────────────────────
|
||||
class AdminDocumentResponse(BaseModel):
|
||||
id: int
|
||||
source: str
|
||||
ext_id: str
|
||||
title: str
|
||||
authors: list = Field(default_factory=list)
|
||||
year: int | None = None
|
||||
lang: str | None = None
|
||||
doi: str | None = None
|
||||
url: str | None = None
|
||||
indexed_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ─── Источники парсинга ───────────────────────────────────────────────────────
|
||||
class ParseSourceCreate(BaseModel):
|
||||
source_type: str = Field(description="openalex / cyberleninka / arxiv")
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
query: str | None = None
|
||||
lang: str | None = None
|
||||
year_from: int | None = None
|
||||
year_to: int | None = None
|
||||
limit: int = Field(default=1000, ge=1, le=100000)
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class ParseSourceUpdate(BaseModel):
|
||||
name: str | None = None
|
||||
query: str | None = None
|
||||
lang: str | None = None
|
||||
year_from: int | None = None
|
||||
year_to: int | None = None
|
||||
limit: int | None = Field(default=None, ge=1, le=100000)
|
||||
enabled: bool | None = None
|
||||
|
||||
|
||||
class ParseSourceResponse(BaseModel):
|
||||
id: int
|
||||
source_type: str
|
||||
name: str
|
||||
query: str | None = None
|
||||
lang: str | None = None
|
||||
year_from: int | None = None
|
||||
year_to: int | None = None
|
||||
limit: int
|
||||
enabled: bool
|
||||
last_status: str
|
||||
last_error: str | None = None
|
||||
last_run_at: datetime | None = None
|
||||
docs_added: int
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ─── Отстойник ────────────────────────────────────────────────────────────────
|
||||
class StagedWorkResponse(BaseModel):
|
||||
id: int
|
||||
user_id: int | None = None
|
||||
task_id: str | None = None
|
||||
filename: str | None = None
|
||||
title: str | None = None
|
||||
authors: list = Field(default_factory=list)
|
||||
year: int | None = None
|
||||
lang: str | None = None
|
||||
word_count: int | None = None
|
||||
status: str
|
||||
document_id: int | None = None
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class StagedWorkDetail(StagedWorkResponse):
|
||||
text_preview: str | None = None
|
||||
|
||||
|
||||
# ─── Дашборд ──────────────────────────────────────────────────────────────────
|
||||
class ServiceHealth(BaseModel):
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class AdminStats(BaseModel):
|
||||
users_total: int
|
||||
tasks_by_status: dict[str, int]
|
||||
documents_total: int
|
||||
documents_by_source: dict[str, int]
|
||||
staging_pending: int
|
||||
storage: dict[str, Any]
|
||||
queues: dict[str, Any]
|
||||
@@ -26,6 +26,7 @@ class UserInToken(BaseModel):
|
||||
name: str
|
||||
plan: str
|
||||
is_verified: bool
|
||||
is_admin: bool = False
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ pydantic==2.7.1
|
||||
pydantic-settings==2.2.1
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
bcrypt==4.0.1 # passlib 1.7.4 несовместим с bcrypt>=4.1 (ошибка __about__ / 72-byte limit)
|
||||
python-multipart==0.0.9
|
||||
minio==7.2.7
|
||||
httpx==0.27.0
|
||||
|
||||
13
services/frontend/Dockerfile.dev
Normal file
13
services/frontend/Dockerfile.dev
Normal file
@@ -0,0 +1,13 @@
|
||||
# Dev-образ фронтенда: Vite dev-server с hot reload
|
||||
FROM node:20-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Сначала зависимости (кэш слоёв)
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 5173
|
||||
CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0", "--port", "5173"]
|
||||
2968
services/frontend/package-lock.json
generated
Normal file
2968
services/frontend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -9,28 +9,28 @@
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-query": "^5.40.0",
|
||||
"axios": "^1.7.2",
|
||||
"clsx": "^2.1.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"lucide-react": "^0.390.0",
|
||||
"react": "^18.3.0",
|
||||
"react-dom": "^18.3.0",
|
||||
"react-router-dom": "^6.23.0",
|
||||
"@tanstack/react-query": "^5.40.0",
|
||||
"zustand": "^4.5.2",
|
||||
"axios": "^1.7.2",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"react-dropzone": "^14.4.1",
|
||||
"react-hot-toast": "^2.4.1",
|
||||
"lucide-react": "^0.390.0",
|
||||
"clsx": "^2.1.1",
|
||||
"react-router-dom": "^6.23.0",
|
||||
"tailwind-merge": "^2.3.0",
|
||||
"date-fns": "^3.6.0"
|
||||
"zustand": "^4.5.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.3.3",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react": "^4.3.0",
|
||||
"typescript": "^5.4.5",
|
||||
"vite": "^5.2.12",
|
||||
"tailwindcss": "^3.4.4",
|
||||
"postcss": "^8.4.38",
|
||||
"autoprefixer": "^10.4.19",
|
||||
"eslint": "^9.4.0"
|
||||
"eslint": "^9.4.0",
|
||||
"postcss": "^8.4.38",
|
||||
"tailwindcss": "^3.4.4",
|
||||
"typescript": "^5.4.5",
|
||||
"vite": "^5.2.12"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,3 +78,42 @@ export const reportsApi = {
|
||||
get: (taskId: string) =>
|
||||
api.get(`/reports/${taskId}`),
|
||||
};
|
||||
|
||||
// ─── Админка ────────────────────────────────────────────────────────────────
|
||||
export const adminApi = {
|
||||
openSession: () => api.post('/admin/session'),
|
||||
verifySession: (code: string) => api.get(`/admin/session/verify/${code}`),
|
||||
|
||||
health: () => api.get('/admin/health'),
|
||||
stats: () => api.get('/admin/stats'),
|
||||
|
||||
users: (params?: { q?: string; limit?: number; offset?: number }) =>
|
||||
api.get('/admin/users', { params }),
|
||||
updateUser: (id: number, data: { plan?: string; is_verified?: boolean; is_admin?: boolean }) =>
|
||||
api.patch(`/admin/users/${id}`, data),
|
||||
deleteUser: (id: number) => api.delete(`/admin/users/${id}`),
|
||||
|
||||
tasks: (params?: { status?: string; type?: string; user_id?: number; limit?: number; offset?: number }) =>
|
||||
api.get('/admin/tasks', { params }),
|
||||
task: (publicId: string) => api.get(`/admin/tasks/${publicId}`),
|
||||
requeueTask: (publicId: string) => api.post(`/admin/tasks/${publicId}/requeue`),
|
||||
deleteTask: (publicId: string) => api.delete(`/admin/tasks/${publicId}`),
|
||||
|
||||
documents: (params?: { q?: string; source?: string; year?: number; limit?: number; offset?: number }) =>
|
||||
api.get('/admin/documents', { params }),
|
||||
deleteDocument: (id: number) => api.delete(`/admin/documents/${id}`),
|
||||
|
||||
storage: () => api.get('/admin/storage'),
|
||||
|
||||
sources: () => api.get('/admin/sources'),
|
||||
createSource: (data: Record<string, unknown>) => api.post('/admin/sources', data),
|
||||
updateSource: (id: number, data: Record<string, unknown>) => api.patch(`/admin/sources/${id}`, data),
|
||||
deleteSource: (id: number) => api.delete(`/admin/sources/${id}`),
|
||||
runSource: (id: number) => api.post(`/admin/sources/${id}/run`),
|
||||
|
||||
staging: (params?: { status?: string; limit?: number; offset?: number }) =>
|
||||
api.get('/admin/staging', { params }),
|
||||
stagingDetail: (id: number) => api.get(`/admin/staging/${id}`),
|
||||
approveStaging: (id: number) => api.post(`/admin/staging/${id}/approve`),
|
||||
rejectStaging: (id: number) => api.post(`/admin/staging/${id}/reject`),
|
||||
};
|
||||
|
||||
@@ -71,7 +71,7 @@ export function TaskCard({ task }: TaskCardProps) {
|
||||
|
||||
return (
|
||||
<Link
|
||||
to={`/tasks/${task.id}`}
|
||||
to={`/tasks/${task.public_id}`}
|
||||
className="block group"
|
||||
>
|
||||
<div className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-100 hover:border-brand-200 hover:shadow-sm transition-all duration-200">
|
||||
|
||||
@@ -14,6 +14,14 @@ import { Task } from './pages/Task';
|
||||
import { Pricing } from './pages/Pricing';
|
||||
import { Login } from './pages/Login';
|
||||
import { Register } from './pages/Register';
|
||||
import { AdminLayout } from './pages/admin/AdminLayout';
|
||||
import { Dashboard } from './pages/admin/Dashboard';
|
||||
import { Users as AdminUsers } from './pages/admin/Users';
|
||||
import { Works as AdminWorks } from './pages/admin/Works';
|
||||
import { Documents as AdminDocuments } from './pages/admin/Documents';
|
||||
import { Storage as AdminStorage } from './pages/admin/Storage';
|
||||
import { Sources as AdminSources } from './pages/admin/Sources';
|
||||
import { Staging as AdminStaging } from './pages/admin/Staging';
|
||||
|
||||
import './index.css';
|
||||
|
||||
@@ -26,23 +34,51 @@ const queryClient = new QueryClient({
|
||||
},
|
||||
});
|
||||
|
||||
// Маршруты пользовательской части (обёрнуты в общий Layout)
|
||||
function PublicApp() {
|
||||
return (
|
||||
<Layout>
|
||||
<Routes>
|
||||
<Route path="/" element={<Home />} />
|
||||
<Route path="/search" element={<Search />} />
|
||||
<Route path="/check" element={<Check />} />
|
||||
<Route path="/bibliography" element={<Bibliography />} />
|
||||
<Route path="/cabinet" element={<Cabinet />} />
|
||||
<Route path="/tasks/:taskId" element={<Task />} />
|
||||
<Route path="/pricing" element={<Pricing />} />
|
||||
<Route path="/login" element={<Login />} />
|
||||
<Route path="/register" element={<Register />} />
|
||||
</Routes>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<BrowserRouter>
|
||||
<Layout>
|
||||
<Routes>
|
||||
<Route path="/" element={<Home />} />
|
||||
<Route path="/search" element={<Search />} />
|
||||
<Route path="/check" element={<Check />} />
|
||||
<Route path="/bibliography" element={<Bibliography />} />
|
||||
<Route path="/cabinet" element={<Cabinet />} />
|
||||
<Route path="/tasks/:taskId" element={<Task />} />
|
||||
<Route path="/pricing" element={<Pricing />} />
|
||||
<Route path="/login" element={<Login />} />
|
||||
<Route path="/register" element={<Register />} />
|
||||
</Routes>
|
||||
</Layout>
|
||||
<Routes>
|
||||
{/* Админка — собственный layout, защита по is_admin + коду сессии */}
|
||||
<Route
|
||||
path="/admin/:code/*"
|
||||
element={
|
||||
<AdminLayout>
|
||||
<Routes>
|
||||
<Route path="dashboard" element={<Dashboard />} />
|
||||
<Route path="users" element={<AdminUsers />} />
|
||||
<Route path="works" element={<AdminWorks />} />
|
||||
<Route path="documents" element={<AdminDocuments />} />
|
||||
<Route path="storage" element={<AdminStorage />} />
|
||||
<Route path="sources" element={<AdminSources />} />
|
||||
<Route path="staging" element={<AdminStaging />} />
|
||||
<Route path="*" element={<Dashboard />} />
|
||||
</Routes>
|
||||
</AdminLayout>
|
||||
}
|
||||
/>
|
||||
{/* Всё остальное — пользовательская часть */}
|
||||
<Route path="/*" element={<PublicApp />} />
|
||||
</Routes>
|
||||
</BrowserRouter>
|
||||
<Toaster
|
||||
position="top-right"
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import React from 'react';
|
||||
import { Navigate } from 'react-router-dom';
|
||||
import { Navigate, useNavigate } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Crown, Search, Shield, BookOpen } from 'lucide-react';
|
||||
import { Crown, Search, Shield, BookOpen, ShieldAlert } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { TaskCard } from '../components/TaskCard';
|
||||
import { tasksApi } from '../api/client';
|
||||
import { tasksApi, adminApi } from '../api/client';
|
||||
import { useAuthStore } from '../store/auth';
|
||||
import { PLAN_LIMITS, type Task } from '../types';
|
||||
|
||||
@@ -16,6 +17,16 @@ const PLAN_BADGE_STYLES = {
|
||||
|
||||
export function Cabinet() {
|
||||
const { isAuthenticated, user } = useAuthStore();
|
||||
const navigate = useNavigate();
|
||||
|
||||
const openAdmin = async () => {
|
||||
try {
|
||||
const { data } = await adminApi.openSession();
|
||||
navigate(`${data.url}/dashboard`);
|
||||
} catch {
|
||||
toast.error('Не удалось открыть админ-панель');
|
||||
}
|
||||
};
|
||||
|
||||
if (!isAuthenticated) {
|
||||
return <Navigate to="/login" replace />;
|
||||
@@ -49,9 +60,20 @@ export function Cabinet() {
|
||||
</div>
|
||||
<p className="text-gray-500 text-sm">{user?.email}</p>
|
||||
</div>
|
||||
<button className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors">
|
||||
Обновить план
|
||||
</button>
|
||||
<div className="flex items-center gap-2">
|
||||
{user?.is_admin && (
|
||||
<button
|
||||
onClick={openAdmin}
|
||||
className="flex items-center gap-1.5 px-4 py-2 text-sm font-medium bg-gray-900 text-white rounded-lg hover:bg-gray-800 transition-colors"
|
||||
>
|
||||
<ShieldAlert className="w-4 h-4" />
|
||||
Админ-панель
|
||||
</button>
|
||||
)}
|
||||
<button className="px-4 py-2 text-sm font-medium bg-brand-600 text-white rounded-lg hover:bg-brand-700 transition-colors">
|
||||
Обновить план
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -116,7 +138,7 @@ export function Cabinet() {
|
||||
{tasks && tasks.length > 0 && (
|
||||
<div className="space-y-2">
|
||||
{tasks.map((task) => (
|
||||
<TaskCard key={task.id} task={task} />
|
||||
<TaskCard key={task.public_id} task={task} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -53,14 +53,14 @@ export function Check() {
|
||||
</div>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">Проверка запущена</h2>
|
||||
<p className="text-gray-500 mb-2">
|
||||
Файл <strong>{(submittedTask.input_data as any).filename}</strong> передан на проверку.
|
||||
Файл <strong>{file?.name ?? 'документ'}</strong> передан на проверку.
|
||||
</p>
|
||||
<p className="text-sm text-gray-400 mb-6">
|
||||
Вы получите email когда проверка завершится. Обычно это занимает 1-3 минуты.
|
||||
</p>
|
||||
<div className="flex flex-col sm:flex-row gap-3 justify-center">
|
||||
<Link
|
||||
to={`/tasks/${submittedTask.id}`}
|
||||
to={`/tasks/${submittedTask.public_id}`}
|
||||
className="flex items-center justify-center gap-2 px-5 py-2.5 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700 transition-colors"
|
||||
>
|
||||
Следить за прогрессом
|
||||
|
||||
@@ -27,7 +27,7 @@ export function Search() {
|
||||
mutationFn: searchApi.create,
|
||||
onSuccess: (response) => {
|
||||
const task: Task = response.data;
|
||||
setTaskId(task.id);
|
||||
setTaskId(task.public_id);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
const msg = error.response?.data?.detail || 'Ошибка поиска';
|
||||
|
||||
@@ -60,7 +60,7 @@ export function Task() {
|
||||
</h1>
|
||||
<div className="flex items-center gap-2 mt-0.5">
|
||||
<StatusBadge status={task.status} />
|
||||
<span className="text-xs text-gray-400">{task.id}</span>
|
||||
<span className="text-xs text-gray-400">{task.public_id}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -91,8 +91,7 @@ export function Task() {
|
||||
{task.status === 'done' && task.type === 'search' && task.result && (
|
||||
<div className="space-y-4">
|
||||
<p className="text-sm text-gray-500">
|
||||
Запрос: <strong>«{(task.input_data as any).query}»</strong>
|
||||
{' · '}{(task.result as SearchResultData).total} источников
|
||||
Найдено {(task.result as SearchResultData).total} источников
|
||||
</p>
|
||||
{(task.result as SearchResultData).sources.map((source) => (
|
||||
<SourceCard key={source.id} source={source} />
|
||||
@@ -104,7 +103,7 @@ export function Task() {
|
||||
{task.status === 'done' && task.type === 'plagiarism' && task.result && (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-6">
|
||||
<h2 className="text-base font-semibold text-gray-900 mb-4">
|
||||
Файл: {(task.input_data as any).filename}
|
||||
Результат проверки плагиата
|
||||
</h2>
|
||||
<PlagiarismReport data={task.result as PlagiarismResultData} />
|
||||
</div>
|
||||
|
||||
102
services/frontend/src/pages/admin/AdminLayout.tsx
Normal file
102
services/frontend/src/pages/admin/AdminLayout.tsx
Normal file
@@ -0,0 +1,102 @@
|
||||
import React from 'react';
|
||||
import { NavLink, useParams, useNavigate, Navigate } from 'react-router-dom';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
LayoutDashboard, Users, FileText, Database, HardDrive, Download, Inbox, LogOut, ShieldAlert,
|
||||
} from 'lucide-react';
|
||||
import { clsx } from 'clsx';
|
||||
import { adminApi } from '../../api/client';
|
||||
import { useAuthStore } from '../../store/auth';
|
||||
|
||||
const NAV = [
|
||||
{ to: 'dashboard', label: 'Дашборд', icon: LayoutDashboard },
|
||||
{ to: 'users', label: 'Клиенты', icon: Users },
|
||||
{ to: 'works', label: 'Работы', icon: FileText },
|
||||
{ to: 'documents', label: 'База', icon: Database },
|
||||
{ to: 'storage', label: 'Хранилище', icon: HardDrive },
|
||||
{ to: 'sources', label: 'Источники', icon: Download },
|
||||
{ to: 'staging', label: 'Отстойник', icon: Inbox },
|
||||
];
|
||||
|
||||
interface AdminLayoutProps {
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
export function AdminLayout({ children }: AdminLayoutProps) {
|
||||
const { code } = useParams<{ code: string }>();
|
||||
const { isAuthenticated, user, logout } = useAuthStore();
|
||||
const navigate = useNavigate();
|
||||
|
||||
// Доступ: только аутентифицированный админ
|
||||
const { data: verify, isLoading } = useQuery({
|
||||
queryKey: ['admin-verify', code],
|
||||
queryFn: () => adminApi.verifySession(code!).then((r) => r.data),
|
||||
enabled: !!code && isAuthenticated && !!user?.is_admin,
|
||||
retry: false,
|
||||
});
|
||||
|
||||
if (!isAuthenticated || !user?.is_admin) {
|
||||
return <Navigate to="/login" replace />;
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return <div className="min-h-screen flex items-center justify-center text-gray-400">Проверка доступа…</div>;
|
||||
}
|
||||
|
||||
if (!verify?.valid) {
|
||||
return (
|
||||
<div className="min-h-screen flex flex-col items-center justify-center gap-4 bg-gray-50">
|
||||
<ShieldAlert className="w-12 h-12 text-red-400" />
|
||||
<p className="text-gray-600">Код доступа недействителен или истёк.</p>
|
||||
<button
|
||||
onClick={() => navigate('/cabinet')}
|
||||
className="px-4 py-2 bg-brand-600 text-white rounded-lg text-sm"
|
||||
>
|
||||
В кабинет
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="min-h-screen flex bg-gray-50">
|
||||
{/* Сайдбар */}
|
||||
<aside className="w-56 bg-gray-900 text-gray-300 flex flex-col fixed inset-y-0 left-0">
|
||||
<div className="px-5 py-5 border-b border-gray-800 flex items-center gap-2">
|
||||
<ShieldAlert className="w-5 h-5 text-amber-400" />
|
||||
<span className="font-semibold text-white">Админ-панель</span>
|
||||
</div>
|
||||
<nav className="flex-1 px-3 py-4 space-y-1">
|
||||
{NAV.map(({ to, label, icon: Icon }) => (
|
||||
<NavLink
|
||||
key={to}
|
||||
to={`/admin/${code}/${to}`}
|
||||
className={({ isActive }) =>
|
||||
clsx(
|
||||
'flex items-center gap-3 px-3 py-2.5 rounded-lg text-sm font-medium transition-colors',
|
||||
isActive ? 'bg-brand-600 text-white' : 'hover:bg-gray-800 hover:text-white'
|
||||
)
|
||||
}
|
||||
>
|
||||
<Icon className="w-4 h-4" />
|
||||
{label}
|
||||
</NavLink>
|
||||
))}
|
||||
</nav>
|
||||
<div className="px-3 py-4 border-t border-gray-800">
|
||||
<div className="px-3 mb-2 text-xs text-gray-500">{user?.email}</div>
|
||||
<button
|
||||
onClick={() => { logout(); navigate('/'); }}
|
||||
className="w-full flex items-center gap-3 px-3 py-2.5 rounded-lg text-sm hover:bg-gray-800 hover:text-white transition-colors"
|
||||
>
|
||||
<LogOut className="w-4 h-4" />
|
||||
Выйти
|
||||
</button>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* Контент */}
|
||||
<main className="flex-1 ml-56 p-8">{children}</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
119
services/frontend/src/pages/admin/Dashboard.tsx
Normal file
119
services/frontend/src/pages/admin/Dashboard.tsx
Normal file
@@ -0,0 +1,119 @@
|
||||
import React from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { CheckCircle2, XCircle, Users, FileText, Database, Inbox } from 'lucide-react';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface Health { name: string; ok: boolean; detail?: string }
|
||||
interface Stats {
|
||||
users_total: number;
|
||||
tasks_by_status: Record<string, number>;
|
||||
documents_total: number;
|
||||
documents_by_source: Record<string, number>;
|
||||
staging_pending: number;
|
||||
storage: Record<string, { objects: number; bytes: number } | unknown>;
|
||||
queues: Record<string, number | string>;
|
||||
}
|
||||
|
||||
function fmtBytes(b: number): string {
|
||||
if (b < 1024) return `${b} Б`;
|
||||
if (b < 1024 ** 2) return `${(b / 1024).toFixed(1)} КБ`;
|
||||
if (b < 1024 ** 3) return `${(b / 1024 ** 2).toFixed(1)} МБ`;
|
||||
return `${(b / 1024 ** 3).toFixed(2)} ГБ`;
|
||||
}
|
||||
|
||||
export function Dashboard() {
|
||||
const { data: health } = useQuery({
|
||||
queryKey: ['admin-health'],
|
||||
queryFn: () => adminApi.health().then((r) => r.data as Health[]),
|
||||
refetchInterval: 10000,
|
||||
});
|
||||
const { data: stats } = useQuery({
|
||||
queryKey: ['admin-stats'],
|
||||
queryFn: () => adminApi.stats().then((r) => r.data as Stats),
|
||||
refetchInterval: 10000,
|
||||
});
|
||||
|
||||
const totalStorage = Object.values(stats?.storage || {}).reduce(
|
||||
(acc: number, v) => acc + (typeof v === 'object' && v && 'bytes' in v ? (v as { bytes: number }).bytes : 0),
|
||||
0
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Дашборд</h1>
|
||||
|
||||
{/* Счётчики */}
|
||||
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
<StatCard icon={Users} label="Клиентов" value={stats?.users_total ?? '—'} color="text-blue-600" />
|
||||
<StatCard icon={FileText} label="Работ всего" value={Object.values(stats?.tasks_by_status || {}).reduce((a, b) => a + b, 0)} color="text-purple-600" />
|
||||
<StatCard icon={Database} label="Документов в базе" value={stats?.documents_total ?? '—'} color="text-emerald-600" />
|
||||
<StatCard icon={Inbox} label="В отстойнике" value={stats?.staging_pending ?? '—'} color="text-amber-600" />
|
||||
</div>
|
||||
|
||||
<div className="grid lg:grid-cols-2 gap-6">
|
||||
{/* Здоровье сервисов */}
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5">
|
||||
<h2 className="font-semibold text-gray-800 mb-4">Сервисы</h2>
|
||||
<div className="space-y-2">
|
||||
{health?.map((h) => (
|
||||
<div key={h.name} className="flex items-center justify-between py-1.5">
|
||||
<div className="flex items-center gap-2">
|
||||
{h.ok ? <CheckCircle2 className="w-4 h-4 text-emerald-500" /> : <XCircle className="w-4 h-4 text-red-500" />}
|
||||
<span className="text-sm text-gray-700">{h.name}</span>
|
||||
</div>
|
||||
<span className={`text-xs ${h.ok ? 'text-gray-400' : 'text-red-500'}`}>{h.detail || (h.ok ? 'OK' : 'недоступен')}</span>
|
||||
</div>
|
||||
))}
|
||||
{!health && <div className="text-sm text-gray-400">Загрузка…</div>}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Работы по статусам + очереди */}
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5 space-y-4">
|
||||
<div>
|
||||
<h2 className="font-semibold text-gray-800 mb-3">Работы по статусам</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{Object.entries(stats?.tasks_by_status || {}).map(([s, c]) => (
|
||||
<span key={s} className="px-3 py-1 bg-gray-100 rounded-lg text-sm text-gray-700">{s}: <b>{c}</b></span>
|
||||
))}
|
||||
{!Object.keys(stats?.tasks_by_status || {}).length && <span className="text-sm text-gray-400">нет данных</span>}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<h2 className="font-semibold text-gray-800 mb-3">Очереди</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{Object.entries(stats?.queues || {}).map(([q, n]) => (
|
||||
<span key={q} className="px-3 py-1 bg-gray-100 rounded-lg text-sm text-gray-700">{q}: <b>{String(n)}</b></span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<h2 className="font-semibold text-gray-800 mb-2">Хранилище</h2>
|
||||
<p className="text-sm text-gray-600">Всего: <b>{fmtBytes(totalStorage)}</b></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* База по источникам */}
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5">
|
||||
<h2 className="font-semibold text-gray-800 mb-3">База документов по источникам</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{Object.entries(stats?.documents_by_source || {}).map(([s, c]) => (
|
||||
<span key={s} className="px-3 py-1 bg-emerald-50 text-emerald-700 rounded-lg text-sm">{s}: <b>{c}</b></span>
|
||||
))}
|
||||
{!Object.keys(stats?.documents_by_source || {}).length && <span className="text-sm text-gray-400">база пуста</span>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StatCard({ icon: Icon, label, value, color }: { icon: React.ElementType; label: string; value: React.ReactNode; color: string }) {
|
||||
return (
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5">
|
||||
<Icon className={`w-6 h-6 ${color} mb-2`} />
|
||||
<div className="text-2xl font-bold text-gray-900">{value}</div>
|
||||
<div className="text-sm text-gray-500">{label}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
71
services/frontend/src/pages/admin/Documents.tsx
Normal file
71
services/frontend/src/pages/admin/Documents.tsx
Normal file
@@ -0,0 +1,71 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { Trash2, Search } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface Doc {
|
||||
id: number; source: string; title: string; authors: Array<{ last_name?: string }>;
|
||||
year: number | null; lang: string | null; url: string | null; indexed_at: string;
|
||||
}
|
||||
|
||||
export function Documents() {
|
||||
const qc = useQueryClient();
|
||||
const [q, setQ] = useState('');
|
||||
const [source, setSource] = useState('');
|
||||
const { data: docs } = useQuery({
|
||||
queryKey: ['admin-docs', q, source],
|
||||
queryFn: () => adminApi.documents({ q: q || undefined, source: source || undefined }).then((r) => r.data as Doc[]),
|
||||
});
|
||||
const del = useMutation({
|
||||
mutationFn: (id: number) => adminApi.deleteDocument(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-docs'] }); toast.success('Удалён из базы'); },
|
||||
onError: () => toast.error('Ошибка'),
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">База документов</h1>
|
||||
<div className="flex gap-3">
|
||||
<div className="relative flex-1 max-w-sm">
|
||||
<Search className="w-4 h-4 text-gray-400 absolute left-3 top-1/2 -translate-y-1/2" />
|
||||
<input value={q} onChange={(e) => setQ(e.target.value)} placeholder="Поиск по названию"
|
||||
className="w-full pl-9 pr-3 py-2 border border-gray-200 rounded-lg text-sm" />
|
||||
</div>
|
||||
<select value={source} onChange={(e) => setSource(e.target.value)}
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm">
|
||||
<option value="">все источники</option>
|
||||
<option value="openalex">openalex</option>
|
||||
<option value="cyberleninka">cyberleninka</option>
|
||||
<option value="arxiv">arxiv</option>
|
||||
<option value="user_submission">работы пользователей</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-gray-50 text-gray-500 text-left">
|
||||
<tr>
|
||||
<th className="px-4 py-3">Название</th><th className="px-4 py-3">Источник</th>
|
||||
<th className="px-4 py-3">Год</th><th className="px-4 py-3">Язык</th><th className="px-4 py-3"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-50">
|
||||
{docs?.map((d) => (
|
||||
<tr key={d.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 text-gray-800 max-w-md truncate">{d.title}</td>
|
||||
<td className="px-4 py-3"><span className="px-2 py-0.5 bg-emerald-50 text-emerald-700 rounded text-xs">{d.source}</span></td>
|
||||
<td className="px-4 py-3 text-gray-600">{d.year || '—'}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{d.lang || '—'}</td>
|
||||
<td className="px-4 py-3">
|
||||
<button onClick={() => { if (confirm('Удалить документ из базы?')) del.mutate(d.id); }}
|
||||
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!docs?.length && <div className="p-6 text-center text-gray-400 text-sm">База пуста</div>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
114
services/frontend/src/pages/admin/Sources.tsx
Normal file
114
services/frontend/src/pages/admin/Sources.tsx
Normal file
@@ -0,0 +1,114 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { Play, Trash2, Plus } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface Source {
|
||||
id: number; source_type: string; name: string; query: string | null;
|
||||
lang: string | null; year_from: number | null; year_to: number | null;
|
||||
limit: number; enabled: boolean; last_status: string; last_error: string | null;
|
||||
last_run_at: string | null; docs_added: number;
|
||||
}
|
||||
|
||||
const STATUS_COLORS: Record<string, string> = {
|
||||
idle: 'bg-gray-100 text-gray-600',
|
||||
running: 'bg-blue-100 text-blue-700',
|
||||
done: 'bg-emerald-100 text-emerald-700',
|
||||
error: 'bg-red-100 text-red-700',
|
||||
};
|
||||
|
||||
export function Sources() {
|
||||
const qc = useQueryClient();
|
||||
const [form, setForm] = useState({ source_type: 'openalex', name: '', query: '', lang: '', limit: 100 });
|
||||
const { data: sources } = useQuery({
|
||||
queryKey: ['admin-sources'],
|
||||
queryFn: () => adminApi.sources().then((r) => r.data as Source[]),
|
||||
refetchInterval: 5000,
|
||||
});
|
||||
|
||||
const create = useMutation({
|
||||
mutationFn: (data: Record<string, unknown>) => adminApi.createSource(data),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Источник добавлен'); setForm({ source_type: 'openalex', name: '', query: '', lang: '', limit: 100 }); },
|
||||
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
|
||||
});
|
||||
const run = useMutation({
|
||||
mutationFn: (id: number) => adminApi.runSource(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Парсинг запущен'); },
|
||||
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
|
||||
});
|
||||
const del = useMutation({
|
||||
mutationFn: (id: number) => adminApi.deleteSource(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-sources'] }); toast.success('Удалён'); },
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Источники парсинга</h1>
|
||||
|
||||
{/* Форма добавления */}
|
||||
<div className="bg-white rounded-xl border border-gray-100 p-5">
|
||||
<h2 className="font-semibold text-gray-800 mb-3">Добавить источник</h2>
|
||||
<div className="grid grid-cols-2 lg:grid-cols-5 gap-3">
|
||||
<select value={form.source_type} onChange={(e) => setForm({ ...form, source_type: e.target.value })}
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm">
|
||||
<option value="openalex">OpenAlex</option>
|
||||
<option value="cyberleninka">КиберЛенинка</option>
|
||||
<option value="arxiv">arXiv</option>
|
||||
</select>
|
||||
<input value={form.name} onChange={(e) => setForm({ ...form, name: e.target.value })} placeholder="Название"
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
|
||||
<input value={form.query} onChange={(e) => setForm({ ...form, query: e.target.value })} placeholder="Запрос"
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
|
||||
<input value={form.lang} onChange={(e) => setForm({ ...form, lang: e.target.value })} placeholder="Язык (ru/en)"
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
|
||||
<input type="number" value={form.limit} onChange={(e) => setForm({ ...form, limit: Number(e.target.value) })} placeholder="Лимит"
|
||||
className="border border-gray-200 rounded-lg px-3 py-2 text-sm" />
|
||||
</div>
|
||||
<button
|
||||
onClick={() => {
|
||||
if (!form.name) { toast.error('Укажите название'); return; }
|
||||
create.mutate({ ...form, lang: form.lang || null, query: form.query || null });
|
||||
}}
|
||||
className="mt-3 flex items-center gap-2 px-4 py-2 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700"
|
||||
>
|
||||
<Plus className="w-4 h-4" /> Добавить
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Список */}
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-gray-50 text-gray-500 text-left">
|
||||
<tr>
|
||||
<th className="px-4 py-3">Название</th><th className="px-4 py-3">Тип</th>
|
||||
<th className="px-4 py-3">Запрос</th><th className="px-4 py-3">Лимит</th>
|
||||
<th className="px-4 py-3">Статус</th><th className="px-4 py-3">Добавлено</th><th className="px-4 py-3"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-50">
|
||||
{sources?.map((s) => (
|
||||
<tr key={s.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 text-gray-800">{s.name}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{s.source_type}</td>
|
||||
<td className="px-4 py-3 text-gray-500 max-w-[160px] truncate">{s.query || '—'}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{s.limit}</td>
|
||||
<td className="px-4 py-3">
|
||||
<span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[s.last_status] || 'bg-gray-100'}`} title={s.last_error || ''}>{s.last_status}</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-gray-600">{s.docs_added}</td>
|
||||
<td className="px-4 py-3 flex gap-1">
|
||||
<button onClick={() => run.mutate(s.id)} disabled={s.last_status === 'running'} title="Запустить"
|
||||
className="p-1.5 text-gray-400 hover:text-emerald-600 rounded disabled:opacity-40"><Play className="w-4 h-4" /></button>
|
||||
<button onClick={() => { if (confirm('Удалить источник?')) del.mutate(s.id); }} title="Удалить"
|
||||
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!sources?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет источников</div>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
119
services/frontend/src/pages/admin/Staging.tsx
Normal file
119
services/frontend/src/pages/admin/Staging.tsx
Normal file
@@ -0,0 +1,119 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { Check, X, Eye } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface Staged {
|
||||
id: number; user_id: number | null; filename: string | null; title: string | null;
|
||||
word_count: number | null; status: string; document_id: number | null; created_at: string;
|
||||
}
|
||||
interface StagedDetail extends Staged { text_preview: string | null }
|
||||
|
||||
const STATUS_COLORS: Record<string, string> = {
|
||||
pending: 'bg-amber-100 text-amber-700',
|
||||
approved: 'bg-emerald-100 text-emerald-700',
|
||||
rejected: 'bg-red-100 text-red-700',
|
||||
};
|
||||
|
||||
export function Staging() {
|
||||
const qc = useQueryClient();
|
||||
const [status, setStatus] = useState('pending');
|
||||
const [preview, setPreview] = useState<StagedDetail | null>(null);
|
||||
|
||||
const { data: items } = useQuery({
|
||||
queryKey: ['admin-staging', status],
|
||||
queryFn: () => adminApi.staging({ status }).then((r) => r.data as Staged[]),
|
||||
refetchInterval: 8000,
|
||||
});
|
||||
|
||||
const approve = useMutation({
|
||||
mutationFn: (id: number) => adminApi.approveStaging(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-staging'] }); setPreview(null); toast.success('Одобрено — добавляется в базу'); },
|
||||
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
|
||||
});
|
||||
const reject = useMutation({
|
||||
mutationFn: (id: number) => adminApi.rejectStaging(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-staging'] }); setPreview(null); toast.success('Отклонено'); },
|
||||
});
|
||||
|
||||
const openPreview = async (id: number) => {
|
||||
const { data } = await adminApi.stagingDetail(id);
|
||||
setPreview(data as StagedDetail);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Отстойник проверенных работ</h1>
|
||||
<p className="text-sm text-gray-500">Работы, прошедшие проверку. Одобрение добавляет их в базу источников для будущих сравнений.</p>
|
||||
|
||||
<div className="flex gap-2">
|
||||
{['pending', 'approved', 'rejected', 'all'].map((s) => (
|
||||
<button key={s} onClick={() => setStatus(s)}
|
||||
className={`px-3 py-1.5 rounded-lg text-sm ${status === s ? 'bg-brand-600 text-white' : 'bg-white border border-gray-200 text-gray-600'}`}>
|
||||
{s === 'all' ? 'все' : s}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-gray-50 text-gray-500 text-left">
|
||||
<tr>
|
||||
<th className="px-4 py-3">Файл</th><th className="px-4 py-3">Клиент</th>
|
||||
<th className="px-4 py-3">Слов</th><th className="px-4 py-3">Статус</th>
|
||||
<th className="px-4 py-3">Дата</th><th className="px-4 py-3"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-50">
|
||||
{items?.map((s) => (
|
||||
<tr key={s.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 text-gray-800 max-w-xs truncate">{s.filename || s.title || `#${s.id}`}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{s.user_id ? `#${s.user_id}` : '—'}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{s.word_count ?? '—'}</td>
|
||||
<td className="px-4 py-3"><span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[s.status]}`}>{s.status}</span></td>
|
||||
<td className="px-4 py-3 text-gray-400 text-xs">{new Date(s.created_at).toLocaleString('ru')}</td>
|
||||
<td className="px-4 py-3 flex gap-1">
|
||||
<button onClick={() => openPreview(s.id)} title="Просмотр"
|
||||
className="p-1.5 text-gray-400 hover:text-brand-600 rounded"><Eye className="w-4 h-4" /></button>
|
||||
{s.status === 'pending' && (
|
||||
<>
|
||||
<button onClick={() => approve.mutate(s.id)} title="Одобрить"
|
||||
className="p-1.5 text-gray-400 hover:text-emerald-600 rounded"><Check className="w-4 h-4" /></button>
|
||||
<button onClick={() => reject.mutate(s.id)} title="Отклонить"
|
||||
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><X className="w-4 h-4" /></button>
|
||||
</>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!items?.length && <div className="p-6 text-center text-gray-400 text-sm">Пусто</div>}
|
||||
</div>
|
||||
|
||||
{/* Превью */}
|
||||
{preview && (
|
||||
<div className="fixed inset-0 bg-black/40 flex items-center justify-center z-50 p-6" onClick={() => setPreview(null)}>
|
||||
<div className="bg-white rounded-xl max-w-2xl w-full max-h-[80vh] flex flex-col" onClick={(e) => e.stopPropagation()}>
|
||||
<div className="px-5 py-4 border-b border-gray-100 flex items-center justify-between">
|
||||
<h3 className="font-semibold text-gray-800 truncate">{preview.filename || preview.title}</h3>
|
||||
<button onClick={() => setPreview(null)} className="text-gray-400 hover:text-gray-600"><X className="w-5 h-5" /></button>
|
||||
</div>
|
||||
<div className="p-5 overflow-auto text-sm text-gray-700 whitespace-pre-wrap flex-1">
|
||||
{preview.text_preview || '(нет текста)'}
|
||||
</div>
|
||||
{preview.status === 'pending' && (
|
||||
<div className="px-5 py-4 border-t border-gray-100 flex gap-2 justify-end">
|
||||
<button onClick={() => reject.mutate(preview.id)}
|
||||
className="px-4 py-2 border border-gray-200 text-gray-600 rounded-lg text-sm">Отклонить</button>
|
||||
<button onClick={() => approve.mutate(preview.id)}
|
||||
className="px-4 py-2 bg-emerald-600 text-white rounded-lg text-sm">Одобрить → в базу</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
57
services/frontend/src/pages/admin/Storage.tsx
Normal file
57
services/frontend/src/pages/admin/Storage.tsx
Normal file
@@ -0,0 +1,57 @@
|
||||
import React from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { HardDrive } from 'lucide-react';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface Bucket {
|
||||
name: string; objects_total?: number; bytes?: number;
|
||||
sample?: Array<{ name: string; size: number }>; error?: string;
|
||||
}
|
||||
|
||||
function fmtBytes(b: number): string {
|
||||
if (b < 1024) return `${b} Б`;
|
||||
if (b < 1024 ** 2) return `${(b / 1024).toFixed(1)} КБ`;
|
||||
if (b < 1024 ** 3) return `${(b / 1024 ** 2).toFixed(1)} МБ`;
|
||||
return `${(b / 1024 ** 3).toFixed(2)} ГБ`;
|
||||
}
|
||||
|
||||
export function Storage() {
|
||||
const { data } = useQuery({
|
||||
queryKey: ['admin-storage'],
|
||||
queryFn: () => adminApi.storage().then((r) => r.data as { buckets: Bucket[] }),
|
||||
refetchInterval: 15000,
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Хранилище (MinIO)</h1>
|
||||
<div className="grid lg:grid-cols-3 gap-4">
|
||||
{data?.buckets.map((b) => (
|
||||
<div key={b.name} className="bg-white rounded-xl border border-gray-100 p-5">
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<HardDrive className="w-5 h-5 text-brand-600" />
|
||||
<span className="font-semibold text-gray-800">{b.name}</span>
|
||||
</div>
|
||||
{b.error ? (
|
||||
<p className="text-sm text-red-500">{b.error}</p>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-sm text-gray-600">Объектов: <b>{b.objects_total}</b></p>
|
||||
<p className="text-sm text-gray-600">Объём: <b>{fmtBytes(b.bytes || 0)}</b></p>
|
||||
<div className="mt-3 max-h-40 overflow-auto space-y-1">
|
||||
{b.sample?.map((o) => (
|
||||
<div key={o.name} className="flex justify-between text-xs text-gray-400">
|
||||
<span className="truncate max-w-[160px]">{o.name}</span>
|
||||
<span>{fmtBytes(o.size)}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{!data?.buckets.length && <div className="text-gray-400 text-sm">Нет бакетов</div>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
92
services/frontend/src/pages/admin/Users.tsx
Normal file
92
services/frontend/src/pages/admin/Users.tsx
Normal file
@@ -0,0 +1,92 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { Trash2, Search } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface AdminUser {
|
||||
id: number; email: string; name: string; plan: string;
|
||||
is_verified: boolean; is_admin: boolean; created_at: string; tasks_count: number;
|
||||
}
|
||||
|
||||
const PLANS = ['free', 'student', 'premium', 'science'];
|
||||
|
||||
export function Users() {
|
||||
const qc = useQueryClient();
|
||||
const [q, setQ] = useState('');
|
||||
const { data: users } = useQuery({
|
||||
queryKey: ['admin-users', q],
|
||||
queryFn: () => adminApi.users({ q: q || undefined }).then((r) => r.data as AdminUser[]),
|
||||
});
|
||||
|
||||
const update = useMutation({
|
||||
mutationFn: ({ id, data }: { id: number; data: Record<string, unknown> }) => adminApi.updateUser(id, data),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-users'] }); toast.success('Сохранено'); },
|
||||
onError: () => toast.error('Ошибка'),
|
||||
});
|
||||
const del = useMutation({
|
||||
mutationFn: (id: number) => adminApi.deleteUser(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-users'] }); toast.success('Удалён'); },
|
||||
onError: () => toast.error('Ошибка удаления'),
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Клиенты</h1>
|
||||
<div className="relative max-w-sm">
|
||||
<Search className="w-4 h-4 text-gray-400 absolute left-3 top-1/2 -translate-y-1/2" />
|
||||
<input
|
||||
value={q} onChange={(e) => setQ(e.target.value)} placeholder="Поиск по email/имени"
|
||||
className="w-full pl-9 pr-3 py-2 border border-gray-200 rounded-lg text-sm"
|
||||
/>
|
||||
</div>
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-gray-50 text-gray-500 text-left">
|
||||
<tr>
|
||||
<th className="px-4 py-3">Email</th><th className="px-4 py-3">Имя</th>
|
||||
<th className="px-4 py-3">Тариф</th><th className="px-4 py-3">Работ</th>
|
||||
<th className="px-4 py-3">Verified</th><th className="px-4 py-3">Admin</th>
|
||||
<th className="px-4 py-3"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-50">
|
||||
{users?.map((u) => (
|
||||
<tr key={u.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 text-gray-800">{u.email}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{u.name}</td>
|
||||
<td className="px-4 py-3">
|
||||
<select
|
||||
value={u.plan}
|
||||
onChange={(e) => update.mutate({ id: u.id, data: { plan: e.target.value } })}
|
||||
className="border border-gray-200 rounded px-2 py-1 text-xs"
|
||||
>
|
||||
{PLANS.map((p) => <option key={p} value={p}>{p}</option>)}
|
||||
</select>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-gray-600">{u.tasks_count}</td>
|
||||
<td className="px-4 py-3">
|
||||
<input type="checkbox" checked={u.is_verified}
|
||||
onChange={(e) => update.mutate({ id: u.id, data: { is_verified: e.target.checked } })} />
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<input type="checkbox" checked={u.is_admin}
|
||||
onChange={(e) => update.mutate({ id: u.id, data: { is_admin: e.target.checked } })} />
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<button
|
||||
onClick={() => { if (confirm(`Удалить ${u.email}?`)) del.mutate(u.id); }}
|
||||
className="p-1.5 text-gray-400 hover:text-red-500 rounded"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!users?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет клиентов</div>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
81
services/frontend/src/pages/admin/Works.tsx
Normal file
81
services/frontend/src/pages/admin/Works.tsx
Normal file
@@ -0,0 +1,81 @@
|
||||
import React, { useState } from 'react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { RefreshCw, Trash2 } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { adminApi } from '../../api/client';
|
||||
|
||||
interface AdminTask {
|
||||
public_id: string; user_id: number; type: string; status: string;
|
||||
error: string | null; created_at: string;
|
||||
}
|
||||
|
||||
const STATUS_COLORS: Record<string, string> = {
|
||||
queued: 'bg-gray-100 text-gray-600',
|
||||
processing: 'bg-blue-100 text-blue-700',
|
||||
done: 'bg-emerald-100 text-emerald-700',
|
||||
failed: 'bg-red-100 text-red-700',
|
||||
};
|
||||
|
||||
export function Works() {
|
||||
const qc = useQueryClient();
|
||||
const [status, setStatus] = useState('');
|
||||
const { data: tasks } = useQuery({
|
||||
queryKey: ['admin-tasks', status],
|
||||
queryFn: () => adminApi.tasks({ status: status || undefined }).then((r) => r.data as AdminTask[]),
|
||||
refetchInterval: 8000,
|
||||
});
|
||||
|
||||
const requeue = useMutation({
|
||||
mutationFn: (id: string) => adminApi.requeueTask(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-tasks'] }); toast.success('Перезапущено'); },
|
||||
onError: (e: any) => toast.error(e.response?.data?.detail || 'Ошибка'),
|
||||
});
|
||||
const del = useMutation({
|
||||
mutationFn: (id: string) => adminApi.deleteTask(id),
|
||||
onSuccess: () => { qc.invalidateQueries({ queryKey: ['admin-tasks'] }); toast.success('Удалено'); },
|
||||
onError: () => toast.error('Ошибка'),
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Работы</h1>
|
||||
<div className="flex gap-2">
|
||||
{['', 'queued', 'processing', 'done', 'failed'].map((s) => (
|
||||
<button key={s} onClick={() => setStatus(s)}
|
||||
className={`px-3 py-1.5 rounded-lg text-sm ${status === s ? 'bg-brand-600 text-white' : 'bg-white border border-gray-200 text-gray-600'}`}>
|
||||
{s || 'все'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="bg-white rounded-xl border border-gray-100 overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-gray-50 text-gray-500 text-left">
|
||||
<tr>
|
||||
<th className="px-4 py-3">ID</th><th className="px-4 py-3">Клиент</th>
|
||||
<th className="px-4 py-3">Тип</th><th className="px-4 py-3">Статус</th>
|
||||
<th className="px-4 py-3">Создано</th><th className="px-4 py-3"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-50">
|
||||
{tasks?.map((t) => (
|
||||
<tr key={t.public_id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 font-mono text-xs text-gray-500">{t.public_id.slice(0, 10)}…</td>
|
||||
<td className="px-4 py-3 text-gray-600">#{t.user_id}</td>
|
||||
<td className="px-4 py-3 text-gray-600">{t.type}</td>
|
||||
<td className="px-4 py-3"><span className={`px-2 py-0.5 rounded text-xs ${STATUS_COLORS[t.status] || 'bg-gray-100'}`}>{t.status}</span></td>
|
||||
<td className="px-4 py-3 text-gray-400 text-xs">{new Date(t.created_at).toLocaleString('ru')}</td>
|
||||
<td className="px-4 py-3 flex gap-1">
|
||||
<button onClick={() => requeue.mutate(t.public_id)} title="Перезапустить"
|
||||
className="p-1.5 text-gray-400 hover:text-brand-600 rounded"><RefreshCw className="w-4 h-4" /></button>
|
||||
<button onClick={() => { if (confirm('Удалить работу?')) del.mutate(t.public_id); }} title="Удалить"
|
||||
className="p-1.5 text-gray-400 hover:text-red-500 rounded"><Trash2 className="w-4 h-4" /></button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!tasks?.length && <div className="p-6 text-center text-gray-400 text-sm">Нет работ</div>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -12,6 +12,7 @@ export interface User {
|
||||
name: string;
|
||||
plan: UserPlan;
|
||||
is_verified: boolean;
|
||||
is_admin?: boolean;
|
||||
}
|
||||
|
||||
// ─── Источник в результатах поиска ────────────────────────────────────────────
|
||||
@@ -88,12 +89,12 @@ export interface GostResultData {
|
||||
export type TaskResult = SearchResultData | PlagiarismResultData | GostResultData | null;
|
||||
|
||||
export interface Task {
|
||||
id: string;
|
||||
// Бэкенд отдаёт public_id (а не внутренний id) и не возвращает input_data
|
||||
public_id: string;
|
||||
type: TaskType;
|
||||
status: TaskStatus;
|
||||
queue_position?: number;
|
||||
eta_seconds?: number;
|
||||
input_data: Record<string, unknown>;
|
||||
result?: TaskResult;
|
||||
error?: string;
|
||||
created_at: string;
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
|
||||
const apiTarget = process.env.VITE_API_TARGET || 'http://localhost:8000';
|
||||
const wsTarget = apiTarget.replace(/^http/, 'ws');
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 5173,
|
||||
host: true,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8000',
|
||||
target: apiTarget,
|
||||
changeOrigin: true,
|
||||
},
|
||||
'/ws': {
|
||||
target: 'ws://localhost:8000',
|
||||
target: wsTarget,
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
|
||||
39
services/worker-gost/app/models/__init__.py
Normal file
39
services/worker-gost/app/models/__init__.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Минимальные модели для gost-воркера."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(Integer)
|
||||
type: Mapped[str] = mapped_column(String(20))
|
||||
status: Mapped[str] = mapped_column(String(20))
|
||||
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Document(Base):
|
||||
__tablename__ = "documents"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(50))
|
||||
ext_id: Mapped[str] = mapped_column(String(255))
|
||||
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -12,10 +12,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Сделать python3.11 дефолтным
|
||||
# Сделать python3.11 дефолтным и поставить pip именно для 3.11 (на CUDA-базе
|
||||
# python3-pip ставит pip для системного python3.10 — он не виден из python3.11)
|
||||
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
|
||||
update-alternatives --install /usr/bin/python python python3.11 1 && \
|
||||
python3.11 -m pip install --upgrade pip
|
||||
update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
|
||||
curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ class Settings(BaseSettings):
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore", # игнорировать поля общего .env, не относящиеся к воркеру
|
||||
)
|
||||
|
||||
# PostgreSQL
|
||||
|
||||
3
services/worker-gpu/app/models/__init__.py
Normal file
3
services/worker-gpu/app/models/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from app.models.base import Base, Document, Task, User
|
||||
|
||||
__all__ = ["Base", "User", "Task", "Document"]
|
||||
67
services/worker-gpu/app/models/base.py
Normal file
67
services/worker-gpu/app/models/base.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Базовые модели данных для GPU воркера (минимальный набор для работы с БД)."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""Базовый класс SQLAlchemy моделей."""
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Минимальная модель пользователя для GPU воркера."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(255))
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
plan: Mapped[str] = mapped_column(String(20), default="free")
|
||||
is_verified: Mapped[bool] = mapped_column(default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Task(Base):
|
||||
"""Модель задачи."""
|
||||
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
|
||||
type: Mapped[str] = mapped_column(String(20))
|
||||
status: Mapped[str] = mapped_column(String(20), default="queued")
|
||||
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
queue_position: Mapped[int | None] = mapped_column(nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
updated_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
|
||||
|
||||
class Document(Base):
|
||||
"""Модель документа в базе знаний."""
|
||||
|
||||
__tablename__ = "documents"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(50))
|
||||
ext_id: Mapped[str] = mapped_column(String(255))
|
||||
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
faiss_id: Mapped[int | None] = mapped_column(nullable=True)
|
||||
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
@@ -3,7 +3,7 @@ redis==5.0.4
|
||||
sqlalchemy==2.0.30
|
||||
psycopg2-binary==2.9.9
|
||||
sentence-transformers==3.0.0
|
||||
faiss-gpu==1.7.4
|
||||
faiss-cpu==1.8.0 # faiss-gpu нет в pip для py3.11; индексы в коде CPU-типа, GPU занят эмбеддингами (torch) и LLM (Ollama)
|
||||
torch==2.3.0
|
||||
numpy==1.26.4
|
||||
httpx==0.27.0
|
||||
|
||||
@@ -31,9 +31,12 @@ def hash_ngram(ngram: str) -> int:
|
||||
ngram: n-грамма для хэширования
|
||||
|
||||
Returns:
|
||||
64-битный хэш
|
||||
Знаковый 64-битный хэш (умещается в PostgreSQL BIGINT)
|
||||
"""
|
||||
return xxhash.xxh64(ngram.encode("utf-8")).intdigest()
|
||||
# xxh64 даёт unsigned 64-bit (до 2^64-1), а PostgreSQL BIGINT — signed
|
||||
# (макс 2^63-1). Приводим к диапазону signed int64.
|
||||
unsigned = xxhash.xxh64(ngram.encode("utf-8")).intdigest()
|
||||
return unsigned - (1 << 64) if unsigned >= (1 << 63) else unsigned
|
||||
|
||||
|
||||
def winnow(text: str, k: int = 5, window: int = 4) -> set[int]:
|
||||
|
||||
@@ -10,6 +10,7 @@ class Settings(BaseSettings):
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# PostgreSQL
|
||||
@@ -30,6 +31,7 @@ class Settings(BaseSettings):
|
||||
MINIO_ACCESS_KEY: str = "minioadmin"
|
||||
MINIO_SECRET_KEY: str = "changeme"
|
||||
MINIO_BUCKET_DOCS: str = "documents"
|
||||
MINIO_BUCKET_STAGING: str = "staging"
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_URL: str = "http://elasticsearch:9200"
|
||||
|
||||
103
services/worker-indexer/app/models/__init__.py
Normal file
103
services/worker-indexer/app/models/__init__.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Минимальные модели для индексер-воркера."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(255))
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
plan: Mapped[str] = mapped_column(String(20), default="free")
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
|
||||
type: Mapped[str] = mapped_column(String(20))
|
||||
status: Mapped[str] = mapped_column(String(20), default="queued")
|
||||
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
queue_position: Mapped[int | None] = mapped_column(nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
updated_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
|
||||
|
||||
class Document(Base):
|
||||
__tablename__ = "documents"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(50))
|
||||
ext_id: Mapped[str] = mapped_column(String(255), unique=True)
|
||||
doi: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
journal: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
abstract: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
faiss_id: Mapped[int | None] = mapped_column(nullable=True)
|
||||
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Fingerprint(Base):
|
||||
__tablename__ = "fingerprints"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
doc_id: Mapped[int] = mapped_column(ForeignKey("documents.id"))
|
||||
hash_value: Mapped[int] = mapped_column(BigInteger)
|
||||
position: Mapped[int] = mapped_column()
|
||||
|
||||
|
||||
class ParseSource(Base):
|
||||
__tablename__ = "parse_sources"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_type: Mapped[str] = mapped_column(String(50))
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
query: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
year_from: Mapped[int | None] = mapped_column(nullable=True)
|
||||
year_to: Mapped[int | None] = mapped_column(nullable=True)
|
||||
limit: Mapped[int] = mapped_column(default=1000)
|
||||
enabled: Mapped[bool] = mapped_column(default=True)
|
||||
last_status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
docs_added: Mapped[int] = mapped_column(default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class StagedWork(Base):
|
||||
__tablename__ = "staged_works"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(nullable=True)
|
||||
task_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
filename: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
minio_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
text_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
authors: Mapped[list] = mapped_column(JSON, default=list)
|
||||
year: Mapped[int | None] = mapped_column(nullable=True)
|
||||
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
|
||||
word_count: Mapped[int | None] = mapped_column(nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
document_id: Mapped[int | None] = mapped_column(nullable=True)
|
||||
reviewed_by: Mapped[int | None] = mapped_column(nullable=True)
|
||||
reviewed_at: Mapped[datetime | None] = mapped_column(nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
__all__ = ["Base", "User", "Task", "Document", "Fingerprint", "ParseSource", "StagedWork"]
|
||||
@@ -123,7 +123,13 @@ def extract_and_check(
|
||||
if not text.strip():
|
||||
raise ValueError("Не удалось извлечь текст из документа")
|
||||
|
||||
logger.info(f"Текст извлечён: {len(text)} символов, {len(text.split())} слов")
|
||||
word_count = len(text.split())
|
||||
logger.info(f"Текст извлечён: {len(text)} символов, {word_count} слов")
|
||||
|
||||
# ──── Автосбор в отстойник (буфер для пополнения базы) ────────────────
|
||||
# Сохраняем извлечённый текст и метаданные для последующего ручного
|
||||
# одобрения админом. Не дублируем в базу источников до решения.
|
||||
_stage_work(task_id, minio_key, filename, text, word_count)
|
||||
|
||||
# Разбить на фрагменты
|
||||
fragments = _split_into_fragments(
|
||||
@@ -327,3 +333,145 @@ def add_document(doc_data: dict[str, Any]) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
return {"status": "indexed", "doc_id": doc_id}
|
||||
|
||||
|
||||
def _stage_work(
|
||||
task_id: str,
|
||||
minio_key: str,
|
||||
filename: str,
|
||||
text: str,
|
||||
word_count: int,
|
||||
) -> None:
|
||||
"""Сохранить проверенную работу в отстойник (StagedWork) + текст в MinIO.
|
||||
|
||||
Идемпотентно по task_id: повторный вызов (например при requeue) не дублирует.
|
||||
Ошибки логируются, но не валят основную проверку плагиата.
|
||||
"""
|
||||
try:
|
||||
from app.models import StagedWork
|
||||
|
||||
# Сохранить извлечённый текст в bucket staging
|
||||
text_key = f"text/{task_id}.txt"
|
||||
minio = get_minio()
|
||||
bucket = settings.MINIO_BUCKET_STAGING
|
||||
if not minio.bucket_exists(bucket):
|
||||
minio.make_bucket(bucket)
|
||||
data = text.encode("utf-8")
|
||||
minio.put_object(
|
||||
bucket, text_key, io.BytesIO(data), length=len(data),
|
||||
content_type="text/plain; charset=utf-8",
|
||||
)
|
||||
|
||||
with db_session() as session:
|
||||
existing = session.execute(
|
||||
select(StagedWork).where(StagedWork.task_id == task_id)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
return
|
||||
|
||||
# Достать user_id из задачи
|
||||
from app.models import Task
|
||||
task = session.get(Task, task_id)
|
||||
user_id = task.user_id if task else None
|
||||
|
||||
session.add(StagedWork(
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
filename=filename,
|
||||
minio_key=minio_key,
|
||||
text_key=text_key,
|
||||
title=Path(filename).stem if filename else None,
|
||||
word_count=word_count,
|
||||
status="pending",
|
||||
))
|
||||
session.commit()
|
||||
logger.info(f"Работа задачи {task_id!r} добавлена в отстойник")
|
||||
except Exception as e:
|
||||
logger.warning(f"Не удалось добавить работу {task_id!r} в отстойник: {e}")
|
||||
|
||||
|
||||
@celery_app.task(name="index.run_parser")
|
||||
def run_parser(source_id: int) -> dict[str, Any]:
|
||||
"""Запустить парсинг источника и наполнить базу документов.
|
||||
|
||||
Переиспользует парсеры из scripts/parsers (BaseParser.run) и
|
||||
задачу add_document для каждого полученного документа.
|
||||
"""
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.models import ParseSource
|
||||
|
||||
# Парсеры лежат в /parsers (скопированы в образ)
|
||||
if "/parsers" not in sys.path:
|
||||
sys.path.insert(0, "/parsers")
|
||||
|
||||
with db_session() as session:
|
||||
src = session.get(ParseSource, source_id)
|
||||
if src is None:
|
||||
return {"status": "error", "reason": "источник не найден"}
|
||||
cfg = {
|
||||
"source_type": src.source_type,
|
||||
"query": src.query,
|
||||
"lang": src.lang,
|
||||
"year_from": src.year_from,
|
||||
"year_to": src.year_to,
|
||||
"limit": src.limit,
|
||||
}
|
||||
src.last_status = "running"
|
||||
session.commit()
|
||||
|
||||
added = 0
|
||||
error_msg = None
|
||||
try:
|
||||
# Выбрать парсер по типу и собрать совместимые с его fetch() аргументы
|
||||
stype = cfg["source_type"]
|
||||
if stype == "openalex":
|
||||
from openalex import OpenAlexParser as P
|
||||
fetch_kwargs = {
|
||||
"query": cfg.get("query") or "",
|
||||
"limit": cfg["limit"],
|
||||
"lang": cfg.get("lang"),
|
||||
"year_from": cfg.get("year_from"),
|
||||
"year_to": cfg.get("year_to"),
|
||||
}
|
||||
elif stype == "cyberleninka":
|
||||
from cyberleninka import CyberLeninkaParser as P
|
||||
fetch_kwargs = {"query": cfg.get("query") or "", "limit": cfg["limit"]}
|
||||
elif stype == "arxiv":
|
||||
from arxiv import ArxivParser as P
|
||||
fetch_kwargs = {
|
||||
"query": cfg.get("query") or "",
|
||||
"limit": cfg["limit"],
|
||||
"year_from": cfg.get("year_from"),
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"неизвестный тип источника: {stype}")
|
||||
|
||||
parser = P()
|
||||
# fetch+transform без записи в JSONL — работаем in-memory
|
||||
raw_docs = parser.fetch(**fetch_kwargs)
|
||||
for raw in raw_docs:
|
||||
try:
|
||||
doc = parser.transform(raw)
|
||||
if not (doc and doc.get("title") and doc.get("ext_id")):
|
||||
continue
|
||||
result = add_document(doc)
|
||||
if result.get("status") == "indexed":
|
||||
added += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"run_parser: ошибка документа: {e}")
|
||||
except Exception as e:
|
||||
error_msg = str(e)[:500]
|
||||
logger.error(f"run_parser source={source_id} ошибка: {e}", exc_info=True)
|
||||
|
||||
with db_session() as session:
|
||||
src = session.get(ParseSource, source_id)
|
||||
if src:
|
||||
src.last_status = "error" if error_msg else "done"
|
||||
src.last_error = error_msg
|
||||
src.docs_added = (src.docs_added or 0) + added
|
||||
src.last_run_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
|
||||
return {"status": "error" if error_msg else "done", "added": added, "error": error_msg}
|
||||
|
||||
30
services/worker-notifier/app/models/__init__.py
Normal file
30
services/worker-notifier/app/models/__init__.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""Минимальные модели для notifier-воркера."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(255))
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
plan: Mapped[str] = mapped_column(String(20), default="free")
|
||||
is_verified: Mapped[bool] = mapped_column(default=False)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
|
||||
type: Mapped[str] = mapped_column(String(20))
|
||||
status: Mapped[str] = mapped_column(String(20))
|
||||
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
Reference in New Issue
Block a user