feat: админ-панель, лицензия, тестовый стек на распределённой инфраструктуре

Админ-панель (доступ: is_admin + секретный код сессии в /admin/<code>):
- разделы: дашборд (здоровье сервисов, статистика), клиенты, работы,
  база документов, хранилище MinIO, источники парсинга, отстойник работ
- backend: модели ParseSource/StagedWork/AdminSession, миграция 003,
  core/admin (авторизация), роутер api/admin
- frontend: AdminLayout + 7 вкладок, adminApi, вход из кабинета
- Celery index.run_parser (фоновый парсинг), автосбор проверенных работ
  в отстойник с ручным одобрением → индексация в базу

Исправления:
- openalex parser: тип article (не journal-article), убран is_oa-фильтр
- winnowing: приведение xxh64 к signed int64 (фикс bigint out of range)
- bcrypt пин 4.0.1 (совместимость с passlib 1.7.4)
- alembic: prepend_sys_path + asyncpg-драйвер
- frontend: контракт Task (public_id вместо id), react-dropzone

Инфраструктура:
- docker-compose.test.yml: api+frontend+воркеры локально, БД/кэш/хранилище/
  брокер/LLM — на удалённых серверах через .env
- .gitignore: исправлено ошибочное игнорирование кода services/*/app/models/
- LICENSE: проприетарная

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-05-30 20:45:53 +05:00
parent 9ee5ccdc3f
commit 9894fa9320
50 changed files with 5680 additions and 58 deletions

View File

@@ -4,6 +4,9 @@
# Путь к директории с миграциями
script_location = alembic
# Добавить корень проекта (.) в sys.path, чтобы импортировался пакет app
prepend_sys_path = .
# Формат имени файла миграции
file_template = %%(rev)s_%%(slug)s

View File

@@ -30,7 +30,7 @@ def get_url() -> str:
db = os.getenv("POSTGRES_DB", "antiplagiator")
user = os.getenv("POSTGRES_USER", "antiplagiator")
password = os.getenv("POSTGRES_PASSWORD", "changeme")
return f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{db}"
return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{db}"
def run_migrations_offline() -> None:

View File

@@ -0,0 +1,88 @@
"""Админ-панель: is_admin, источники парсинга, отстойник, сессии админа.
Revision ID: 003
Revises: 002
Create Date: 2026-05-30
"""
from alembic import op
import sqlalchemy as sa
revision = "003"
down_revision = "002"
branch_labels = None
depends_on = None
def upgrade() -> None:
# users.is_admin
op.add_column(
"users",
sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.false()),
)
# parse_sources
op.create_table(
"parse_sources",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("source_type", sa.String(50), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("query", sa.Text(), nullable=True),
sa.Column("lang", sa.String(10), nullable=True),
sa.Column("year_from", sa.Integer(), nullable=True),
sa.Column("year_to", sa.Integer(), nullable=True),
sa.Column("limit", sa.Integer(), nullable=False, server_default="1000"),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("last_status", sa.String(20), nullable=False, server_default="idle"),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("last_run_at", sa.DateTime(), nullable=True),
sa.Column("docs_added", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
)
op.create_index("ix_parse_sources_source_type", "parse_sources", ["source_type"])
# staged_works
op.create_table(
"staged_works",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=True),
sa.Column("task_id", sa.String(36), nullable=True),
sa.Column("filename", sa.Text(), nullable=True),
sa.Column("minio_key", sa.Text(), nullable=True),
sa.Column("text_key", sa.Text(), nullable=True),
sa.Column("title", sa.Text(), nullable=True),
sa.Column("authors", sa.JSON(), nullable=True),
sa.Column("year", sa.Integer(), nullable=True),
sa.Column("lang", sa.String(10), nullable=True),
sa.Column("word_count", sa.Integer(), nullable=True),
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
sa.Column("document_id", sa.Integer(), nullable=True),
sa.Column("reviewed_by", sa.Integer(), nullable=True),
sa.Column("reviewed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
)
op.create_index("ix_staged_works_user_id", "staged_works", ["user_id"])
op.create_index("ix_staged_works_task_id", "staged_works", ["task_id"])
op.create_index("ix_staged_works_status", "staged_works", ["status"])
op.create_index("ix_staged_works_created_at", "staged_works", ["created_at"])
# admin_sessions
op.create_table(
"admin_sessions",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id"), nullable=False),
sa.Column("code", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
sa.Column("expires_at", sa.DateTime(), nullable=False),
sa.Column("last_used_at", sa.DateTime(), nullable=True),
)
op.create_index("ix_admin_sessions_user_id", "admin_sessions", ["user_id"])
op.create_index("ix_admin_sessions_code", "admin_sessions", ["code"], unique=True)
def downgrade() -> None:
op.drop_table("admin_sessions")
op.drop_table("staged_works")
op.drop_index("ix_parse_sources_source_type", table_name="parse_sources")
op.drop_table("parse_sources")
op.drop_column("users", "is_admin")

View File

@@ -0,0 +1,571 @@
"""Админ-панель: дашборд, клиенты, работы, документы, хранилище, источники, отстойник.
Все эндпоинты требуют is_admin (get_admin_user). Эндпоинты под /admin/panel/{code}/*
дополнительно проверяют секретный код сессии (verify_admin_code).
"""
import logging
from datetime import datetime, timezone
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.core.admin import create_admin_session, get_admin_user
from app.core.celery_app import celery_app
from app.core.minio_client import get_minio
from app.core.redis_client import get_redis
from app.database import get_db
from app.models.admin import ParseSource, StagedWork
from app.models.document import Document, Fingerprint
from app.models.task import Task
from app.models.user import User
from app.schemas.admin import (
AdminDocumentResponse,
AdminSessionResponse,
AdminStats,
AdminTaskResponse,
AdminUserResponse,
AdminUserUpdate,
ParseSourceCreate,
ParseSourceResponse,
ParseSourceUpdate,
ServiceHealth,
StagedWorkDetail,
StagedWorkResponse,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin", tags=["admin"], dependencies=[Depends(get_admin_user)])
# ═══════════════════════════════════════════════════════════════════════════════
# СЕССИЯ ДОСТУПА
# ═══════════════════════════════════════════════════════════════════════════════
@router.post("/session", response_model=AdminSessionResponse)
async def open_admin_session(
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> AdminSessionResponse:
"""Сгенерировать секретный код доступа и ссылку /admin/<code>."""
code = await create_admin_session(admin, db)
from app.models.admin import AdminSession
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
sess = result.scalar_one()
return AdminSessionResponse(
code=code,
url=f"/admin/{code}",
expires_at=sess.expires_at,
)
@router.get("/session/verify/{code}")
async def verify_session(
code: str,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Проверить валидность кода (для фронта при загрузке /admin/<code>)."""
from app.models.admin import AdminSession
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
sess = result.scalar_one_or_none()
if sess is None or sess.user_id != admin.id:
return {"valid": False}
expires = sess.expires_at
if expires.tzinfo is not None:
expires = expires.replace(tzinfo=None)
return {"valid": expires >= datetime.utcnow()}
# ═══════════════════════════════════════════════════════════════════════════════
# ДАШБОРД / СЕРВИСЫ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/health", response_model=list[ServiceHealth])
async def health(db: AsyncSession = Depends(get_db)) -> list[ServiceHealth]:
"""Статус всех сервисов инфраструктуры."""
checks: list[ServiceHealth] = []
# PostgreSQL
try:
await db.execute(select(func.now()))
checks.append(ServiceHealth(name="PostgreSQL", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="PostgreSQL", ok=False, detail=str(e)[:200]))
# Redis
try:
r = get_redis()
await r.ping()
checks.append(ServiceHealth(name="Redis", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="Redis", ok=False, detail=str(e)[:200]))
# MinIO
try:
get_minio().bucket_exists(settings.MINIO_BUCKET_DOCS)
checks.append(ServiceHealth(name="MinIO", ok=True))
except Exception as e:
checks.append(ServiceHealth(name="MinIO", ok=False, detail=str(e)[:200]))
# Elasticsearch
try:
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(f"{settings.ELASTICSEARCH_URL}/_cluster/health")
ok = resp.status_code == 200 and resp.json().get("status") != "red"
checks.append(ServiceHealth(name="Elasticsearch", ok=ok,
detail=resp.json().get("status")))
except Exception as e:
checks.append(ServiceHealth(name="Elasticsearch", ok=False, detail=str(e)[:200]))
# Ollama
try:
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(f"{settings.OLLAMA_URL}/api/version")
checks.append(ServiceHealth(name="Ollama", ok=resp.status_code == 200,
detail=resp.json().get("version")))
except Exception as e:
checks.append(ServiceHealth(name="Ollama", ok=False, detail=str(e)[:200]))
# RabbitMQ + воркеры через Celery ping
try:
insp = celery_app.control.inspect(timeout=3)
pong = insp.ping() or {}
workers = list(pong.keys())
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=bool(workers),
detail=f"воркеров онлайн: {len(workers)}"))
except Exception as e:
checks.append(ServiceHealth(name="RabbitMQ/Celery", ok=False, detail=str(e)[:200]))
return checks
@router.get("/stats", response_model=AdminStats)
async def stats(db: AsyncSession = Depends(get_db)) -> AdminStats:
"""Сводные счётчики для дашборда."""
users_total = (await db.execute(select(func.count()).select_from(User))).scalar_one()
rows = (await db.execute(select(Task.status, func.count()).group_by(Task.status))).all()
tasks_by_status = {s: c for s, c in rows}
docs_total = (await db.execute(select(func.count()).select_from(Document))).scalar_one()
rows = (await db.execute(select(Document.source, func.count()).group_by(Document.source))).all()
docs_by_source = {s: c for s, c in rows}
staging_pending = (
await db.execute(
select(func.count()).select_from(StagedWork).where(StagedWork.status == "pending")
)
).scalar_one()
# Хранилище MinIO
storage: dict = {}
try:
client = get_minio()
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
try:
if not client.bucket_exists(bucket):
continue
total = 0
count = 0
for obj in client.list_objects(bucket, recursive=True):
total += obj.size or 0
count += 1
storage[bucket] = {"objects": count, "bytes": total}
except Exception:
continue
except Exception as e:
storage = {"error": str(e)[:200]}
# Длины очередей через RabbitMQ management API (если доступно)
queues: dict = {}
try:
rmq_user = settings.RABBITMQ_URL.split("//")[1].split(":")[0]
rmq_pass = settings.RABBITMQ_URL.split(":")[2].split("@")[0]
rmq_host = settings.RABBITMQ_URL.split("@")[1].split(":")[0]
async with httpx.AsyncClient(timeout=5) as c:
resp = await c.get(
f"http://{rmq_host}:15672/api/queues",
auth=(rmq_user, rmq_pass),
)
if resp.status_code == 200:
for q in resp.json():
name = q.get("name", "")
if name.startswith("queue."):
queues[name] = q.get("messages", 0)
except Exception as e:
queues = {"error": str(e)[:120]}
return AdminStats(
users_total=users_total,
tasks_by_status=tasks_by_status,
documents_total=docs_total,
documents_by_source=docs_by_source,
staging_pending=staging_pending,
storage=storage,
queues=queues,
)
# ═══════════════════════════════════════════════════════════════════════════════
# КЛИЕНТЫ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/users", response_model=list[AdminUserResponse])
async def list_users(
q: str | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminUserResponse]:
stmt = select(User)
if q:
like = f"%{q.lower()}%"
stmt = stmt.where(func.lower(User.email).like(like) | func.lower(User.name).like(like))
stmt = stmt.order_by(User.created_at.desc()).limit(limit).offset(offset)
users = (await db.execute(stmt)).scalars().all()
result = []
for u in users:
cnt = (
await db.execute(select(func.count()).select_from(Task).where(Task.user_id == u.id))
).scalar_one()
item = AdminUserResponse.model_validate(u)
item.tasks_count = cnt
result.append(item)
return result
@router.patch("/users/{user_id}", response_model=AdminUserResponse)
async def update_user(
user_id: int,
data: AdminUserUpdate,
db: AsyncSession = Depends(get_db),
) -> AdminUserResponse:
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
if user is None:
raise HTTPException(status_code=404, detail="Пользователь не найден")
if data.plan is not None:
user.plan = data.plan
if data.is_verified is not None:
user.is_verified = data.is_verified
if data.is_admin is not None:
user.is_admin = data.is_admin
await db.commit()
await db.refresh(user)
# Сбросить кэш пользователя
try:
await get_redis().delete(f"user:cache:{user_id}")
except Exception:
pass
return AdminUserResponse.model_validate(user)
@router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_user(user_id: int, db: AsyncSession = Depends(get_db)) -> None:
user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
if user is None:
raise HTTPException(status_code=404, detail="Пользователь не найден")
await db.execute(delete(Task).where(Task.user_id == user_id))
await db.delete(user)
await db.commit()
# ═══════════════════════════════════════════════════════════════════════════════
# РАБОТЫ (задачи)
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/tasks", response_model=list[AdminTaskResponse])
async def list_tasks(
status_filter: str | None = Query(None, alias="status"),
type_filter: str | None = Query(None, alias="type"),
user_id: int | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminTaskResponse]:
stmt = select(Task)
if status_filter:
stmt = stmt.where(Task.status == status_filter)
if type_filter:
stmt = stmt.where(Task.type == type_filter)
if user_id:
stmt = stmt.where(Task.user_id == user_id)
stmt = stmt.order_by(Task.created_at.desc()).limit(limit).offset(offset)
tasks = (await db.execute(stmt)).scalars().all()
return [AdminTaskResponse.model_validate(t) for t in tasks]
@router.get("/tasks/{public_id}")
async def get_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
return {
"public_id": task.public_id,
"user_id": task.user_id,
"type": task.type,
"status": task.status,
"input_data": task.input_data,
"result": task.result,
"error": task.error,
"created_at": task.created_at,
}
@router.post("/tasks/{public_id}/requeue")
async def requeue_task(public_id: str, db: AsyncSession = Depends(get_db)) -> dict:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
if task.type != "plagiarism":
raise HTTPException(status_code=400, detail="Перезапуск поддержан только для проверок плагиата")
minio_key = (task.input_data or {}).get("minio_key")
filename = (task.input_data or {}).get("filename")
if not minio_key:
raise HTTPException(status_code=400, detail="Нет minio_key для перезапуска")
task.status = "queued"
task.error = None
task.result = None
await db.commit()
celery_app.send_task(
"index.extract_and_check",
args=[task.id, minio_key, filename],
queue="queue.index",
)
return {"status": "requeued", "public_id": public_id}
@router.delete("/tasks/{public_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_task(public_id: str, db: AsyncSession = Depends(get_db)) -> None:
task = (
await db.execute(select(Task).where(Task.public_id == public_id))
).scalar_one_or_none()
if task is None:
raise HTTPException(status_code=404, detail="Задача не найдена")
await db.delete(task)
await db.commit()
# ═══════════════════════════════════════════════════════════════════════════════
# БАЗА ДОКУМЕНТОВ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/documents", response_model=list[AdminDocumentResponse])
async def list_documents(
q: str | None = None,
source: str | None = None,
year: int | None = None,
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[AdminDocumentResponse]:
stmt = select(Document)
if q:
stmt = stmt.where(func.lower(Document.title).like(f"%{q.lower()}%"))
if source:
stmt = stmt.where(Document.source == source)
if year:
stmt = stmt.where(Document.year == year)
stmt = stmt.order_by(Document.indexed_at.desc()).limit(limit).offset(offset)
docs = (await db.execute(stmt)).scalars().all()
return [AdminDocumentResponse.model_validate(d) for d in docs]
@router.delete("/documents/{doc_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_document(doc_id: int, db: AsyncSession = Depends(get_db)) -> None:
doc = (await db.execute(select(Document).where(Document.id == doc_id))).scalar_one_or_none()
if doc is None:
raise HTTPException(status_code=404, detail="Документ не найден")
await db.execute(delete(Fingerprint).where(Fingerprint.doc_id == doc_id))
await db.delete(doc)
await db.commit()
# Удалить из Elasticsearch (best-effort)
try:
async with httpx.AsyncClient(timeout=5) as c:
await c.delete(f"{settings.ELASTICSEARCH_URL}/documents/_doc/{doc_id}")
except Exception:
pass
# ═══════════════════════════════════════════════════════════════════════════════
# ХРАНИЛИЩЕ
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/storage")
async def storage_info() -> dict:
client = get_minio()
result: dict = {"buckets": []}
for bucket in (settings.MINIO_BUCKET_DOCS, settings.MINIO_BUCKET_BACKUPS, "staging"):
try:
if not client.bucket_exists(bucket):
continue
total = 0
objs = []
for obj in client.list_objects(bucket, recursive=True):
total += obj.size or 0
if len(objs) < 100:
objs.append({"name": obj.object_name, "size": obj.size})
result["buckets"].append({
"name": bucket,
"objects_total": len(objs),
"bytes": total,
"sample": objs,
})
except Exception as e:
result["buckets"].append({"name": bucket, "error": str(e)[:120]})
return result
# ═══════════════════════════════════════════════════════════════════════════════
# ИСТОЧНИКИ ПАРСИНГА
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/sources", response_model=list[ParseSourceResponse])
async def list_sources(db: AsyncSession = Depends(get_db)) -> list[ParseSourceResponse]:
rows = (await db.execute(select(ParseSource).order_by(ParseSource.created_at.desc()))).scalars().all()
return [ParseSourceResponse.model_validate(s) for s in rows]
@router.post("/sources", response_model=ParseSourceResponse, status_code=201)
async def create_source(data: ParseSourceCreate, db: AsyncSession = Depends(get_db)) -> ParseSourceResponse:
if data.source_type not in ("openalex", "cyberleninka", "arxiv"):
raise HTTPException(status_code=400, detail="Недопустимый тип источника")
src = ParseSource(**data.model_dump())
db.add(src)
await db.commit()
await db.refresh(src)
return ParseSourceResponse.model_validate(src)
@router.patch("/sources/{source_id}", response_model=ParseSourceResponse)
async def update_source(
source_id: int, data: ParseSourceUpdate, db: AsyncSession = Depends(get_db)
) -> ParseSourceResponse:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
for k, v in data.model_dump(exclude_unset=True).items():
setattr(src, k, v)
await db.commit()
await db.refresh(src)
return ParseSourceResponse.model_validate(src)
@router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_source(source_id: int, db: AsyncSession = Depends(get_db)) -> None:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
await db.delete(src)
await db.commit()
@router.post("/sources/{source_id}/run")
async def run_source(source_id: int, db: AsyncSession = Depends(get_db)) -> dict:
src = (await db.execute(select(ParseSource).where(ParseSource.id == source_id))).scalar_one_or_none()
if src is None:
raise HTTPException(status_code=404, detail="Источник не найден")
if src.last_status == "running":
raise HTTPException(status_code=409, detail="Источник уже парсится")
src.last_status = "running"
src.last_error = None
src.last_run_at = datetime.utcnow()
await db.commit()
celery_app.send_task("index.run_parser", args=[source_id], queue="queue.index")
return {"status": "started", "source_id": source_id}
# ═══════════════════════════════════════════════════════════════════════════════
# ОТСТОЙНИК
# ═══════════════════════════════════════════════════════════════════════════════
@router.get("/staging", response_model=list[StagedWorkResponse])
async def list_staging(
status_filter: str = Query("pending", alias="status"),
limit: int = Query(50, le=200),
offset: int = 0,
db: AsyncSession = Depends(get_db),
) -> list[StagedWorkResponse]:
stmt = select(StagedWork)
if status_filter != "all":
stmt = stmt.where(StagedWork.status == status_filter)
stmt = stmt.order_by(StagedWork.created_at.desc()).limit(limit).offset(offset)
rows = (await db.execute(stmt)).scalars().all()
return [StagedWorkResponse.model_validate(s) for s in rows]
@router.get("/staging/{staged_id}", response_model=StagedWorkDetail)
async def get_staging(staged_id: int, db: AsyncSession = Depends(get_db)) -> StagedWorkDetail:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
detail = StagedWorkDetail.model_validate(sw)
# Превью текста из MinIO
if sw.text_key:
try:
obj = get_minio().get_object("staging", sw.text_key)
text = obj.read().decode("utf-8", errors="replace")
detail.text_preview = text[:5000]
except Exception as e:
detail.text_preview = f"(не удалось прочитать текст: {e})"
return detail
@router.post("/staging/{staged_id}/approve")
async def approve_staging(
staged_id: int,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
if sw.status != "pending":
raise HTTPException(status_code=409, detail=f"Работа уже {sw.status}")
# Прочитать извлечённый текст
full_text = ""
if sw.text_key:
try:
obj = get_minio().get_object("staging", sw.text_key)
full_text = obj.read().decode("utf-8", errors="replace")
except Exception as e:
raise HTTPException(status_code=500, detail=f"Не удалось прочитать текст: {e}")
# Добавить в базу документов через существующую задачу индексатора
doc_data = {
"source": "user_submission",
"ext_id": f"staged:{sw.id}",
"title": sw.title or sw.filename or f"Работа #{sw.id}",
"authors": sw.authors or [],
"year": sw.year,
"lang": sw.lang,
"abstract": full_text[:2000],
"full_text": full_text,
}
celery_app.send_task("index.add_document", args=[doc_data], queue="queue.index")
sw.status = "approved"
sw.reviewed_by = admin.id
sw.reviewed_at = datetime.utcnow()
await db.commit()
return {"status": "approved", "id": staged_id}
@router.post("/staging/{staged_id}/reject")
async def reject_staging(
staged_id: int,
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> dict:
sw = (await db.execute(select(StagedWork).where(StagedWork.id == staged_id))).scalar_one_or_none()
if sw is None:
raise HTTPException(status_code=404, detail="Работа не найдена")
sw.status = "rejected"
sw.reviewed_by = admin.id
sw.reviewed_at = datetime.utcnow()
await db.commit()
return {"status": "rejected", "id": staged_id}

View File

@@ -0,0 +1,73 @@
"""Авторизация админ-панели: проверка роли is_admin и секретного кода сессии."""
import secrets
from datetime import datetime, timedelta, timezone
from fastapi import Depends, HTTPException, Path, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.database import get_db
from app.models.admin import AdminSession
from app.models.user import User
# Срок жизни секретного кода доступа к админке
ADMIN_SESSION_TTL_HOURS = 8
async def get_admin_user(current_user: User = Depends(get_current_user)) -> User:
"""Dependency: пользователь должен быть аутентифицирован И иметь is_admin."""
if not getattr(current_user, "is_admin", False):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Доступ только для администраторов",
)
return current_user
async def create_admin_session(user: User, db: AsyncSession) -> str:
"""Создать новый секретный код доступа к админке, вернуть код."""
code = secrets.token_urlsafe(32)
# Колонки TIMESTAMP WITHOUT TIME ZONE — храним naive UTC
session = AdminSession(
user_id=user.id,
code=code,
expires_at=datetime.utcnow() + timedelta(hours=ADMIN_SESSION_TTL_HOURS),
)
db.add(session)
await db.commit()
return code
async def verify_admin_code(
code: str = Path(..., description="Секретный код сессии из ссылки /admin/<code>"),
admin: User = Depends(get_admin_user),
db: AsyncSession = Depends(get_db),
) -> User:
"""Dependency: проверяет И роль администратора, И валидность кода из пути.
Двухфакторная защита: ссылка /admin/<code> бесполезна без JWT админа,
а JWT админа без верного кода — тоже.
"""
result = await db.execute(select(AdminSession).where(AdminSession.code == code))
session = result.scalar_one_or_none()
if session is None or session.user_id != admin.id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Неверный код доступа к админке",
)
expires = session.expires_at
if expires.tzinfo is not None:
expires = expires.replace(tzinfo=None)
if expires < datetime.utcnow():
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Код доступа истёк, сгенерируйте новый",
)
session.last_used_at = datetime.utcnow()
await db.commit()
return admin

View File

@@ -92,6 +92,7 @@ async def _load_user(user_id: int, db: AsyncSession):
"name": user.name,
"plan": user.plan,
"is_verified": user.is_verified,
"is_admin": user.is_admin,
}
await r.setex(cache_key, _USER_CACHE_TTL, json.dumps(safe))

View File

@@ -8,7 +8,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api import auth, documents, reports, search, tasks
from app.api import admin, auth, documents, reports, search, tasks
from app.config import settings
from app.core.minio_client import ensure_bucket
from app.core.redis_client import close_pool
@@ -78,6 +78,7 @@ app.include_router(tasks.router, prefix="/api")
app.include_router(search.router, prefix="/api")
app.include_router(documents.router, prefix="/api")
app.include_router(reports.router, prefix="/api")
app.include_router(admin.router, prefix="/api")
# ─── WebSocket ────────────────────────────────────────────────────────────────

View File

@@ -0,0 +1,19 @@
"""Экспорт всех моделей для Alembic автоопределения."""
from app.models.admin import AdminSession, ParseSource, StagedWork
from app.models.document import Document, Fingerprint, UsageLog
from app.models.task import Task, TaskStatus, TaskType
from app.models.user import User
__all__ = [
"User",
"Task",
"TaskType",
"TaskStatus",
"Document",
"Fingerprint",
"UsageLog",
"ParseSource",
"StagedWork",
"AdminSession",
]

View File

@@ -0,0 +1,92 @@
"""Модели для админ-панели: источники парсинга, отстойник работ, сессии админа."""
from datetime import datetime
from sqlalchemy import JSON, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.database import Base
class ParseSource(Base):
"""Источник парсинга для наполнения базы документов.
Админ задаёт тип источника и параметры запроса; задача index.run_parser
запускает парсинг в фоне и наполняет базу.
"""
__tablename__ = "parse_sources"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
# Тип парсера: openalex / cyberleninka / arxiv
source_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
# Человекочитаемое имя
name: Mapped[str] = mapped_column(String(255), nullable=False)
query: Mapped[str | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
year_from: Mapped[int | None] = mapped_column(nullable=True)
year_to: Mapped[int | None] = mapped_column(nullable=True)
limit: Mapped[int] = mapped_column(default=1000)
enabled: Mapped[bool] = mapped_column(default=True)
# Статус последнего запуска: idle / running / done / error
last_status: Mapped[str] = mapped_column(String(20), default="idle")
last_error: Mapped[str | None] = mapped_column(nullable=True)
last_run_at: Mapped[datetime | None] = mapped_column(nullable=True)
docs_added: Mapped[int] = mapped_column(default=0)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
def __repr__(self) -> str:
return f"<ParseSource id={self.id} type={self.source_type!r} name={self.name!r}>"
class StagedWork(Base):
"""Отстойник: проверенная пользовательская работа, ожидающая решения админа.
После одобрения работа добавляется в базу документов (index.add_document),
после отклонения — помечается rejected.
"""
__tablename__ = "staged_works"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True, index=True)
# Внутренний UUID задачи проверки (Task.id)
task_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
filename: Mapped[str | None] = mapped_column(nullable=True)
# Ключ исходного файла и извлечённого текста в MinIO
minio_key: Mapped[str | None] = mapped_column(nullable=True)
text_key: Mapped[str | None] = mapped_column(nullable=True)
title: Mapped[str | None] = mapped_column(nullable=True)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True)
word_count: Mapped[int | None] = mapped_column(nullable=True)
# pending / approved / rejected
status: Mapped[str] = mapped_column(String(20), default="pending", index=True)
# ID документа в базе после одобрения
document_id: Mapped[int | None] = mapped_column(nullable=True)
reviewed_by: Mapped[int | None] = mapped_column(nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
def __repr__(self) -> str:
return f"<StagedWork id={self.id} status={self.status!r} title={(self.title or '')[:30]!r}>"
class AdminSession(Base):
"""Секретный код доступа к админке (часть ссылки /admin/<code>).
Доступ проверяется по двум факторам: JWT админа (is_admin) И валидный код.
"""
__tablename__ = "admin_sessions"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), nullable=False, index=True)
code: Mapped[str] = mapped_column(String(64), unique=True, index=True, nullable=False)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
expires_at: Mapped[datetime] = mapped_column(nullable=False)
last_used_at: Mapped[datetime | None] = mapped_column(nullable=True)
def __repr__(self) -> str:
return f"<AdminSession id={self.id} user_id={self.user_id}>"

View File

@@ -0,0 +1,82 @@
"""Модели документов, fingerprints и логов использования."""
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import JSON, BigInteger, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.user import User
class Document(Base):
"""Документ в базе знаний системы (статья, книга, веб-страница)."""
__tablename__ = "documents"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
# Источник: openalex, cyberleninka, arxiv, wikipedia_ru, wikipedia_en
source: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
# Внешний ID (OpenAlex ID, arXiv ID и т.д.) — уникален в рамках источника
ext_id: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
doi: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
title: Mapped[str] = mapped_column(nullable=False)
authors: Mapped[list] = mapped_column(JSON, default=list)
year: Mapped[int | None] = mapped_column(nullable=True, index=True)
lang: Mapped[str | None] = mapped_column(String(10), nullable=True, index=True)
journal: Mapped[str | None] = mapped_column(nullable=True)
volume: Mapped[str | None] = mapped_column(String(50), nullable=True)
issue: Mapped[str | None] = mapped_column(String(50), nullable=True)
pages: Mapped[str | None] = mapped_column(String(50), nullable=True)
abstract: Mapped[str | None] = mapped_column(nullable=True)
url: Mapped[str | None] = mapped_column(nullable=True)
# Ключ объекта в MinIO (полный текст)
minio_key: Mapped[str | None] = mapped_column(nullable=True)
# ID вектора в FAISS индексе
faiss_id: Mapped[int | None] = mapped_column(nullable=True, index=True)
indexed_at: Mapped[datetime] = mapped_column(server_default=func.now())
# Отпечатки (Winnowing хэши)
fingerprints: Mapped[list["Fingerprint"]] = relationship(
back_populates="document",
lazy="select",
)
def __repr__(self) -> str:
return f"<Document id={self.id} source={self.source!r} title={self.title[:40]!r}>"
class Fingerprint(Base):
"""Winnowing fingerprint (хэш n-граммы) для быстрого поиска плагиата."""
__tablename__ = "fingerprints"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
doc_id: Mapped[int] = mapped_column(ForeignKey("documents.id"), index=True)
hash_value: Mapped[int] = mapped_column(BigInteger, index=True)
position: Mapped[int] = mapped_column() # Позиция в документе
document: Mapped["Document"] = relationship(back_populates="fingerprints")
def __repr__(self) -> str:
return f"<Fingerprint doc_id={self.doc_id} hash={self.hash_value}>"
class UsageLog(Base):
"""Лог использования для подсчёта лимитов по тарифу."""
__tablename__ = "usage_logs"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
# Действие: search, plagiarism, summarize, gost
action: Mapped[str] = mapped_column(String(50), nullable=False)
created_at: Mapped[datetime] = mapped_column(server_default=func.now(), index=True)
user: Mapped["User"] = relationship(back_populates="usage")
def __repr__(self) -> str:
return f"<UsageLog user_id={self.user_id} action={self.action!r}>"

View File

@@ -0,0 +1,75 @@
"""Модель задачи (асинхронная единица работы)."""
import secrets
from datetime import datetime
from enum import Enum
from typing import TYPE_CHECKING
from uuid import uuid4
from sqlalchemy import JSON, ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.user import User
class TaskType(str, Enum):
"""Тип задачи."""
SEARCH = "search"
PLAGIARISM = "plagiarism"
SUMMARIZE = "summarize"
GOST = "gost"
class TaskStatus(str, Enum):
"""Статус задачи."""
QUEUED = "queued"
PROCESSING = "processing"
DONE = "done"
FAILED = "failed"
class Task(Base):
"""Задача пользователя, обрабатываемая воркерами Celery."""
__tablename__ = "tasks"
# Внутренний UUID — не светится в API/URL
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
# Публичный ID для URL: /tasks/{public_id}
# token_urlsafe(16) → 22 символа base64url, не угадываем, не связан с internal UUID
public_id: Mapped[str] = mapped_column(
String(32),
unique=True,
index=True,
default=lambda: secrets.token_urlsafe(16),
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
type: Mapped[str] = mapped_column(String(20), nullable=False)
status: Mapped[str] = mapped_column(String(20), default=TaskStatus.QUEUED)
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
# Входные данные: query, filename, doc_ids и т.д.
input_data: Mapped[dict] = mapped_column(JSON, default=dict)
# Результат работы воркера
result: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(nullable=True)
queue_position: Mapped[int | None] = mapped_column(nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime | None] = mapped_column(
nullable=True,
onupdate=func.now(),
)
# Связи
user: Mapped["User"] = relationship(back_populates="tasks")
def __repr__(self) -> str:
return f"<Task id={self.id!r} type={self.type!r} status={self.status!r}>"

View File

@@ -0,0 +1,42 @@
"""Модель пользователя."""
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import func, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
if TYPE_CHECKING:
from app.models.task import Task
from app.models.document import UsageLog
class User(Base):
"""Пользователь системы."""
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
email: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
hashed_password: Mapped[str] = mapped_column(String(255), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
is_verified: Mapped[bool] = mapped_column(default=False)
# Администратор системы (доступ к админ-панели)
is_admin: Mapped[bool] = mapped_column(default=False)
# Тарифный план: free / student / premium / science
plan: Mapped[str] = mapped_column(String(20), default="free")
verification_token: Mapped[str | None] = mapped_column(String(255), nullable=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
server_default=func.now(),
onupdate=func.now(),
)
# Связи
tasks: Mapped[list["Task"]] = relationship(back_populates="user", lazy="select")
usage: Mapped[list["UsageLog"]] = relationship(back_populates="user", lazy="select")
def __repr__(self) -> str:
return f"<User id={self.id} email={self.email!r} plan={self.plan!r}>"

View File

@@ -0,0 +1,141 @@
"""Pydantic-схемы для админ-панели."""
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
# ─── Сессия доступа ───────────────────────────────────────────────────────────
class AdminSessionResponse(BaseModel):
code: str
url: str
expires_at: datetime
# ─── Клиенты ──────────────────────────────────────────────────────────────────
class AdminUserResponse(BaseModel):
id: int
email: str
name: str
plan: str
is_verified: bool
is_admin: bool
created_at: datetime
tasks_count: int = 0
model_config = {"from_attributes": True}
class AdminUserUpdate(BaseModel):
plan: str | None = None
is_verified: bool | None = None
is_admin: bool | None = None
# ─── Работы ─────────────────────────────────────────────────────────────────
class AdminTaskResponse(BaseModel):
public_id: str
user_id: int
type: str
status: str
error: str | None = None
created_at: datetime
model_config = {"from_attributes": True}
# ─── Документы базы ───────────────────────────────────────────────────────────
class AdminDocumentResponse(BaseModel):
id: int
source: str
ext_id: str
title: str
authors: list = Field(default_factory=list)
year: int | None = None
lang: str | None = None
doi: str | None = None
url: str | None = None
indexed_at: datetime
model_config = {"from_attributes": True}
# ─── Источники парсинга ───────────────────────────────────────────────────────
class ParseSourceCreate(BaseModel):
source_type: str = Field(description="openalex / cyberleninka / arxiv")
name: str = Field(min_length=1, max_length=255)
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int = Field(default=1000, ge=1, le=100000)
enabled: bool = True
class ParseSourceUpdate(BaseModel):
name: str | None = None
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int | None = Field(default=None, ge=1, le=100000)
enabled: bool | None = None
class ParseSourceResponse(BaseModel):
id: int
source_type: str
name: str
query: str | None = None
lang: str | None = None
year_from: int | None = None
year_to: int | None = None
limit: int
enabled: bool
last_status: str
last_error: str | None = None
last_run_at: datetime | None = None
docs_added: int
created_at: datetime
model_config = {"from_attributes": True}
# ─── Отстойник ────────────────────────────────────────────────────────────────
class StagedWorkResponse(BaseModel):
id: int
user_id: int | None = None
task_id: str | None = None
filename: str | None = None
title: str | None = None
authors: list = Field(default_factory=list)
year: int | None = None
lang: str | None = None
word_count: int | None = None
status: str
document_id: int | None = None
created_at: datetime
model_config = {"from_attributes": True}
class StagedWorkDetail(StagedWorkResponse):
text_preview: str | None = None
# ─── Дашборд ──────────────────────────────────────────────────────────────────
class ServiceHealth(BaseModel):
name: str
ok: bool
detail: str | None = None
class AdminStats(BaseModel):
users_total: int
tasks_by_status: dict[str, int]
documents_total: int
documents_by_source: dict[str, int]
staging_pending: int
storage: dict[str, Any]
queues: dict[str, Any]

View File

@@ -26,6 +26,7 @@ class UserInToken(BaseModel):
name: str
plan: str
is_verified: bool
is_admin: bool = False
model_config = {"from_attributes": True}

View File

@@ -10,6 +10,7 @@ pydantic==2.7.1
pydantic-settings==2.2.1
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
bcrypt==4.0.1 # passlib 1.7.4 несовместим с bcrypt>=4.1 (ошибка __about__ / 72-byte limit)
python-multipart==0.0.9
minio==7.2.7
httpx==0.27.0