355 lines
12 KiB
Python
355 lines
12 KiB
Python
from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import StreamingResponse
|
||
import uuid
|
||
import io
|
||
import asyncio
|
||
import json as _json
|
||
from sqlalchemy.orm import Session
|
||
from app.config import settings
|
||
from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User
|
||
from app.schemas import (
|
||
CheckResponse, ReportResponse, ReportSummary,
|
||
ReportsListResponse, StatsResponse, HealthResponse, MatchResult,
|
||
)
|
||
from app.core.embeddings import EmbeddingIndex
|
||
from app.api.auth import router as auth_router, get_current_user
|
||
from app.api.admin import router as admin_router
|
||
import pdfplumber
|
||
import docx as python_docx
|
||
from minio import Minio
|
||
from minio.error import S3Error
|
||
from typing import Optional
|
||
|
||
Base.metadata.create_all(bind=engine)
|
||
|
||
app = FastAPI(
|
||
title=settings.API_TITLE,
|
||
version=settings.API_VERSION,
|
||
description=settings.API_DESCRIPTION,
|
||
)
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=["*"],
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
app.include_router(auth_router)
|
||
app.include_router(admin_router)
|
||
|
||
_minio: Minio | None = None
|
||
_embedding_index: EmbeddingIndex | None = None
|
||
|
||
|
||
def get_minio() -> Minio:
|
||
global _minio
|
||
if _minio is None:
|
||
_minio = Minio(
|
||
settings.MINIO_ENDPOINT,
|
||
access_key=settings.MINIO_ACCESS_KEY,
|
||
secret_key=settings.MINIO_SECRET_KEY,
|
||
secure=settings.MINIO_SECURE,
|
||
)
|
||
try:
|
||
if not _minio.bucket_exists(settings.MINIO_BUCKET):
|
||
_minio.make_bucket(settings.MINIO_BUCKET)
|
||
except S3Error:
|
||
pass
|
||
return _minio
|
||
|
||
|
||
def get_embedding_index() -> EmbeddingIndex:
|
||
global _embedding_index
|
||
if _embedding_index is None:
|
||
_embedding_index = EmbeddingIndex()
|
||
return _embedding_index
|
||
|
||
|
||
def extract_text(content: bytes, filename: str) -> str:
|
||
if filename.endswith(".txt"):
|
||
return content.decode("utf-8", errors="replace")
|
||
|
||
if filename.endswith(".pdf"):
|
||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||
return "\n".join(page.extract_text() or "" for page in pdf.pages)
|
||
|
||
if filename.endswith(".docx"):
|
||
doc = python_docx.Document(io.BytesIO(content))
|
||
return "\n".join(p.text for p in doc.paragraphs)
|
||
|
||
return content.decode("utf-8", errors="replace")
|
||
|
||
|
||
@app.get("/api/health", response_model=HealthResponse)
|
||
async def health_check():
|
||
return {"status": "ok", "version": settings.API_VERSION}
|
||
|
||
|
||
@app.post("/api/check", response_model=CheckResponse)
|
||
async def check_document(
|
||
file: UploadFile = File(...),
|
||
db: Session = Depends(get_db),
|
||
current_user: Optional[User] = Depends(get_current_user),
|
||
):
|
||
if not file.filename:
|
||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename")
|
||
|
||
allowed = {".txt", ".pdf", ".docx"}
|
||
ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
|
||
if ext not in allowed:
|
||
raise HTTPException(
|
||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||
detail=f"Unsupported file type. Allowed: {', '.join(allowed)}",
|
||
)
|
||
|
||
task_id = uuid.uuid4()
|
||
content = await file.read()
|
||
|
||
minio_key = f"uploads/{task_id}/{file.filename}"
|
||
try:
|
||
minio = get_minio()
|
||
minio.put_object(
|
||
settings.MINIO_BUCKET,
|
||
minio_key,
|
||
io.BytesIO(content),
|
||
length=len(content),
|
||
content_type=file.content_type or "application/octet-stream",
|
||
)
|
||
except Exception:
|
||
minio_key = None
|
||
|
||
try:
|
||
text = extract_text(content, file.filename)
|
||
except Exception as e:
|
||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}")
|
||
|
||
if not text.strip():
|
||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable")
|
||
|
||
task = AnalysisTask(
|
||
id=task_id,
|
||
status="processing",
|
||
source_text=text,
|
||
user_id=current_user.id if current_user else None,
|
||
)
|
||
db.add(task)
|
||
db.commit()
|
||
|
||
from app.tasks import analyze_document
|
||
analyze_document.delay(text, str(task_id))
|
||
|
||
return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"}
|
||
|
||
|
||
@app.get("/api/report/{task_id}", response_model=ReportResponse)
|
||
async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||
if not task:
|
||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||
|
||
matches = []
|
||
if task.status == "completed":
|
||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||
for m in db_matches:
|
||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||
matches.append(
|
||
MatchResult(
|
||
method=m.method,
|
||
similarity=float(m.similarity),
|
||
source_title=doc.title if doc else None,
|
||
source_db=doc.source if doc else None,
|
||
url=doc.url if doc else None,
|
||
fragment_text=m.fragment_text,
|
||
fragment_start=m.fragment_start,
|
||
fragment_end=m.fragment_end,
|
||
)
|
||
)
|
||
|
||
return ReportResponse(
|
||
task_id=str(task.id),
|
||
status=task.status,
|
||
overall_similarity=task.overall_similarity,
|
||
matches=matches or None,
|
||
source_text=task.source_text,
|
||
created_at=task.created_at,
|
||
completed_at=task.completed_at,
|
||
)
|
||
|
||
|
||
@app.get("/api/report/{task_id}/pdf")
|
||
async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||
"""Generate and return a PDF report."""
|
||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||
if not task:
|
||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||
if task.status != "completed":
|
||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet")
|
||
|
||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||
matches_data = []
|
||
for m in db_matches:
|
||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||
matches_data.append({
|
||
"method": m.method,
|
||
"similarity": m.similarity,
|
||
"source_title": doc.title if doc else "—",
|
||
"source_db": doc.source if doc else "—",
|
||
"url": doc.url if doc else "",
|
||
"fragment_text": m.fragment_text or "",
|
||
})
|
||
|
||
pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data)
|
||
|
||
return StreamingResponse(
|
||
io.BytesIO(pdf_bytes),
|
||
media_type="application/pdf",
|
||
headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'},
|
||
)
|
||
|
||
|
||
def _build_pdf(task_id: str, overall: float, matches: list) -> bytes:
|
||
from reportlab.lib.pagesizes import A4
|
||
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
||
from reportlab.lib.units import cm
|
||
from reportlab.lib import colors
|
||
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable
|
||
from reportlab.pdfbase import pdfmetrics
|
||
from reportlab.pdfbase.ttfonts import TTFont
|
||
import os
|
||
|
||
buf = io.BytesIO()
|
||
doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
|
||
styles = getSampleStyleSheet()
|
||
|
||
METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"}
|
||
METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")}
|
||
|
||
title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6)
|
||
h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4)
|
||
body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14)
|
||
small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey)
|
||
|
||
story = [
|
||
Paragraph("Отчёт о проверке на плагиат", title_style),
|
||
Paragraph(f"ID задачи: {task_id}", small_style),
|
||
Spacer(1, 0.3*cm),
|
||
HRFlowable(width="100%", thickness=1, color=colors.lightgrey),
|
||
Spacer(1, 0.3*cm),
|
||
Paragraph(f"Общий процент заимствований: <b>{overall:.1f}%</b>", body_style),
|
||
Spacer(1, 0.5*cm),
|
||
]
|
||
|
||
if not matches:
|
||
story.append(Paragraph("Совпадений не обнаружено.", body_style))
|
||
else:
|
||
story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style))
|
||
for i, m in enumerate(matches, 1):
|
||
bg = METHOD_COLOR.get(m["method"], colors.whitesmoke)
|
||
data = [
|
||
[Paragraph(f"<b>#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%</b>", body_style), ""],
|
||
["Источник:", Paragraph(m["source_title"], body_style)],
|
||
["База:", m["source_db"]],
|
||
]
|
||
if m["url"]:
|
||
data.append(["URL:", Paragraph(f'<link href="{m["url"]}">{m["url"]}</link>', small_style)])
|
||
if m["fragment_text"]:
|
||
data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)])
|
||
|
||
tbl = Table(data, colWidths=[3.5*cm, None])
|
||
tbl.setStyle(TableStyle([
|
||
("BACKGROUND", (0, 0), (-1, 0), bg),
|
||
("SPAN", (0, 0), (-1, 0)),
|
||
("FONTSIZE", (0, 0), (-1, -1), 9),
|
||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||
("TOPPADDING", (0, 0), (-1, -1), 3),
|
||
("BOTTOMPADDING", (0, 0), (-1, -1), 3),
|
||
("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey),
|
||
]))
|
||
story.append(tbl)
|
||
story.append(Spacer(1, 0.3*cm))
|
||
|
||
doc.build(story)
|
||
return buf.getvalue()
|
||
|
||
|
||
@app.get("/api/reports", response_model=ReportsListResponse)
|
||
async def list_reports(
|
||
user_id: str | None = Query(default=None),
|
||
db: Session = Depends(get_db),
|
||
current_user: Optional[User] = Depends(get_current_user),
|
||
):
|
||
query = db.query(AnalysisTask)
|
||
# Авторизованный пользователь видит только свои задачи
|
||
if current_user:
|
||
query = query.filter(AnalysisTask.user_id == current_user.id)
|
||
elif user_id:
|
||
query = query.filter(AnalysisTask.user_id == user_id)
|
||
|
||
tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all()
|
||
|
||
return ReportsListResponse(
|
||
tasks=[
|
||
ReportSummary(
|
||
task_id=str(t.id),
|
||
status=t.status,
|
||
overall_similarity=t.overall_similarity,
|
||
created_at=t.created_at,
|
||
)
|
||
for t in tasks
|
||
]
|
||
)
|
||
|
||
|
||
@app.get("/api/stats", response_model=StatsResponse)
|
||
async def get_stats(db: Session = Depends(get_db)):
|
||
total_documents = db.query(Document).count()
|
||
index = get_embedding_index()
|
||
|
||
return StatsResponse(
|
||
total_documents_indexed=total_documents,
|
||
faiss_index_size=index.get_index_size(),
|
||
elasticsearch_available=True,
|
||
api_version=settings.API_VERSION,
|
||
)
|
||
|
||
|
||
@app.websocket("/ws/progress/{task_id}")
|
||
async def ws_progress(websocket: WebSocket, task_id: str):
|
||
await websocket.accept()
|
||
import redis.asyncio as aioredis
|
||
|
||
r = aioredis.from_url(settings.REDIS_URL)
|
||
pubsub = r.pubsub()
|
||
await pubsub.subscribe(f"progress:{task_id}")
|
||
|
||
try:
|
||
async for message in pubsub.listen():
|
||
if message["type"] != "message":
|
||
continue
|
||
try:
|
||
data = _json.loads(message["data"])
|
||
except Exception:
|
||
continue
|
||
await websocket.send_json(data)
|
||
if data.get("stage") in ("done", "failed"):
|
||
break
|
||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||
pass
|
||
finally:
|
||
try:
|
||
await pubsub.unsubscribe(f"progress:{task_id}")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
await r.aclose()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import uvicorn
|
||
uvicorn.run(app, host="0.0.0.0", port=8000)
|