This commit is contained in:
34
.env.example
Normal file
34
.env.example
Normal file
@@ -0,0 +1,34 @@
|
||||
# PostgreSQL
|
||||
DATABASE_URL=postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_HOST=localhost
|
||||
ELASTICSEARCH_PORT=9200
|
||||
|
||||
# Redis / Celery
|
||||
REDIS_URL=redis://localhost:6379/0
|
||||
CELERY_BROKER_URL=redis://localhost:6379/0
|
||||
CELERY_RESULT_BACKEND=redis://localhost:6379/0
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT=localhost:9000
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=minioadmin
|
||||
MINIO_BUCKET=antiplagiator
|
||||
MINIO_SECURE=false
|
||||
|
||||
# JWT — change in production!
|
||||
SECRET_KEY=change-me-in-production
|
||||
ALGORITHM=HS256
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES=30
|
||||
|
||||
# Analysis thresholds
|
||||
FINGERPRINT_THRESHOLD=0.5
|
||||
SEMANTIC_THRESHOLD=0.6
|
||||
TOP_K_RESULTS=10
|
||||
|
||||
# Data directories
|
||||
DATA_DIR=./data
|
||||
RAW_DATA_DIR=./data/raw
|
||||
PROCESSED_DATA_DIR=./data/processed
|
||||
INDEX_DIR=./data/index
|
||||
52
.github/workflows/deploy.yml
vendored
Normal file
52
.github/workflows/deploy.yml
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
name: Deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build & push backend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./backend
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-backend:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Build & push frontend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./frontend
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-frontend:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Deploy to server via SSH
|
||||
uses: appleboy/ssh-action@v1
|
||||
with:
|
||||
host: ${{ secrets.SERVER_HOST }}
|
||||
username: ${{ secrets.SERVER_USER }}
|
||||
key: ${{ secrets.SERVER_SSH_KEY }}
|
||||
script: |
|
||||
cd /opt/antiplagiator
|
||||
docker compose pull
|
||||
docker compose up -d --remove-orphans
|
||||
docker image prune -f
|
||||
23
backend/Dockerfile
Normal file
23
backend/Dockerfile
Normal file
@@ -0,0 +1,23 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy application
|
||||
COPY . .
|
||||
|
||||
# Create data directories
|
||||
RUN mkdir -p /app/data/{raw,processed,index}
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
0
backend/app/__init__.py
Normal file
0
backend/app/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
115
backend/app/api/admin.py
Normal file
115
backend/app/api/admin.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from fastapi import APIRouter, HTTPException, status, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
import asyncio
|
||||
import json as _json
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
SOURCES = {
|
||||
"wikipedia_ru": "Wikipedia RU (~8 GB)",
|
||||
"wikipedia_en": "Wikipedia EN (~22 GB)",
|
||||
}
|
||||
|
||||
|
||||
class JobResponse(BaseModel):
|
||||
status: str
|
||||
source: str | None = None
|
||||
label: str | None = None
|
||||
progress: int = 0
|
||||
message: str = ""
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
docs_done: int = 0
|
||||
|
||||
|
||||
@router.get("/index/sources")
|
||||
def list_sources():
|
||||
"""Список доступных источников для индексации."""
|
||||
return [{"key": k, "label": v} for k, v in SOURCES.items()]
|
||||
|
||||
|
||||
@router.post("/index/download/{source_key}", response_model=JobResponse)
|
||||
def start_download(source_key: str):
|
||||
"""Скачать дамп и проиндексировать с нуля."""
|
||||
if source_key not in SOURCES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
|
||||
|
||||
from app.tasks.indexing import get_job_status, download_and_index
|
||||
current = get_job_status()
|
||||
if current.get("status") == "running":
|
||||
raise HTTPException(status_code=409, detail="Задание уже выполняется")
|
||||
|
||||
download_and_index.delay(source_key)
|
||||
return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск скачивания…")
|
||||
|
||||
|
||||
@router.post("/index/update/{source_key}", response_model=JobResponse)
|
||||
def start_update(source_key: str):
|
||||
"""Обновить индекс — только новые документы (дамп не скачивается заново если уже есть)."""
|
||||
if source_key not in SOURCES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
|
||||
|
||||
from app.tasks.indexing import get_job_status, update_index
|
||||
current = get_job_status()
|
||||
if current.get("status") == "running":
|
||||
raise HTTPException(status_code=409, detail="Задание уже выполняется")
|
||||
|
||||
update_index.delay(source_key)
|
||||
return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск обновления…")
|
||||
|
||||
|
||||
@router.get("/index/status", response_model=JobResponse)
|
||||
def job_status():
|
||||
"""Текущий статус задания индексации."""
|
||||
from app.tasks.indexing import get_job_status
|
||||
data = get_job_status()
|
||||
return JobResponse(
|
||||
status=data.get("status", "idle"),
|
||||
source=data.get("source"),
|
||||
label=data.get("label"),
|
||||
progress=data.get("progress", 0),
|
||||
message=data.get("message", ""),
|
||||
started_at=data.get("started_at"),
|
||||
finished_at=data.get("finished_at"),
|
||||
docs_done=data.get("docs_done", 0),
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/ws/index-progress")
|
||||
async def ws_index_progress(websocket: WebSocket):
|
||||
"""Stream indexing job progress via Redis pub/sub."""
|
||||
await websocket.accept()
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from app.config import settings
|
||||
|
||||
r = aioredis.from_url(settings.REDIS_URL)
|
||||
pubsub = r.pubsub()
|
||||
await pubsub.subscribe("index_job_progress")
|
||||
|
||||
# Send current state immediately
|
||||
from app.tasks.indexing import get_job_status
|
||||
await websocket.send_json(get_job_status())
|
||||
|
||||
try:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
try:
|
||||
data = _json.loads(message["data"])
|
||||
except Exception:
|
||||
continue
|
||||
await websocket.send_json(data)
|
||||
if data.get("status") in ("done", "error"):
|
||||
break
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
await pubsub.unsubscribe("index_job_progress")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await r.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
115
backend/app/api/auth.py
Normal file
115
backend/app/api/auth.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
from pydantic import BaseModel, EmailStr
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models.db import get_db, User
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
|
||||
|
||||
|
||||
# ── Schemas ────────────────────────────────────────────────────────────────
|
||||
|
||||
class RegisterRequest(BaseModel):
|
||||
email: EmailStr
|
||||
password: str
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
id: str
|
||||
email: str
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
def create_token(user_id) -> str:
|
||||
expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
return jwt.encode(
|
||||
{"sub": str(user_id), "exp": expire},
|
||||
settings.SECRET_KEY,
|
||||
algorithm=settings.ALGORITHM,
|
||||
)
|
||||
|
||||
|
||||
def get_current_user(
|
||||
token: Optional[str] = Depends(oauth2_scheme),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Optional[User]:
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
|
||||
user_id_str: str = payload.get("sub")
|
||||
if not user_id_str:
|
||||
return None
|
||||
import uuid as _uuid
|
||||
user_id = _uuid.UUID(user_id_str)
|
||||
except (JWTError, ValueError):
|
||||
return None
|
||||
return db.query(User).filter(User.id == user_id).first()
|
||||
|
||||
|
||||
def require_user(current_user: Optional[User] = Depends(get_current_user)) -> User:
|
||||
if not current_user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
# ── Endpoints ──────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/register", response_model=UserOut, status_code=status.HTTP_201_CREATED)
|
||||
def register(body: RegisterRequest, db: Session = Depends(get_db)):
|
||||
if db.query(User).filter(User.email == body.email).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered")
|
||||
user = User(
|
||||
email=body.email,
|
||||
hashed_password=hash_password(body.password),
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
def login(form: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):
|
||||
user = db.query(User).filter(User.email == form.username).first()
|
||||
if not user or not verify_password(form.password, user.hashed_password):
|
||||
raise HTTPException(status_code=400, detail="Incorrect email or password")
|
||||
return {"access_token": create_token(user.id)}
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(current_user: User = Depends(require_user)):
|
||||
return current_user
|
||||
18
backend/app/celery_app.py
Normal file
18
backend/app/celery_app.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from celery import Celery
|
||||
from app.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"antiplagiator",
|
||||
broker=settings.CELERY_BROKER_URL,
|
||||
backend=settings.CELERY_RESULT_BACKEND,
|
||||
include=["app.tasks", "app.tasks.indexing"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
)
|
||||
50
backend/app/config.py
Normal file
50
backend/app/config.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# Database
|
||||
DATABASE_URL: str = "postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator"
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_HOST: str = "localhost"
|
||||
ELASTICSEARCH_PORT: int = 9200
|
||||
|
||||
# Redis
|
||||
REDIS_URL: str = "redis://localhost:6379/0"
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: str = "localhost:9000"
|
||||
MINIO_ACCESS_KEY: str = "minioadmin"
|
||||
MINIO_SECRET_KEY: str = "minioadmin"
|
||||
MINIO_BUCKET: str = "antiplagiator"
|
||||
MINIO_SECURE: bool = False
|
||||
|
||||
# JWT
|
||||
SECRET_KEY: str = "your-secret-key-change-in-prod"
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
|
||||
|
||||
# Celery
|
||||
CELERY_BROKER_URL: str = "redis://localhost:6379/0"
|
||||
CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
|
||||
|
||||
# API
|
||||
API_TITLE: str = "Antiplagiator API"
|
||||
API_VERSION: str = "0.1.0"
|
||||
API_DESCRIPTION: str = "Text plagiarism detection system"
|
||||
|
||||
# Paths
|
||||
DATA_DIR: str = "./data"
|
||||
RAW_DATA_DIR: str = "./data/raw"
|
||||
PROCESSED_DATA_DIR: str = "./data/processed"
|
||||
INDEX_DIR: str = "./data/index"
|
||||
|
||||
# Analysis
|
||||
FINGERPRINT_THRESHOLD: float = 0.5
|
||||
SEMANTIC_THRESHOLD: float = 0.6
|
||||
TOP_K_RESULTS: int = 10
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
settings = Settings()
|
||||
0
backend/app/core/__init__.py
Normal file
0
backend/app/core/__init__.py
Normal file
149
backend/app/core/analyzer.py
Normal file
149
backend/app/core/analyzer.py
Normal file
@@ -0,0 +1,149 @@
|
||||
import re
|
||||
from typing import Dict, List, Any, Tuple
|
||||
from app.core.fingerprint import WinNowing, MinHashLSHIndex
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.core.fulltext import FulltextSearch
|
||||
from app.config import settings
|
||||
|
||||
FRAGMENT_WORDS = 40
|
||||
|
||||
|
||||
def _split_sentences(text: str) -> List[str]:
|
||||
parts = re.split(r'(?<=[.!?])\s+', text.strip())
|
||||
return [p for p in parts if p]
|
||||
|
||||
|
||||
def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]:
|
||||
"""Return (fragment_text, char_start, char_end) around the given sentence index."""
|
||||
sentences = _split_sentences(text)
|
||||
start_i = max(0, sentence_idx - context_sentences)
|
||||
end_i = min(len(sentences), sentence_idx + context_sentences + 1)
|
||||
|
||||
fragment = " ".join(sentences[start_i:end_i])
|
||||
char_start = text.find(sentences[start_i]) if sentences else 0
|
||||
char_end = char_start + len(fragment)
|
||||
return fragment, char_start, char_end
|
||||
|
||||
|
||||
def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str:
|
||||
words = text.split()
|
||||
if len(words) <= max_words:
|
||||
return text
|
||||
return " ".join(words[:max_words]) + "…"
|
||||
|
||||
|
||||
class DocumentAnalyzer:
|
||||
def __init__(self):
|
||||
self.winnowing = WinNowing()
|
||||
self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD)
|
||||
self.embeddings = EmbeddingIndex()
|
||||
self.fulltext = FulltextSearch()
|
||||
|
||||
def analyze(self, text: str, task_id: str) -> Dict[str, Any]:
|
||||
matches: List[Dict[str, Any]] = []
|
||||
matches.extend(self._level_1_exact_match(text))
|
||||
matches.extend(self._level_2_fuzzy_match(text))
|
||||
matches.extend(self._level_3_semantic_match(text))
|
||||
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"status": "completed",
|
||||
"overall_similarity": self._calculate_overall_similarity(matches),
|
||||
"matches": matches[:settings.TOP_K_RESULTS],
|
||||
}
|
||||
|
||||
def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
sentences = _split_sentences(text)
|
||||
if not sentences:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for idx, sentence in enumerate(sentences[:20]):
|
||||
if len(sentence.split()) < 6:
|
||||
continue
|
||||
try:
|
||||
es_results = self.fulltext.search_phrase(sentence, top_k=2)
|
||||
for r in es_results:
|
||||
r["_sentence_idx"] = idx
|
||||
results.extend(es_results)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
matches = []
|
||||
seen_docs: set = set()
|
||||
for result in results:
|
||||
if result["doc_id"] in seen_docs:
|
||||
continue
|
||||
idx = result.get("_sentence_idx", 0)
|
||||
fragment, char_start, char_end = _extract_fragment(text, idx)
|
||||
matches.append({
|
||||
"method": "exact",
|
||||
"similarity": min(result["score"] * 20, 100),
|
||||
"source_id": str(result["doc_id"]), # передаём doc_id напрямую
|
||||
"source_title": result["title"],
|
||||
"source_db": result["source"],
|
||||
"url": result["url"],
|
||||
"fragment_text": _truncate(fragment),
|
||||
"fragment_start": char_start,
|
||||
"fragment_end": char_end,
|
||||
})
|
||||
seen_docs.add(result["doc_id"])
|
||||
|
||||
return matches[:top_k]
|
||||
|
||||
def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
fuzzy_results = self.minhash_lsh.query(text, top_k=top_k)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
sentences = _split_sentences(text)
|
||||
fragment = _truncate(sentences[0]) if sentences else ""
|
||||
matches = []
|
||||
for doc_id, similarity in fuzzy_results:
|
||||
if similarity >= settings.FINGERPRINT_THRESHOLD:
|
||||
matches.append({
|
||||
"method": "fuzzy",
|
||||
"similarity": similarity * 100,
|
||||
"source_id": doc_id,
|
||||
"fragment_text": fragment,
|
||||
"fragment_start": 0,
|
||||
"fragment_end": len(fragment),
|
||||
})
|
||||
return matches
|
||||
|
||||
def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
embedding_results = self.embeddings.search(text, top_k=top_k)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
sentences = _split_sentences(text)
|
||||
fragment = _truncate(sentences[0]) if sentences else ""
|
||||
matches = []
|
||||
for doc_id, similarity in embedding_results:
|
||||
if similarity >= settings.SEMANTIC_THRESHOLD:
|
||||
matches.append({
|
||||
"method": "semantic",
|
||||
"similarity": similarity * 100,
|
||||
"source_id": doc_id,
|
||||
"fragment_text": fragment,
|
||||
"fragment_start": 0,
|
||||
"fragment_end": len(fragment),
|
||||
})
|
||||
return matches
|
||||
|
||||
def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float:
|
||||
if not matches:
|
||||
return 0.0
|
||||
|
||||
weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3}
|
||||
total_weight = 0.0
|
||||
total_similarity = 0.0
|
||||
|
||||
for match in matches:
|
||||
w = weights.get(match["method"], 0.1)
|
||||
total_similarity += match["similarity"] * w
|
||||
total_weight += w
|
||||
|
||||
return round(total_similarity / total_weight if total_weight > 0 else 0, 1)
|
||||
76
backend/app/core/embeddings.py
Normal file
76
backend/app/core/embeddings.py
Normal file
@@ -0,0 +1,76 @@
|
||||
import numpy as np
|
||||
import faiss
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from typing import List, Tuple
|
||||
import os
|
||||
from app.config import settings
|
||||
|
||||
class EmbeddingIndex:
|
||||
def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"):
|
||||
self.model = SentenceTransformer(model_name)
|
||||
self.embedding_dim = self.model.get_sentence_embedding_dimension()
|
||||
self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32)
|
||||
self.doc_ids = []
|
||||
self.embeddings = None
|
||||
|
||||
# Try to load existing index
|
||||
self._load_index()
|
||||
|
||||
def add_document(self, doc_id: str, text: str):
|
||||
embedding = self.model.encode([text], convert_to_numpy=True)[0]
|
||||
self.index.add(np.array([embedding], dtype=np.float32))
|
||||
self.doc_ids.append(doc_id)
|
||||
|
||||
if self.embeddings is None:
|
||||
self.embeddings = np.array([embedding])
|
||||
else:
|
||||
self.embeddings = np.vstack([self.embeddings, embedding])
|
||||
|
||||
def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32):
|
||||
doc_ids = [doc[0] for doc in docs]
|
||||
texts = [doc[1] for doc in docs]
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch_texts = texts[i:i+batch_size]
|
||||
embeddings = self.model.encode(batch_texts, convert_to_numpy=True)
|
||||
|
||||
self.index.add(embeddings.astype(np.float32))
|
||||
self.doc_ids.extend(doc_ids[i:i+batch_size])
|
||||
|
||||
if self.embeddings is None:
|
||||
self.embeddings = embeddings
|
||||
else:
|
||||
self.embeddings = np.vstack([self.embeddings, embeddings])
|
||||
|
||||
def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
|
||||
query_embedding = self.model.encode([text], convert_to_numpy=True)[0]
|
||||
query_embedding = np.array([query_embedding], dtype=np.float32)
|
||||
|
||||
distances, indices = self.index.search(query_embedding, top_k)
|
||||
|
||||
results = []
|
||||
for idx, distance in zip(indices[0], distances[0]):
|
||||
if idx < len(self.doc_ids):
|
||||
similarity = 1.0 / (1.0 + distance)
|
||||
results.append((self.doc_ids[idx], similarity))
|
||||
|
||||
return results
|
||||
|
||||
def _load_index(self):
|
||||
index_path = os.path.join(settings.INDEX_DIR, "faiss_index")
|
||||
ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy")
|
||||
|
||||
if os.path.exists(index_path) and os.path.exists(ids_path):
|
||||
try:
|
||||
self.index = faiss.read_index(index_path)
|
||||
self.doc_ids = list(np.load(ids_path, allow_pickle=True))
|
||||
except Exception as e:
|
||||
print(f"Failed to load index: {e}")
|
||||
|
||||
def save_index(self):
|
||||
os.makedirs(settings.INDEX_DIR, exist_ok=True)
|
||||
faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index"))
|
||||
np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object))
|
||||
|
||||
def get_index_size(self) -> int:
|
||||
return self.index.ntotal
|
||||
82
backend/app/core/fingerprint.py
Normal file
82
backend/app/core/fingerprint.py
Normal file
@@ -0,0 +1,82 @@
|
||||
import xxhash
|
||||
from typing import Set, Tuple, List
|
||||
from datasketch import MinHash, MinHashLSH
|
||||
import numpy as np
|
||||
|
||||
class WinNowing:
|
||||
def __init__(self, k: int = 5, window_size: int = 4):
|
||||
self.k = k
|
||||
self.window_size = window_size
|
||||
|
||||
def get_kgrams(self, text: str) -> List[str]:
|
||||
words = text.split()
|
||||
kgrams = []
|
||||
for i in range(len(words) - self.k + 1):
|
||||
kgram = ' '.join(words[i:i+self.k])
|
||||
kgrams.append(kgram)
|
||||
return kgrams
|
||||
|
||||
def fingerprint(self, text: str) -> Set[int]:
|
||||
kgrams = self.get_kgrams(text)
|
||||
hashes = []
|
||||
for kgram in kgrams:
|
||||
h = int(xxhash.xxh64(kgram).hexdigest(), 16)
|
||||
hashes.append(h)
|
||||
|
||||
if not hashes:
|
||||
return set()
|
||||
|
||||
fingerprints = set()
|
||||
for i in range(len(hashes) - self.window_size + 1):
|
||||
window = hashes[i:i+self.window_size]
|
||||
min_hash = min(window)
|
||||
fingerprints.add(min_hash)
|
||||
|
||||
return fingerprints
|
||||
|
||||
def compare(self, fp1: Set[int], fp2: Set[int]) -> float:
|
||||
if not fp1 or not fp2:
|
||||
return 0.0
|
||||
intersection = len(fp1 & fp2)
|
||||
union = len(fp1 | fp2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
class MinHashLSHIndex:
|
||||
def __init__(self, num_perm: int = 128, threshold: float = 0.5):
|
||||
self.num_perm = num_perm
|
||||
self.threshold = threshold
|
||||
self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
|
||||
self.documents = {}
|
||||
|
||||
def add_document(self, doc_id: str, text: str):
|
||||
kgrams = self._get_kgrams(text)
|
||||
m = MinHash(num_perm=self.num_perm)
|
||||
for kgram in kgrams:
|
||||
m.update(kgram.encode())
|
||||
|
||||
self.lsh.insert(doc_id, m)
|
||||
self.documents[doc_id] = m
|
||||
|
||||
def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
|
||||
kgrams = self._get_kgrams(text)
|
||||
query_m = MinHash(num_perm=self.num_perm)
|
||||
for kgram in kgrams:
|
||||
query_m.update(kgram.encode())
|
||||
|
||||
candidates = self.lsh.query(query_m)
|
||||
|
||||
results = []
|
||||
for doc_id in candidates:
|
||||
similarity = query_m.jaccard(self.documents[doc_id])
|
||||
results.append((doc_id, similarity))
|
||||
|
||||
results.sort(key=lambda x: x[1], reverse=True)
|
||||
return results[:top_k]
|
||||
|
||||
def _get_kgrams(self, text: str, k: int = 5) -> List[str]:
|
||||
words = text.split()
|
||||
kgrams = []
|
||||
for i in range(max(1, len(words) - k + 1)):
|
||||
kgram = ' '.join(words[i:i+k])
|
||||
kgrams.append(kgram)
|
||||
return kgrams
|
||||
135
backend/app/core/fulltext.py
Normal file
135
backend/app/core/fulltext.py
Normal file
@@ -0,0 +1,135 @@
|
||||
from elasticsearch import Elasticsearch
|
||||
from typing import List, Tuple, Dict, Any
|
||||
from app.config import settings
|
||||
|
||||
class FulltextSearch:
|
||||
def __init__(self):
|
||||
self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"])
|
||||
self.index_name = "documents"
|
||||
self._create_index()
|
||||
|
||||
def _create_index(self):
|
||||
if not self.es.indices.exists(index=self.index_name):
|
||||
self.es.indices.create(
|
||||
index=self.index_name,
|
||||
body={
|
||||
"settings": {
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"ru_en": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "ru_stop", "ru_stem"]
|
||||
}
|
||||
},
|
||||
"filter": {
|
||||
"ru_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_russian_"
|
||||
},
|
||||
"ru_stem": {
|
||||
"type": "stemmer",
|
||||
"language": "russian"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"doc_id": {"type": "keyword"},
|
||||
"text": {"type": "text", "analyzer": "ru_en"},
|
||||
"title": {"type": "text", "analyzer": "ru_en"},
|
||||
"source": {"type": "keyword"},
|
||||
"lang": {"type": "keyword"},
|
||||
"url": {"type": "keyword"}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""):
|
||||
self.es.index(
|
||||
index=self.index_name,
|
||||
id=doc_id,
|
||||
body={
|
||||
"doc_id": doc_id,
|
||||
"title": title,
|
||||
"text": text,
|
||||
"source": source,
|
||||
"lang": lang,
|
||||
"url": url
|
||||
}
|
||||
)
|
||||
|
||||
def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]:
|
||||
must = {
|
||||
"multi_match": {
|
||||
"query": query,
|
||||
"fields": ["text^2", "title^3"],
|
||||
"type": "best_fields",
|
||||
"analyzer": "ru_en",
|
||||
}
|
||||
}
|
||||
|
||||
if lang:
|
||||
search_body = {
|
||||
"size": top_k,
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must,
|
||||
"filter": {"term": {"lang": lang}},
|
||||
}
|
||||
},
|
||||
}
|
||||
else:
|
||||
search_body = {"size": top_k, "query": must}
|
||||
|
||||
results = self.es.search(index=self.index_name, body=search_body)
|
||||
|
||||
formatted_results = []
|
||||
for hit in results['hits']['hits']:
|
||||
source = hit['_source']
|
||||
formatted_results.append({
|
||||
"doc_id": source['doc_id'],
|
||||
"title": source['title'],
|
||||
"source": source['source'],
|
||||
"url": source.get('url', ''),
|
||||
"score": hit['_score']
|
||||
})
|
||||
|
||||
return formatted_results
|
||||
|
||||
def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]:
|
||||
results = self.es.search(
|
||||
index=self.index_name,
|
||||
body={
|
||||
"size": top_k,
|
||||
"query": {
|
||||
"match_phrase": {
|
||||
"text": phrase
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
formatted_results = []
|
||||
for hit in results['hits']['hits']:
|
||||
source = hit['_source']
|
||||
formatted_results.append({
|
||||
"doc_id": source['doc_id'],
|
||||
"title": source['title'],
|
||||
"source": source['source'],
|
||||
"url": source.get('url', ''),
|
||||
"score": hit['_score']
|
||||
})
|
||||
|
||||
return formatted_results
|
||||
|
||||
def bulk_index(self, documents: List[Dict[str, Any]]):
|
||||
actions = []
|
||||
for doc in documents:
|
||||
actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}})
|
||||
actions.append(doc)
|
||||
|
||||
from elasticsearch.helpers import bulk
|
||||
bulk(self.es, actions)
|
||||
354
backend/app/main.py
Normal file
354
backend/app/main.py
Normal file
@@ -0,0 +1,354 @@
|
||||
from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import StreamingResponse
|
||||
import uuid
|
||||
import io
|
||||
import asyncio
|
||||
import json as _json
|
||||
from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User
|
||||
from app.schemas import (
|
||||
CheckResponse, ReportResponse, ReportSummary,
|
||||
ReportsListResponse, StatsResponse, HealthResponse, MatchResult,
|
||||
)
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.api.auth import router as auth_router, get_current_user
|
||||
from app.api.admin import router as admin_router
|
||||
import pdfplumber
|
||||
import docx as python_docx
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
from typing import Optional
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.API_TITLE,
|
||||
version=settings.API_VERSION,
|
||||
description=settings.API_DESCRIPTION,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(auth_router)
|
||||
app.include_router(admin_router)
|
||||
|
||||
_minio: Minio | None = None
|
||||
_embedding_index: EmbeddingIndex | None = None
|
||||
|
||||
|
||||
def get_minio() -> Minio:
|
||||
global _minio
|
||||
if _minio is None:
|
||||
_minio = Minio(
|
||||
settings.MINIO_ENDPOINT,
|
||||
access_key=settings.MINIO_ACCESS_KEY,
|
||||
secret_key=settings.MINIO_SECRET_KEY,
|
||||
secure=settings.MINIO_SECURE,
|
||||
)
|
||||
try:
|
||||
if not _minio.bucket_exists(settings.MINIO_BUCKET):
|
||||
_minio.make_bucket(settings.MINIO_BUCKET)
|
||||
except S3Error:
|
||||
pass
|
||||
return _minio
|
||||
|
||||
|
||||
def get_embedding_index() -> EmbeddingIndex:
|
||||
global _embedding_index
|
||||
if _embedding_index is None:
|
||||
_embedding_index = EmbeddingIndex()
|
||||
return _embedding_index
|
||||
|
||||
|
||||
def extract_text(content: bytes, filename: str) -> str:
|
||||
if filename.endswith(".txt"):
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
if filename.endswith(".pdf"):
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
return "\n".join(page.extract_text() or "" for page in pdf.pages)
|
||||
|
||||
if filename.endswith(".docx"):
|
||||
doc = python_docx.Document(io.BytesIO(content))
|
||||
return "\n".join(p.text for p in doc.paragraphs)
|
||||
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
@app.get("/api/health", response_model=HealthResponse)
|
||||
async def health_check():
|
||||
return {"status": "ok", "version": settings.API_VERSION}
|
||||
|
||||
|
||||
@app.post("/api/check", response_model=CheckResponse)
|
||||
async def check_document(
|
||||
file: UploadFile = File(...),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: Optional[User] = Depends(get_current_user),
|
||||
):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename")
|
||||
|
||||
allowed = {".txt", ".pdf", ".docx"}
|
||||
ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
|
||||
if ext not in allowed:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unsupported file type. Allowed: {', '.join(allowed)}",
|
||||
)
|
||||
|
||||
task_id = uuid.uuid4()
|
||||
content = await file.read()
|
||||
|
||||
minio_key = f"uploads/{task_id}/{file.filename}"
|
||||
try:
|
||||
minio = get_minio()
|
||||
minio.put_object(
|
||||
settings.MINIO_BUCKET,
|
||||
minio_key,
|
||||
io.BytesIO(content),
|
||||
length=len(content),
|
||||
content_type=file.content_type or "application/octet-stream",
|
||||
)
|
||||
except Exception:
|
||||
minio_key = None
|
||||
|
||||
try:
|
||||
text = extract_text(content, file.filename)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}")
|
||||
|
||||
if not text.strip():
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable")
|
||||
|
||||
task = AnalysisTask(
|
||||
id=task_id,
|
||||
status="processing",
|
||||
source_text=text,
|
||||
user_id=current_user.id if current_user else None,
|
||||
)
|
||||
db.add(task)
|
||||
db.commit()
|
||||
|
||||
from app.tasks import analyze_document
|
||||
analyze_document.delay(text, str(task_id))
|
||||
|
||||
return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"}
|
||||
|
||||
|
||||
@app.get("/api/report/{task_id}", response_model=ReportResponse)
|
||||
async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
|
||||
matches = []
|
||||
if task.status == "completed":
|
||||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||||
for m in db_matches:
|
||||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||||
matches.append(
|
||||
MatchResult(
|
||||
method=m.method,
|
||||
similarity=float(m.similarity),
|
||||
source_title=doc.title if doc else None,
|
||||
source_db=doc.source if doc else None,
|
||||
url=doc.url if doc else None,
|
||||
fragment_text=m.fragment_text,
|
||||
fragment_start=m.fragment_start,
|
||||
fragment_end=m.fragment_end,
|
||||
)
|
||||
)
|
||||
|
||||
return ReportResponse(
|
||||
task_id=str(task.id),
|
||||
status=task.status,
|
||||
overall_similarity=task.overall_similarity,
|
||||
matches=matches or None,
|
||||
source_text=task.source_text,
|
||||
created_at=task.created_at,
|
||||
completed_at=task.completed_at,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/report/{task_id}/pdf")
|
||||
async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||||
"""Generate and return a PDF report."""
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
if task.status != "completed":
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet")
|
||||
|
||||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||||
matches_data = []
|
||||
for m in db_matches:
|
||||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||||
matches_data.append({
|
||||
"method": m.method,
|
||||
"similarity": m.similarity,
|
||||
"source_title": doc.title if doc else "—",
|
||||
"source_db": doc.source if doc else "—",
|
||||
"url": doc.url if doc else "",
|
||||
"fragment_text": m.fragment_text or "",
|
||||
})
|
||||
|
||||
pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data)
|
||||
|
||||
return StreamingResponse(
|
||||
io.BytesIO(pdf_bytes),
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
def _build_pdf(task_id: str, overall: float, matches: list) -> bytes:
|
||||
from reportlab.lib.pagesizes import A4
|
||||
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
||||
from reportlab.lib.units import cm
|
||||
from reportlab.lib import colors
|
||||
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
import os
|
||||
|
||||
buf = io.BytesIO()
|
||||
doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
|
||||
styles = getSampleStyleSheet()
|
||||
|
||||
METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"}
|
||||
METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")}
|
||||
|
||||
title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6)
|
||||
h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4)
|
||||
body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14)
|
||||
small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey)
|
||||
|
||||
story = [
|
||||
Paragraph("Отчёт о проверке на плагиат", title_style),
|
||||
Paragraph(f"ID задачи: {task_id}", small_style),
|
||||
Spacer(1, 0.3*cm),
|
||||
HRFlowable(width="100%", thickness=1, color=colors.lightgrey),
|
||||
Spacer(1, 0.3*cm),
|
||||
Paragraph(f"Общий процент заимствований: <b>{overall:.1f}%</b>", body_style),
|
||||
Spacer(1, 0.5*cm),
|
||||
]
|
||||
|
||||
if not matches:
|
||||
story.append(Paragraph("Совпадений не обнаружено.", body_style))
|
||||
else:
|
||||
story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style))
|
||||
for i, m in enumerate(matches, 1):
|
||||
bg = METHOD_COLOR.get(m["method"], colors.whitesmoke)
|
||||
data = [
|
||||
[Paragraph(f"<b>#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%</b>", body_style), ""],
|
||||
["Источник:", Paragraph(m["source_title"], body_style)],
|
||||
["База:", m["source_db"]],
|
||||
]
|
||||
if m["url"]:
|
||||
data.append(["URL:", Paragraph(f'<link href="{m["url"]}">{m["url"]}</link>', small_style)])
|
||||
if m["fragment_text"]:
|
||||
data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)])
|
||||
|
||||
tbl = Table(data, colWidths=[3.5*cm, None])
|
||||
tbl.setStyle(TableStyle([
|
||||
("BACKGROUND", (0, 0), (-1, 0), bg),
|
||||
("SPAN", (0, 0), (-1, 0)),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 9),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 3),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 3),
|
||||
("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey),
|
||||
]))
|
||||
story.append(tbl)
|
||||
story.append(Spacer(1, 0.3*cm))
|
||||
|
||||
doc.build(story)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@app.get("/api/reports", response_model=ReportsListResponse)
|
||||
async def list_reports(
|
||||
user_id: str | None = Query(default=None),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: Optional[User] = Depends(get_current_user),
|
||||
):
|
||||
query = db.query(AnalysisTask)
|
||||
# Авторизованный пользователь видит только свои задачи
|
||||
if current_user:
|
||||
query = query.filter(AnalysisTask.user_id == current_user.id)
|
||||
elif user_id:
|
||||
query = query.filter(AnalysisTask.user_id == user_id)
|
||||
|
||||
tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all()
|
||||
|
||||
return ReportsListResponse(
|
||||
tasks=[
|
||||
ReportSummary(
|
||||
task_id=str(t.id),
|
||||
status=t.status,
|
||||
overall_similarity=t.overall_similarity,
|
||||
created_at=t.created_at,
|
||||
)
|
||||
for t in tasks
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/stats", response_model=StatsResponse)
|
||||
async def get_stats(db: Session = Depends(get_db)):
|
||||
total_documents = db.query(Document).count()
|
||||
index = get_embedding_index()
|
||||
|
||||
return StatsResponse(
|
||||
total_documents_indexed=total_documents,
|
||||
faiss_index_size=index.get_index_size(),
|
||||
elasticsearch_available=True,
|
||||
api_version=settings.API_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/ws/progress/{task_id}")
|
||||
async def ws_progress(websocket: WebSocket, task_id: str):
|
||||
await websocket.accept()
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
r = aioredis.from_url(settings.REDIS_URL)
|
||||
pubsub = r.pubsub()
|
||||
await pubsub.subscribe(f"progress:{task_id}")
|
||||
|
||||
try:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
try:
|
||||
data = _json.loads(message["data"])
|
||||
except Exception:
|
||||
continue
|
||||
await websocket.send_json(data)
|
||||
if data.get("stage") in ("done", "failed"):
|
||||
break
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
await pubsub.unsubscribe(f"progress:{task_id}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await r.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
0
backend/app/models/__init__.py
Normal file
0
backend/app/models/__init__.py
Normal file
82
backend/app/models/db.py
Normal file
82
backend/app/models/db.py
Normal file
@@ -0,0 +1,82 @@
|
||||
import uuid
|
||||
from sqlalchemy import Column, String, Text, DateTime, Integer, Float, ForeignKey, Index, create_engine, Boolean
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from datetime import datetime
|
||||
from app.config import settings
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
email = Column(String(255), unique=True, nullable=False, index=True)
|
||||
hashed_password = Column(String(255), nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class Document(Base):
|
||||
__tablename__ = "documents"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source = Column(String(50), nullable=False)
|
||||
ext_id = Column(String(200), nullable=True)
|
||||
title = Column(Text, nullable=False)
|
||||
url = Column(Text, nullable=True)
|
||||
lang = Column(String(2), nullable=True)
|
||||
minio_key = Column(Text, nullable=True)
|
||||
indexed_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class Fingerprint(Base):
|
||||
__tablename__ = "fingerprints"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False)
|
||||
hash = Column(String(64), nullable=False) # xxhash hex string
|
||||
position = Column(Integer, nullable=True)
|
||||
|
||||
|
||||
Index("ix_fingerprints_hash", Fingerprint.hash)
|
||||
Index("ix_fingerprints_doc_id", Fingerprint.doc_id)
|
||||
|
||||
|
||||
class AnalysisTask(Base):
|
||||
__tablename__ = "analysis_tasks"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
|
||||
status = Column(String(20), default="pending")
|
||||
overall_similarity = Column(Float, nullable=True)
|
||||
source_text = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
|
||||
|
||||
class Match(Base):
|
||||
__tablename__ = "matches"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
task_id = Column(UUID(as_uuid=True), ForeignKey("analysis_tasks.id"), nullable=False)
|
||||
doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=True)
|
||||
similarity = Column(Float, nullable=False)
|
||||
method = Column(String(20), nullable=False)
|
||||
fragment_start = Column(Integer, nullable=True)
|
||||
fragment_end = Column(Integer, nullable=True)
|
||||
fragment_text = Column(Text, nullable=True)
|
||||
|
||||
|
||||
engine = create_engine(settings.DATABASE_URL, echo=False)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
54
backend/app/schemas/__init__.py
Normal file
54
backend/app/schemas/__init__.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: str
|
||||
version: str
|
||||
|
||||
|
||||
class CheckResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class MatchResult(BaseModel):
|
||||
method: str = Field(..., description="exact | fuzzy | semantic")
|
||||
similarity: float = Field(..., ge=0, le=100)
|
||||
source_title: Optional[str] = None
|
||||
source_db: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
source_id: Optional[str] = None
|
||||
fragment_text: Optional[str] = None
|
||||
fragment_start: Optional[int] = None
|
||||
fragment_end: Optional[int] = None
|
||||
|
||||
|
||||
class ReportResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
overall_similarity: Optional[float] = None
|
||||
matches: Optional[List[MatchResult]] = None
|
||||
source_text: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class ReportSummary(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
overall_similarity: Optional[float] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class ReportsListResponse(BaseModel):
|
||||
tasks: List[ReportSummary]
|
||||
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
total_documents_indexed: int
|
||||
faiss_index_size: int
|
||||
elasticsearch_available: bool
|
||||
api_version: str
|
||||
95
backend/app/tasks/__init__.py
Normal file
95
backend/app/tasks/__init__.py
Normal file
@@ -0,0 +1,95 @@
|
||||
import json
|
||||
import uuid
|
||||
import redis
|
||||
from app.config import settings
|
||||
from app.models.db import SessionLocal, AnalysisTask, Match
|
||||
from app.core.analyzer import DocumentAnalyzer
|
||||
from app.celery_app import celery_app
|
||||
from datetime import datetime
|
||||
|
||||
_analyzer: DocumentAnalyzer | None = None
|
||||
_redis: redis.Redis | None = None
|
||||
|
||||
|
||||
def get_analyzer() -> DocumentAnalyzer:
|
||||
global _analyzer
|
||||
if _analyzer is None:
|
||||
_analyzer = DocumentAnalyzer()
|
||||
return _analyzer
|
||||
|
||||
|
||||
def get_redis() -> redis.Redis:
|
||||
global _redis
|
||||
if _redis is None:
|
||||
_redis = redis.from_url(settings.REDIS_URL)
|
||||
return _redis
|
||||
|
||||
|
||||
def _publish(task_id: str, stage: str, progress: int):
|
||||
try:
|
||||
get_redis().publish(
|
||||
f"progress:{task_id}",
|
||||
json.dumps({"stage": stage, "progress": progress}),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="tasks.analyze_document")
|
||||
def analyze_document(self, text: str, task_id: str):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
analyzer = get_analyzer()
|
||||
|
||||
_publish(task_id, "fingerprint", 10)
|
||||
exact = analyzer._level_1_exact_match(text)
|
||||
|
||||
_publish(task_id, "fuzzy", 40)
|
||||
fuzzy = analyzer._level_2_fuzzy_match(text)
|
||||
|
||||
_publish(task_id, "semantic", 70)
|
||||
semantic = analyzer._level_3_semantic_match(text)
|
||||
|
||||
matches = exact + fuzzy + semantic
|
||||
overall_similarity = analyzer._calculate_overall_similarity(matches)
|
||||
|
||||
_publish(task_id, "saving", 90)
|
||||
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
return
|
||||
|
||||
task.status = "completed"
|
||||
task.overall_similarity = overall_similarity
|
||||
task.completed_at = datetime.utcnow()
|
||||
|
||||
for match in matches[:settings.TOP_K_RESULTS]:
|
||||
raw_id = match.get("source_id")
|
||||
try:
|
||||
doc_id = uuid.UUID(str(raw_id)) if raw_id is not None else None
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
doc_id = None
|
||||
|
||||
db.add(Match(
|
||||
task_id=task_id,
|
||||
doc_id=doc_id,
|
||||
similarity=float(match["similarity"]),
|
||||
method=match["method"],
|
||||
fragment_start=match.get("fragment_start"),
|
||||
fragment_end=match.get("fragment_end"),
|
||||
fragment_text=match.get("fragment_text"),
|
||||
))
|
||||
|
||||
db.commit()
|
||||
_publish(task_id, "done", 100)
|
||||
|
||||
except Exception as exc:
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if task:
|
||||
task.status = "failed"
|
||||
db.commit()
|
||||
_publish(task_id, "failed", 0)
|
||||
raise self.retry(exc=exc, countdown=5, max_retries=2)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
282
backend/app/tasks/indexing.py
Normal file
282
backend/app/tasks/indexing.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
Celery tasks for downloading and indexing document sources.
|
||||
|
||||
Redis key "index_job" stores current job state as JSON:
|
||||
{ status, source, progress, message, started_at, finished_at, docs_total, docs_done }
|
||||
"""
|
||||
import bz2
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
from app.celery_app import celery_app
|
||||
|
||||
from app.config import settings
|
||||
from app.models.db import SessionLocal, Document, Fingerprint
|
||||
|
||||
JOB_KEY = "index_job"
|
||||
|
||||
WIKIPEDIA_SOURCES = {
|
||||
"wikipedia_ru": {
|
||||
"url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-pages-articles.xml.bz2",
|
||||
"meta_url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-md5sums.txt",
|
||||
"lang": "ru",
|
||||
"label": "Wikipedia RU",
|
||||
},
|
||||
"wikipedia_en": {
|
||||
"url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2",
|
||||
"meta_url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-md5sums.txt",
|
||||
"lang": "en",
|
||||
"label": "Wikipedia EN",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Redis helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
def _r() -> redis.Redis:
|
||||
return redis.from_url(settings.REDIS_URL)
|
||||
|
||||
|
||||
def _set_status(data: dict):
|
||||
_r().set(JOB_KEY, json.dumps(data, default=str))
|
||||
|
||||
|
||||
def _get_status() -> dict:
|
||||
raw = _r().get(JOB_KEY)
|
||||
if not raw:
|
||||
return {"status": "idle"}
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def _pub(message: str, progress: int = 0, **extra):
|
||||
state = _get_status()
|
||||
state.update({"message": message, "progress": progress, **extra})
|
||||
_set_status(state)
|
||||
_r().publish("index_job_progress", json.dumps(state, default=str))
|
||||
|
||||
|
||||
# ── Text helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def _clean_wiki(text: str) -> str:
|
||||
text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
|
||||
text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
|
||||
text = re.sub(r'\{\{[^}]*\}\}', '', text)
|
||||
text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
|
||||
text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
text = re.sub(r' +', ' ', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _normalize(text: str) -> str:
|
||||
text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
|
||||
text = re.sub(r'[—–‒―]', '-', text)
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
# ── Download helper ────────────────────────────────────────────────────────
|
||||
|
||||
def _download_file(url: str, dest: Path, label: str) -> Path:
|
||||
"""Stream-download url → dest, publishing progress."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
r = requests.get(url, stream=True, timeout=30)
|
||||
r.raise_for_status()
|
||||
total = int(r.headers.get("Content-Length", 0))
|
||||
done = 0
|
||||
last_pct = -1
|
||||
|
||||
with open(dest, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=1 << 20): # 1 MB
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
pct = int(done / total * 40) if total else 0 # 0-40% of overall
|
||||
if pct != last_pct:
|
||||
_pub(f"Скачивание {label}: {done >> 20} MB", progress=pct)
|
||||
last_pct = pct
|
||||
|
||||
return dest
|
||||
|
||||
|
||||
# ── Indexing helper ────────────────────────────────────────────────────────
|
||||
|
||||
def _index_wikipedia_bz2(bz2_path: Path, source: str, lang: str, update_only: bool) -> int:
|
||||
"""
|
||||
Parse Wikipedia bz2 dump and index documents.
|
||||
update_only=True → skip ext_ids already in DB.
|
||||
Returns count of newly indexed docs.
|
||||
"""
|
||||
from app.core.fingerprint import WinNowing, MinHashLSHIndex
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.core.fulltext import FulltextSearch
|
||||
|
||||
db = SessionLocal()
|
||||
winnowing = WinNowing()
|
||||
minhash = MinHashLSHIndex()
|
||||
emb = EmbeddingIndex()
|
||||
fulltext = FulltextSearch()
|
||||
|
||||
# Pre-load known ext_ids for fast skip
|
||||
known: set[str] = set()
|
||||
if update_only:
|
||||
for (ext_id,) in db.query(Document.ext_id).filter(Document.source == source).yield_per(10_000):
|
||||
if ext_id:
|
||||
known.add(ext_id)
|
||||
|
||||
ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
|
||||
|
||||
embed_batch: list[tuple[str, str]] = []
|
||||
BATCH = 32
|
||||
count = 0
|
||||
parsed = 0
|
||||
|
||||
with bz2.BZ2File(str(bz2_path), 'r') as f:
|
||||
context = ET.iterparse(f, events=('end',))
|
||||
for event, elem in context:
|
||||
if not elem.tag.endswith('page'):
|
||||
continue
|
||||
|
||||
title_el = elem.find('wiki:title', ns) or elem.find('.//title')
|
||||
id_el = elem.find('wiki:id', ns) or elem.find('.//id')
|
||||
rev = elem.find('.//revision')
|
||||
text_el = rev.find('wiki:text', ns) if rev is not None else None
|
||||
if rev is None:
|
||||
text_el = elem.find('.//text')
|
||||
|
||||
if title_el is None or text_el is None:
|
||||
elem.clear(); continue
|
||||
|
||||
title = (title_el.text or '').strip()
|
||||
page_id = (id_el.text or '').strip() if id_el is not None else ''
|
||||
raw_text = text_el.text or ''
|
||||
ext_id = f"wiki_{page_id}"
|
||||
|
||||
if raw_text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
|
||||
elem.clear(); continue
|
||||
|
||||
text = _normalize(_clean_wiki(raw_text))
|
||||
if len(text.split()) < 50:
|
||||
elem.clear(); continue
|
||||
|
||||
parsed += 1
|
||||
|
||||
if update_only and ext_id in known:
|
||||
elem.clear(); continue
|
||||
|
||||
url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}"
|
||||
|
||||
# PostgreSQL
|
||||
doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
|
||||
# Fingerprints
|
||||
fp = winnowing.fingerprint(text)
|
||||
for pos, h in enumerate(list(fp)[:500]):
|
||||
db.add(Fingerprint(doc_id=doc.id, hash=format(h, '016x'), position=pos))
|
||||
|
||||
# MinHash
|
||||
try:
|
||||
minhash.add_document(str(doc.id), text)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Elasticsearch
|
||||
fulltext.index_document(str(doc.id), title, text, source, lang, url)
|
||||
|
||||
embed_batch.append((str(doc.id), text))
|
||||
if len(embed_batch) >= BATCH:
|
||||
emb.batch_add(embed_batch, batch_size=BATCH)
|
||||
embed_batch.clear()
|
||||
|
||||
count += 1
|
||||
if count % 500 == 0:
|
||||
db.commit()
|
||||
pct = 40 + min(int(count / 100000 * 55), 55) # 40-95%
|
||||
_pub(f"Проиндексировано: {count} документов", progress=pct, docs_done=count)
|
||||
|
||||
elem.clear()
|
||||
|
||||
if embed_batch:
|
||||
emb.batch_add(embed_batch, batch_size=BATCH)
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
emb.save_index()
|
||||
|
||||
return count
|
||||
|
||||
|
||||
# ── Celery tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
def _run_indexing(source_key: str, update_only: bool):
|
||||
src = WIKIPEDIA_SOURCES[source_key]
|
||||
label = src["label"]
|
||||
lang = src["lang"]
|
||||
|
||||
_set_status({
|
||||
"status": "running",
|
||||
"source": source_key,
|
||||
"label": label,
|
||||
"progress": 0,
|
||||
"message": "Запуск…",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
"docs_done": 0,
|
||||
})
|
||||
|
||||
data_dir = Path(settings.RAW_DATA_DIR) / source_key
|
||||
bz2_path = data_dir / "dump.xml.bz2"
|
||||
|
||||
# --- Download if needed ---
|
||||
if not bz2_path.exists() or not update_only:
|
||||
_pub(f"Скачивание дампа {label}…", progress=1)
|
||||
try:
|
||||
_download_file(src["url"], bz2_path, label)
|
||||
except Exception as e:
|
||||
_set_status({"status": "error", "message": f"Ошибка скачивания: {e}"})
|
||||
return
|
||||
|
||||
_pub("Индексация…", progress=41)
|
||||
|
||||
try:
|
||||
count = _index_wikipedia_bz2(bz2_path, source_key, lang, update_only)
|
||||
except Exception as e:
|
||||
_set_status({"status": "error", "message": f"Ошибка индексации: {e}"})
|
||||
return
|
||||
|
||||
action = "обновлено" if update_only else "проиндексировано"
|
||||
_set_status({
|
||||
"status": "done",
|
||||
"source": source_key,
|
||||
"label": label,
|
||||
"progress": 100,
|
||||
"message": f"Готово — {action} {count} документов",
|
||||
"finished_at": datetime.now(timezone.utc).isoformat(),
|
||||
"docs_done": count,
|
||||
})
|
||||
_r().publish("index_job_progress", _r().get(JOB_KEY))
|
||||
|
||||
|
||||
@celery_app.task(name="tasks.download_and_index", bind=True)
|
||||
def download_and_index(self, source_key: str):
|
||||
_run_indexing(source_key, update_only=False)
|
||||
|
||||
|
||||
@celery_app.task(name="tasks.update_index", bind=True)
|
||||
def update_index(self, source_key: str):
|
||||
_run_indexing(source_key, update_only=True)
|
||||
|
||||
|
||||
def get_job_status() -> dict:
|
||||
return _get_status()
|
||||
119
backend/scripts/index_documents.py
Normal file
119
backend/scripts/index_documents.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Index pre-parsed documents (JSON files from parse_pdf.py / parse_wikipedia.py)
|
||||
into PostgreSQL, Elasticsearch, FAISS, and MinHash LSH.
|
||||
|
||||
Usage:
|
||||
python scripts/index_documents.py <dir_with_json_files> [--limit N]
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from app.config import settings
|
||||
from app.models.db import SessionLocal, Document, Fingerprint, engine, Base
|
||||
from app.core.fingerprint import WinNowing, MinHashLSHIndex
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.core.fulltext import FulltextSearch
|
||||
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
def load_json_docs(directory: str, limit: int | None = None):
|
||||
docs = []
|
||||
for path in sorted(Path(directory).glob("*.json")):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
doc = json.load(f)
|
||||
if doc.get("text") and doc.get("title"):
|
||||
docs.append(doc)
|
||||
except Exception as e:
|
||||
print(f"Skip {path.name}: {e}")
|
||||
if limit and len(docs) >= limit:
|
||||
break
|
||||
return docs
|
||||
|
||||
|
||||
def index_batch(docs: list, batch_size: int = 32):
|
||||
db = SessionLocal()
|
||||
winnowing = WinNowing()
|
||||
minhash_lsh = MinHashLSHIndex()
|
||||
embeddings = EmbeddingIndex()
|
||||
fulltext = FulltextSearch()
|
||||
|
||||
print(f"Indexing {len(docs)} documents …")
|
||||
|
||||
embed_batch: list[tuple[str, str]] = []
|
||||
|
||||
for i, doc in enumerate(docs):
|
||||
source = doc.get("source", "unknown")
|
||||
ext_id = doc.get("id", "")
|
||||
title = doc["title"]
|
||||
text = doc["text"]
|
||||
url = doc.get("url", "")
|
||||
lang = doc.get("lang", "ru")
|
||||
|
||||
# 1. PostgreSQL
|
||||
db_doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
|
||||
db.add(db_doc)
|
||||
db.flush()
|
||||
|
||||
# 2. Fingerprints → PostgreSQL
|
||||
fp = winnowing.fingerprint(text)
|
||||
for pos, h in enumerate(list(fp)[:500]):
|
||||
db.add(Fingerprint(doc_id=db_doc.id, hash=format(h, '016x'), position=pos))
|
||||
|
||||
# 3. MinHash LSH (in-memory; call save separately if needed)
|
||||
try:
|
||||
minhash_lsh.add_document(str(db_doc.id), text)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 4. Elasticsearch
|
||||
fulltext.index_document(str(db_doc.id), title, text, source, lang, url)
|
||||
|
||||
# 5. Accumulate for batch embedding
|
||||
embed_batch.append((str(db_doc.id), text))
|
||||
|
||||
if len(embed_batch) >= batch_size:
|
||||
embeddings.batch_add(embed_batch, batch_size=batch_size)
|
||||
embed_batch.clear()
|
||||
|
||||
if (i + 1) % 100 == 0:
|
||||
db.commit()
|
||||
print(f" {i + 1}/{len(docs)}")
|
||||
|
||||
# Flush remaining embeddings
|
||||
if embed_batch:
|
||||
embeddings.batch_add(embed_batch, batch_size=batch_size)
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
embeddings.save_index()
|
||||
print("Done. FAISS index saved.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("directory", help="Directory with parsed JSON files")
|
||||
parser.add_argument("--limit", type=int, default=None, help="Max documents to index")
|
||||
parser.add_argument("--batch-size", type=int, default=32)
|
||||
args = parser.parse_args()
|
||||
|
||||
docs = load_json_docs(args.directory, args.limit)
|
||||
print(f"Loaded {len(docs)} documents from {args.directory}")
|
||||
|
||||
if not docs:
|
||||
print("No documents found, exiting.")
|
||||
sys.exit(0)
|
||||
|
||||
index_batch(docs, batch_size=args.batch_size)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
83
backend/scripts/normalize_text.py
Normal file
83
backend/scripts/normalize_text.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
Normalize extracted text before indexing.
|
||||
|
||||
Usage:
|
||||
python scripts/normalize_text.py <input_dir> <output_dir>
|
||||
|
||||
Reads JSON files produced by parse_wikipedia.py / parse_pdf.py,
|
||||
cleans the text field, writes cleaned files to output_dir.
|
||||
"""
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
# Collapse unicode whitespace and control characters
|
||||
text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
|
||||
|
||||
# Normalize various dash/hyphen variants to a regular hyphen
|
||||
text = re.sub(r'[—–‒―]', '-', text)
|
||||
|
||||
# Remove repeated punctuation (e.g. "!!!" → "!")
|
||||
text = re.sub(r'([!?.]){2,}', r'\1', text)
|
||||
|
||||
# Collapse multiple spaces / tabs to a single space
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
|
||||
# Collapse more than two consecutive newlines
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
|
||||
# Strip leading/trailing whitespace per line
|
||||
lines = [line.strip() for line in text.splitlines()]
|
||||
text = '\n'.join(lines)
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
def process_directory(input_dir: str, output_dir: str) -> int:
|
||||
out_path = Path(output_dir)
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
count = 0
|
||||
for src in sorted(Path(input_dir).glob('*.json')):
|
||||
try:
|
||||
with open(src, encoding='utf-8') as f:
|
||||
doc = json.load(f)
|
||||
except Exception as e:
|
||||
print(f'Skip {src.name}: {e}')
|
||||
continue
|
||||
|
||||
if 'text' not in doc:
|
||||
continue
|
||||
|
||||
doc['text'] = normalize(doc['text'])
|
||||
|
||||
if len(doc['text'].split()) < 30:
|
||||
continue
|
||||
|
||||
dest = out_path / src.name
|
||||
with open(dest, 'w', encoding='utf-8') as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=2)
|
||||
|
||||
count += 1
|
||||
if count % 1000 == 0:
|
||||
print(f'Normalized {count} documents')
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('input_dir')
|
||||
parser.add_argument('output_dir')
|
||||
args = parser.parse_args()
|
||||
|
||||
count = process_directory(args.input_dir, args.output_dir)
|
||||
print(f'Done: {count} documents written to {args.output_dir}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
90
backend/scripts/parse_pdf.py
Normal file
90
backend/scripts/parse_pdf.py
Normal file
@@ -0,0 +1,90 @@
|
||||
import os
|
||||
import json
|
||||
import pymupdf
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
|
||||
def extract_text_from_pdf(pdf_path: str) -> Dict[str, Any]:
|
||||
"""Extract text and metadata from PDF"""
|
||||
try:
|
||||
doc = pymupdf.open(pdf_path)
|
||||
|
||||
text = ""
|
||||
for page_num in range(len(doc)):
|
||||
page = doc[page_num]
|
||||
text += page.get_text()
|
||||
|
||||
metadata = doc.metadata
|
||||
doc.close()
|
||||
|
||||
# Basic cleaning
|
||||
text = clean_text(text)
|
||||
|
||||
return {
|
||||
"title": metadata.get('title', Path(pdf_path).stem),
|
||||
"text": text,
|
||||
"author": metadata.get('author', ''),
|
||||
"subject": metadata.get('subject', ''),
|
||||
"num_pages": len(doc)
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"Error processing {pdf_path}: {e}")
|
||||
return None
|
||||
|
||||
def process_pdf_directory(input_dir: str, output_dir: str) -> int:
|
||||
"""Process all PDFs in a directory"""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
count = 0
|
||||
for pdf_file in Path(input_dir).glob("*.pdf"):
|
||||
result = extract_text_from_pdf(str(pdf_file))
|
||||
|
||||
if result and result.get('text'):
|
||||
output_file = os.path.join(output_dir, f"{pdf_file.stem}.json")
|
||||
|
||||
doc = {
|
||||
"id": f"pdf_{pdf_file.stem}",
|
||||
"title": result['title'],
|
||||
"text": result['text'],
|
||||
"source": "pdf",
|
||||
"metadata": {
|
||||
"author": result.get('author'),
|
||||
"subject": result.get('subject'),
|
||||
"pages": result.get('num_pages')
|
||||
}
|
||||
}
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=2)
|
||||
|
||||
count += 1
|
||||
if count % 100 == 0:
|
||||
print(f"Processed {count} PDFs")
|
||||
|
||||
return count
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
"""Clean extracted PDF text"""
|
||||
import re
|
||||
|
||||
# Remove multiple spaces
|
||||
text = re.sub(r' +', ' ', text)
|
||||
# Remove multiple newlines
|
||||
text = re.sub(r'\n\n+', '\n', text)
|
||||
# Remove special characters
|
||||
text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text)
|
||||
|
||||
return text.strip()
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python parse_pdf.py <input_dir> <output_dir>")
|
||||
sys.exit(1)
|
||||
|
||||
input_dir = sys.argv[1]
|
||||
output_dir = sys.argv[2]
|
||||
|
||||
count = process_pdf_directory(input_dir, output_dir)
|
||||
print(f"Successfully processed {count} PDFs")
|
||||
110
backend/scripts/parse_wikipedia.py
Normal file
110
backend/scripts/parse_wikipedia.py
Normal file
@@ -0,0 +1,110 @@
|
||||
import bz2
|
||||
import xml.etree.ElementTree as ET
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Generator, Dict, Any
|
||||
|
||||
def extract_text_from_wikipedia_dump(bz2_path: str, output_dir: str, limit: int = None) -> int:
|
||||
"""Extract text from Wikipedia XML dump"""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
count = 0
|
||||
with bz2.BZ2File(bz2_path, 'r') as f:
|
||||
context = ET.iterparse(f, events=('start', 'end'))
|
||||
context = iter(context)
|
||||
|
||||
event, root = next(context)
|
||||
|
||||
for event, elem in context:
|
||||
if event == 'end' and elem.tag.endswith('page'):
|
||||
doc = parse_page(elem)
|
||||
if doc and doc.get('text'):
|
||||
filename = f"{output_dir}/wiki_{doc['id']}.json"
|
||||
with open(filename, 'w', encoding='utf-8') as out:
|
||||
json.dump(doc, out, ensure_ascii=False)
|
||||
|
||||
count += 1
|
||||
if count % 1000 == 0:
|
||||
print(f"Processed {count} documents")
|
||||
|
||||
if limit and count >= limit:
|
||||
break
|
||||
|
||||
root.clear()
|
||||
|
||||
return count
|
||||
|
||||
def parse_page(page_elem) -> Dict[str, Any]:
|
||||
"""Parse a single Wikipedia page"""
|
||||
ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
|
||||
|
||||
title_elem = page_elem.find('wiki:title', ns) or page_elem.find('.//title')
|
||||
page_id_elem = page_elem.find('wiki:id', ns) or page_elem.find('.//id')
|
||||
revision_elem = page_elem.find('.//revision')
|
||||
|
||||
if revision_elem is None:
|
||||
return None
|
||||
|
||||
text_elem = revision_elem.find('wiki:text', ns) or revision_elem.find('.//text')
|
||||
|
||||
if title_elem is None or text_elem is None:
|
||||
return None
|
||||
|
||||
title = title_elem.text or ""
|
||||
page_id = page_id_elem.text if page_id_elem is not None else ""
|
||||
text = text_elem.text or ""
|
||||
|
||||
# Skip redirects and special pages
|
||||
if text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
|
||||
return None
|
||||
|
||||
# Basic text cleaning
|
||||
text = clean_wikipedia_markup(text)
|
||||
|
||||
if len(text.split()) < 50:
|
||||
return None
|
||||
|
||||
return {
|
||||
"id": f"wiki_{page_id}",
|
||||
"title": title,
|
||||
"text": text,
|
||||
"source": "wikipedia",
|
||||
"lang": "ru" # Change for English
|
||||
}
|
||||
|
||||
def clean_wikipedia_markup(text: str) -> str:
|
||||
"""Remove Wikipedia markup"""
|
||||
import re
|
||||
|
||||
# Remove wiki links
|
||||
text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
|
||||
# Remove external links
|
||||
text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
|
||||
# Remove templates
|
||||
text = re.sub(r'\{\{[^}]*\}\}', '', text)
|
||||
# Remove categories
|
||||
text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
|
||||
# Remove refs
|
||||
text = re.sub(r'<ref[^>]*>[^<]*</ref>', '', text)
|
||||
# Remove HTML tags
|
||||
text = re.sub(r'<[^>]*>', '', text)
|
||||
# Clean whitespace
|
||||
text = re.sub(r'\n\n+', '\n', text)
|
||||
text = re.sub(r' +', ' ', text)
|
||||
|
||||
return text.strip()
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python parse_wikipedia.py <input_bz2> <output_dir> [limit]")
|
||||
sys.exit(1)
|
||||
|
||||
input_file = sys.argv[1]
|
||||
output_dir = sys.argv[2]
|
||||
limit = int(sys.argv[3]) if len(sys.argv) > 3 else None
|
||||
|
||||
count = extract_text_from_wikipedia_dump(input_file, output_dir, limit)
|
||||
print(f"Successfully processed {count} documents")
|
||||
141
docker-compose.yml
Normal file
141
docker-compose.yml
Normal file
@@ -0,0 +1,141 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_DB: antiplagiator
|
||||
POSTGRES_USER: antiplagiator
|
||||
POSTGRES_PASSWORD: antiplagiator_dev
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U antiplagiator"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
elasticsearch:
|
||||
image: elasticsearch:8.13.0
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
|
||||
ports:
|
||||
- "9200:9200"
|
||||
volumes:
|
||||
- elasticsearch_data:/usr/share/elasticsearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200 | grep -q cluster_name"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
minio:
|
||||
image: minio/minio
|
||||
command: server /data --console-address ":9001"
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator
|
||||
ELASTICSEARCH_HOST: elasticsearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
CELERY_BROKER_URL: redis://redis:6379/0
|
||||
CELERY_RESULT_BACKEND: redis://redis:6379/0
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: antiplagiator
|
||||
MINIO_SECURE: "false"
|
||||
volumes:
|
||||
- backend_data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "80:80"
|
||||
depends_on:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
|
||||
celery-worker:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
command: celery -A app.celery_app worker --loglevel=info --concurrency=2
|
||||
environment:
|
||||
DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator
|
||||
ELASTICSEARCH_HOST: elasticsearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
CELERY_BROKER_URL: redis://redis:6379/0
|
||||
CELERY_RESULT_BACKEND: redis://redis:6379/0
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
MINIO_BUCKET: antiplagiator
|
||||
MINIO_SECURE: "false"
|
||||
volumes:
|
||||
- backend_data:/app/data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
elasticsearch_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
backend_data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: antiplagiator_network
|
||||
13
frontend/Dockerfile
Normal file
13
frontend/Dockerfile
Normal file
@@ -0,0 +1,13 @@
|
||||
FROM node:20-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY package.json ./
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
FROM nginx:alpine
|
||||
COPY --from=builder /app/dist /usr/share/nginx/html
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
EXPOSE 80
|
||||
12
frontend/index.html
Normal file
12
frontend/index.html
Normal file
@@ -0,0 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Антиплагиат</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
52
frontend/nginx.conf
Normal file
52
frontend/nginx.conf
Normal file
@@ -0,0 +1,52 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Gzip
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/javascript application/json image/svg+xml;
|
||||
gzip_min_length 1024;
|
||||
gzip_vary on;
|
||||
|
||||
# Cache static assets (JS/CSS с хешами в имени — кешируем надолго)
|
||||
location ~* \.(js|css|woff2?|ttf|svg|png|jpg|ico)$ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
# React SPA — все неизвестные пути → index.html (не кешируем)
|
||||
location / {
|
||||
add_header Cache-Control "no-store";
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Proxy REST API → FastAPI
|
||||
location /api/ {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_read_timeout 120s;
|
||||
proxy_send_timeout 120s;
|
||||
}
|
||||
|
||||
# Proxy WebSocket → FastAPI (/ws/ и /api/admin/ws/)
|
||||
location ~ ^/(ws/|api/admin/ws/) {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_read_timeout 3600s;
|
||||
}
|
||||
|
||||
# MinIO console
|
||||
location /minio/ {
|
||||
proxy_pass http://minio:9001/;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
}
|
||||
29
frontend/package.json
Normal file
29
frontend/package.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "antiplagiator-frontend",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint . --ext .ts,.tsx"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"zustand": "^4.4.0",
|
||||
"axios": "^1.6.0",
|
||||
"react-query": "^3.39.3",
|
||||
"tailwindcss": "^3.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.2.0",
|
||||
"@types/react-dom": "^18.2.0",
|
||||
"@vitejs/plugin-react": "^4.2.0",
|
||||
"vite": "^5.0.0",
|
||||
"typescript": "^5.3.0",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"postcss": "^8.4.31"
|
||||
}
|
||||
}
|
||||
6
frontend/postcss.config.js
Normal file
6
frontend/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
98
frontend/src/App.tsx
Normal file
98
frontend/src/App.tsx
Normal file
@@ -0,0 +1,98 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import AuthPage from './pages/AuthPage'
|
||||
import Home from './pages/Home'
|
||||
import Report from './pages/Report'
|
||||
import HistoryPage from './pages/HistoryPage'
|
||||
import AdminPage from './pages/AdminPage'
|
||||
import { getMe } from './api/client'
|
||||
|
||||
type Screen = 'home' | 'report' | 'history' | 'admin'
|
||||
|
||||
export default function App() {
|
||||
const [authed, setAuthed] = useState<boolean | null>(null) // null = checking
|
||||
const [screen, setScreen] = useState<Screen>('home')
|
||||
const [taskId, setTaskId] = useState<string | null>(null)
|
||||
|
||||
// Check stored token on mount
|
||||
useEffect(() => {
|
||||
const token = localStorage.getItem('token')
|
||||
if (!token) { setAuthed(false); return }
|
||||
getMe()
|
||||
.then(() => setAuthed(true))
|
||||
.catch(() => { localStorage.removeItem('token'); setAuthed(false) })
|
||||
}, [])
|
||||
|
||||
const handleAuth = (token: string) => {
|
||||
if (token) localStorage.setItem('token', token)
|
||||
setAuthed(true)
|
||||
}
|
||||
|
||||
const handleLogout = () => {
|
||||
localStorage.removeItem('token')
|
||||
setAuthed(false)
|
||||
setScreen('home')
|
||||
setTaskId(null)
|
||||
}
|
||||
|
||||
const openReport = (id: string) => {
|
||||
setTaskId(id)
|
||||
setScreen('report')
|
||||
}
|
||||
|
||||
// Loading state while checking token
|
||||
if (authed === null) {
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 flex items-center justify-center text-gray-400">
|
||||
Загрузка…
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (!authed) {
|
||||
return <AuthPage onAuth={handleAuth} />
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="min-h-screen flex flex-col">
|
||||
{/* Top nav — only on home/history */}
|
||||
{screen !== 'report' && (
|
||||
<nav className="bg-white border-b px-6 py-3 flex items-center justify-between">
|
||||
<div className="flex gap-4 text-sm font-medium">
|
||||
{([['home', 'Проверка'], ['history', 'История'], ['admin', 'База']] as const).map(([s, label]) => (
|
||||
<button
|
||||
key={s}
|
||||
onClick={() => setScreen(s)}
|
||||
className={screen === s ? 'text-blue-600' : 'text-gray-500 hover:text-gray-800'}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<button onClick={handleLogout} className="text-xs text-gray-400 hover:text-gray-700">
|
||||
Выйти
|
||||
</button>
|
||||
</nav>
|
||||
)}
|
||||
|
||||
{screen === 'home' && (
|
||||
<Home onTaskCreated={openReport} />
|
||||
)}
|
||||
|
||||
{screen === 'history' && (
|
||||
<HistoryPage
|
||||
onOpen={openReport}
|
||||
onNewCheck={() => setScreen('home')}
|
||||
/>
|
||||
)}
|
||||
|
||||
{screen === 'admin' && <AdminPage />}
|
||||
|
||||
{screen === 'report' && taskId && (
|
||||
<Report
|
||||
taskId={taskId}
|
||||
onBack={() => setScreen('home')}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
160
frontend/src/api/client.ts
Normal file
160
frontend/src/api/client.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import axios from 'axios'
|
||||
|
||||
const api = axios.create({ baseURL: '/api' })
|
||||
|
||||
// Attach token from localStorage on every request
|
||||
api.interceptors.request.use((config) => {
|
||||
const token = localStorage.getItem('token')
|
||||
if (token) config.headers.Authorization = `Bearer ${token}`
|
||||
return config
|
||||
})
|
||||
|
||||
export interface CheckResponse {
|
||||
task_id: string
|
||||
status: string
|
||||
message: string
|
||||
}
|
||||
|
||||
export interface MatchResult {
|
||||
method: string
|
||||
similarity: number
|
||||
source_title?: string
|
||||
source_db?: string
|
||||
url?: string
|
||||
source_id?: string
|
||||
}
|
||||
|
||||
export interface ReportResponse {
|
||||
task_id: string
|
||||
status: string
|
||||
overall_similarity?: number
|
||||
matches?: MatchResult[]
|
||||
source_text?: string
|
||||
created_at?: string
|
||||
completed_at?: string
|
||||
}
|
||||
|
||||
export interface ProgressEvent {
|
||||
stage: 'fingerprint' | 'fuzzy' | 'semantic' | 'saving' | 'done' | 'failed'
|
||||
progress: number
|
||||
}
|
||||
|
||||
export function openProgressSocket(taskId: string, onEvent: (e: ProgressEvent) => void): () => void {
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'
|
||||
const ws = new WebSocket(`${protocol}://${window.location.host}/ws/progress/${taskId}`)
|
||||
ws.onmessage = (e) => {
|
||||
try { onEvent(JSON.parse(e.data)) } catch {}
|
||||
}
|
||||
return () => ws.close()
|
||||
}
|
||||
|
||||
export interface ReportSummary {
|
||||
task_id: string
|
||||
status: string
|
||||
overall_similarity?: number
|
||||
created_at?: string
|
||||
}
|
||||
|
||||
export const uploadDocument = async (file: File): Promise<CheckResponse> => {
|
||||
const form = new FormData()
|
||||
form.append('file', file)
|
||||
const res = await api.post<CheckResponse>('/check', form)
|
||||
return res.data
|
||||
}
|
||||
|
||||
export const getReport = async (taskId: string): Promise<ReportResponse> => {
|
||||
const res = await api.get<ReportResponse>(`/report/${taskId}`)
|
||||
return res.data
|
||||
}
|
||||
|
||||
export const listReports = async (): Promise<ReportSummary[]> => {
|
||||
const res = await api.get<{ tasks: ReportSummary[] }>('/reports')
|
||||
return res.data.tasks
|
||||
}
|
||||
|
||||
export interface UserInfo {
|
||||
id: string
|
||||
email: string
|
||||
created_at: string
|
||||
}
|
||||
|
||||
export const register = async (email: string, password: string): Promise<void> => {
|
||||
await api.post('/auth/register', { email, password })
|
||||
}
|
||||
|
||||
export const login = async (email: string, password: string): Promise<string> => {
|
||||
const form = new URLSearchParams({ username: email, password })
|
||||
const res = await api.post<{ access_token: string }>('/auth/login', form, {
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
})
|
||||
return res.data.access_token
|
||||
}
|
||||
|
||||
export const getMe = async (): Promise<UserInfo> => {
|
||||
const res = await api.get<UserInfo>('/auth/me')
|
||||
return res.data
|
||||
}
|
||||
|
||||
// ── Admin / Index ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface IndexSource {
|
||||
key: string
|
||||
label: string
|
||||
}
|
||||
|
||||
export interface JobStatus {
|
||||
status: string
|
||||
source?: string
|
||||
label?: string
|
||||
progress: number
|
||||
message: string
|
||||
started_at?: string
|
||||
finished_at?: string
|
||||
docs_done: number
|
||||
}
|
||||
|
||||
export const getIndexSources = async (): Promise<IndexSource[]> => {
|
||||
const res = await api.get<IndexSource[]>('/admin/index/sources')
|
||||
return res.data
|
||||
}
|
||||
|
||||
export const startDownload = async (sourceKey: string): Promise<JobStatus> => {
|
||||
const res = await api.post<JobStatus>(`/admin/index/download/${sourceKey}`)
|
||||
return res.data
|
||||
}
|
||||
|
||||
export const startUpdate = async (sourceKey: string): Promise<JobStatus> => {
|
||||
const res = await api.post<JobStatus>(`/admin/index/update/${sourceKey}`)
|
||||
return res.data
|
||||
}
|
||||
|
||||
export const getJobStatus = async (): Promise<JobStatus> => {
|
||||
const res = await api.get<JobStatus>('/admin/index/status')
|
||||
return res.data
|
||||
}
|
||||
|
||||
export function openIndexSocket(onEvent: (e: JobStatus) => void): () => void {
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'
|
||||
const ws = new WebSocket(`${protocol}://${window.location.host}/api/admin/ws/index-progress`)
|
||||
ws.onmessage = (e) => {
|
||||
try { onEvent(JSON.parse(e.data)) } catch {}
|
||||
}
|
||||
return () => ws.close()
|
||||
}
|
||||
|
||||
export const downloadReportPdf = async (taskId: string): Promise<void> => {
|
||||
const token = localStorage.getItem('token')
|
||||
const res = await fetch(`/api/report/${taskId}/pdf`, {
|
||||
headers: token ? { Authorization: `Bearer ${token}` } : {},
|
||||
})
|
||||
if (!res.ok) throw new Error('PDF download failed')
|
||||
const blob = await res.blob()
|
||||
const url = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = url
|
||||
a.download = `report-${taskId.slice(0, 8)}.pdf`
|
||||
document.body.appendChild(a)
|
||||
a.click()
|
||||
document.body.removeChild(a)
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
56
frontend/src/components/DropZone.tsx
Normal file
56
frontend/src/components/DropZone.tsx
Normal file
@@ -0,0 +1,56 @@
|
||||
import { useCallback, useState } from 'react'
|
||||
import { useDropzone } from 'react-dropzone'
|
||||
|
||||
interface Props {
|
||||
onFile: (file: File) => void
|
||||
disabled?: boolean
|
||||
}
|
||||
|
||||
const ACCEPTED = { 'text/plain': ['.txt'], 'application/pdf': ['.pdf'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'] }
|
||||
|
||||
export default function DropZone({ onFile, disabled }: Props) {
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const onDrop = useCallback((accepted: File[], rejected: { errors: { message: string }[] }[]) => {
|
||||
setError(null)
|
||||
if (rejected.length) {
|
||||
setError('Поддерживаются только .txt, .pdf, .docx')
|
||||
return
|
||||
}
|
||||
if (accepted[0]) onFile(accepted[0])
|
||||
}, [onFile])
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive, acceptedFiles } = useDropzone({
|
||||
onDrop,
|
||||
accept: ACCEPTED,
|
||||
maxFiles: 1,
|
||||
disabled,
|
||||
})
|
||||
|
||||
const file = acceptedFiles[0]
|
||||
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-3">
|
||||
<div
|
||||
{...getRootProps()}
|
||||
className={[
|
||||
'w-full max-w-lg border-2 border-dashed rounded-xl p-10 text-center cursor-pointer transition-colors',
|
||||
isDragActive ? 'border-blue-500 bg-blue-50' : 'border-gray-300 hover:border-blue-400',
|
||||
disabled ? 'opacity-50 cursor-not-allowed' : '',
|
||||
].join(' ')}
|
||||
>
|
||||
<input {...getInputProps()} />
|
||||
<p className="text-gray-500 text-sm">
|
||||
{isDragActive
|
||||
? 'Отпустите файл здесь…'
|
||||
: 'Перетащите файл или нажмите для выбора'}
|
||||
</p>
|
||||
<p className="text-xs text-gray-400 mt-1">.txt, .pdf, .docx</p>
|
||||
{file && (
|
||||
<p className="mt-3 text-sm font-medium text-blue-700">{file.name}</p>
|
||||
)}
|
||||
</div>
|
||||
{error && <p className="text-red-500 text-sm">{error}</p>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
109
frontend/src/components/HighlightedText.tsx
Normal file
109
frontend/src/components/HighlightedText.tsx
Normal file
@@ -0,0 +1,109 @@
|
||||
import { useEffect, useRef } from 'react'
|
||||
import { MatchResult } from '../api/client'
|
||||
import { HIGHLIGHT_COLOR } from './MatchCard'
|
||||
|
||||
interface Span {
|
||||
start: number
|
||||
end: number
|
||||
method: string
|
||||
matchIndex: number
|
||||
}
|
||||
|
||||
interface Props {
|
||||
text: string
|
||||
matches: MatchResult[]
|
||||
activeIndex: number | null
|
||||
onMatchClick: (index: number) => void
|
||||
}
|
||||
|
||||
function buildSegments(text: string, spans: Span[]) {
|
||||
// Sort by start, then merge/flatten overlapping spans keeping the one with highest priority
|
||||
const priority: Record<string, number> = { exact: 3, fuzzy: 2, semantic: 1 }
|
||||
|
||||
// Build a char-level annotation array (sparse)
|
||||
const events: Array<{ pos: number; type: 'open' | 'close'; span: Span }> = []
|
||||
for (const s of spans) {
|
||||
events.push({ pos: s.start, type: 'open', span: s })
|
||||
events.push({ pos: s.end, type: 'close', span: s })
|
||||
}
|
||||
events.sort((a, b) => a.pos - b.pos || (a.type === 'close' ? -1 : 1))
|
||||
|
||||
const segments: Array<{ text: string; span: Span | null }> = []
|
||||
let cursor = 0
|
||||
const active = new Map<Span, true>()
|
||||
|
||||
const dominant = () => {
|
||||
let best: Span | null = null
|
||||
for (const s of active.keys()) {
|
||||
if (!best || (priority[s.method] ?? 0) > (priority[best.method] ?? 0)) best = s
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
for (const ev of events) {
|
||||
if (ev.pos > cursor) {
|
||||
segments.push({ text: text.slice(cursor, ev.pos), span: dominant() })
|
||||
cursor = ev.pos
|
||||
}
|
||||
if (ev.type === 'open') active.set(ev.span, true)
|
||||
else active.delete(ev.span)
|
||||
}
|
||||
|
||||
if (cursor < text.length) {
|
||||
segments.push({ text: text.slice(cursor), span: null })
|
||||
}
|
||||
|
||||
return segments
|
||||
}
|
||||
|
||||
export default function HighlightedText({ text, matches, activeIndex, onMatchClick }: Props) {
|
||||
const spanRefs = useRef<Map<number, HTMLElement>>(new Map())
|
||||
|
||||
// Scroll to active match
|
||||
useEffect(() => {
|
||||
if (activeIndex === null) return
|
||||
const el = spanRefs.current.get(activeIndex)
|
||||
el?.scrollIntoView({ behavior: 'smooth', block: 'center' })
|
||||
}, [activeIndex])
|
||||
|
||||
const spans: Span[] = matches
|
||||
.map((m, i) => ({
|
||||
start: m.fragment_start ?? 0,
|
||||
end: m.fragment_end ?? 0,
|
||||
method: m.method,
|
||||
matchIndex: i,
|
||||
}))
|
||||
.filter((s) => s.end > s.start)
|
||||
|
||||
const segments = buildSegments(text, spans)
|
||||
|
||||
return (
|
||||
<div className="font-mono text-sm leading-7 whitespace-pre-wrap break-words">
|
||||
{segments.map((seg, i) => {
|
||||
if (!seg.span) {
|
||||
return <span key={i}>{seg.text}</span>
|
||||
}
|
||||
|
||||
const mi = seg.span.matchIndex
|
||||
const isActive = mi === activeIndex
|
||||
const color = HIGHLIGHT_COLOR[seg.span.method] ?? 'bg-gray-200'
|
||||
|
||||
return (
|
||||
<mark
|
||||
key={i}
|
||||
ref={(el) => { if (el) spanRefs.current.set(mi, el) }}
|
||||
onClick={() => onMatchClick(mi)}
|
||||
className={[
|
||||
color,
|
||||
'cursor-pointer rounded px-0.5 transition-all',
|
||||
isActive ? 'ring-2 ring-blue-500 brightness-90' : 'hover:brightness-90',
|
||||
].join(' ')}
|
||||
title={`${matches[mi]?.source_title ?? ''} — ${matches[mi]?.similarity.toFixed(1)}%`}
|
||||
>
|
||||
{seg.text}
|
||||
</mark>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
76
frontend/src/components/MatchCard.tsx
Normal file
76
frontend/src/components/MatchCard.tsx
Normal file
@@ -0,0 +1,76 @@
|
||||
import { MatchResult } from '../api/client'
|
||||
|
||||
const METHOD_LABEL: Record<string, string> = {
|
||||
exact: 'Точное',
|
||||
fuzzy: 'Нечёткое',
|
||||
semantic: 'Семантика',
|
||||
}
|
||||
|
||||
const METHOD_COLOR: Record<string, string> = {
|
||||
exact: 'bg-red-100 text-red-700 border-red-200',
|
||||
fuzzy: 'bg-orange-100 text-orange-700 border-orange-200',
|
||||
semantic: 'bg-yellow-100 text-yellow-700 border-yellow-200',
|
||||
}
|
||||
|
||||
const HIGHLIGHT_COLOR: Record<string, string> = {
|
||||
exact: 'bg-red-200',
|
||||
fuzzy: 'bg-orange-200',
|
||||
semantic: 'bg-yellow-200',
|
||||
}
|
||||
|
||||
interface Props {
|
||||
match: MatchResult
|
||||
index: number
|
||||
active: boolean
|
||||
onClick: () => void
|
||||
}
|
||||
|
||||
export { HIGHLIGHT_COLOR }
|
||||
|
||||
export default function MatchCard({ match, index, active, onClick }: Props) {
|
||||
const colorClass = METHOD_COLOR[match.method] ?? 'bg-gray-100 text-gray-700 border-gray-200'
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className={[
|
||||
'w-full text-left border rounded-xl p-3 transition-all',
|
||||
active ? 'ring-2 ring-blue-400 shadow-md' : 'hover:shadow-sm',
|
||||
colorClass,
|
||||
].join(' ')}
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 mb-1">
|
||||
<span className="text-xs font-semibold uppercase tracking-wide">
|
||||
{METHOD_LABEL[match.method] ?? match.method}
|
||||
</span>
|
||||
<span className="text-sm font-bold">{match.similarity.toFixed(1)}%</span>
|
||||
</div>
|
||||
|
||||
<p className="text-sm font-medium truncate">
|
||||
{match.source_title ?? match.source_id ?? '—'}
|
||||
</p>
|
||||
|
||||
{match.source_db && (
|
||||
<p className="text-xs opacity-70">{match.source_db}</p>
|
||||
)}
|
||||
|
||||
{match.fragment_text && (
|
||||
<p className="mt-1 text-xs italic opacity-80 line-clamp-2">
|
||||
«{match.fragment_text}»
|
||||
</p>
|
||||
)}
|
||||
|
||||
{match.url && (
|
||||
<a
|
||||
href={match.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="mt-1 text-xs underline opacity-60 block truncate"
|
||||
>
|
||||
{match.url}
|
||||
</a>
|
||||
)}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
33
frontend/src/components/SimilarityMeter.tsx
Normal file
33
frontend/src/components/SimilarityMeter.tsx
Normal file
@@ -0,0 +1,33 @@
|
||||
interface Props {
|
||||
value: number
|
||||
}
|
||||
|
||||
function color(v: number) {
|
||||
if (v < 20) return 'bg-green-500'
|
||||
if (v < 50) return 'bg-yellow-400'
|
||||
return 'bg-red-500'
|
||||
}
|
||||
|
||||
function label(v: number) {
|
||||
if (v < 20) return 'Оригинально'
|
||||
if (v < 50) return 'Умеренное сходство'
|
||||
return 'Высокое заимствование'
|
||||
}
|
||||
|
||||
export default function SimilarityMeter({ value }: Props) {
|
||||
const pct = Math.min(Math.max(value, 0), 100)
|
||||
return (
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex justify-between text-sm font-medium">
|
||||
<span>{label(pct)}</span>
|
||||
<span className="text-gray-700">{pct.toFixed(1)}%</span>
|
||||
</div>
|
||||
<div className="w-full h-3 rounded-full bg-gray-200 overflow-hidden">
|
||||
<div
|
||||
className={`h-full rounded-full transition-all ${color(pct)}`}
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
3
frontend/src/index.css
Normal file
3
frontend/src/index.css
Normal file
@@ -0,0 +1,3 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
10
frontend/src/main.tsx
Normal file
10
frontend/src/main.tsx
Normal file
@@ -0,0 +1,10 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import App from './App'
|
||||
import './index.css'
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
)
|
||||
172
frontend/src/pages/AdminPage.tsx
Normal file
172
frontend/src/pages/AdminPage.tsx
Normal file
@@ -0,0 +1,172 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import {
|
||||
getIndexSources, startDownload, startUpdate,
|
||||
getJobStatus, openIndexSocket,
|
||||
IndexSource, JobStatus,
|
||||
} from '../api/client'
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
idle: 'Ожидание',
|
||||
running: 'Выполняется',
|
||||
done: 'Готово',
|
||||
error: 'Ошибка',
|
||||
started: 'Запуск…',
|
||||
}
|
||||
|
||||
const STATUS_COLOR: Record<string, string> = {
|
||||
idle: 'text-gray-400',
|
||||
running: 'text-blue-600',
|
||||
done: 'text-green-600',
|
||||
error: 'text-red-600',
|
||||
started: 'text-blue-500',
|
||||
}
|
||||
|
||||
const SOURCE_SIZE: Record<string, string> = {
|
||||
wikipedia_ru: '~8 GB',
|
||||
wikipedia_en: '~22 GB',
|
||||
}
|
||||
|
||||
const SOURCE_DESC: Record<string, string> = {
|
||||
wikipedia_ru: 'Русскоязычные статьи Википедии',
|
||||
wikipedia_en: 'Англоязычные статьи Википедии',
|
||||
}
|
||||
|
||||
export default function AdminPage() {
|
||||
const [sources, setSources] = useState<IndexSource[]>([])
|
||||
const [job, setJob] = useState<JobStatus>({ status: 'idle', progress: 0, message: '', docs_done: 0 })
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
// Load sources + current status
|
||||
useEffect(() => {
|
||||
getIndexSources().then(setSources).catch(() => {})
|
||||
getJobStatus().then(setJob).catch(() => {})
|
||||
}, [])
|
||||
|
||||
// Subscribe to WebSocket progress
|
||||
useEffect(() => {
|
||||
const close = openIndexSocket((ev) => {
|
||||
setJob(ev)
|
||||
if (ev.status !== 'running') setBusy(false)
|
||||
})
|
||||
return close
|
||||
}, [])
|
||||
|
||||
const isRunning = job.status === 'running' || busy
|
||||
|
||||
const handleDownload = async (key: string) => {
|
||||
setError(null)
|
||||
setBusy(true)
|
||||
try {
|
||||
const j = await startDownload(key)
|
||||
setJob(j)
|
||||
} catch (e: unknown) {
|
||||
const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
|
||||
setError(detail ?? 'Ошибка запуска')
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleUpdate = async (key: string) => {
|
||||
setError(null)
|
||||
setBusy(true)
|
||||
try {
|
||||
const j = await startUpdate(key)
|
||||
setJob(j)
|
||||
} catch (e: unknown) {
|
||||
const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
|
||||
setError(detail ?? 'Ошибка запуска')
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 p-6">
|
||||
<div className="max-w-2xl mx-auto flex flex-col gap-6">
|
||||
<h1 className="text-xl font-bold text-gray-800">Управление базой документов</h1>
|
||||
|
||||
{/* Current job status */}
|
||||
<div className="bg-white rounded-2xl shadow-sm border border-gray-100 p-5 flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-semibold text-gray-700">Статус задания</span>
|
||||
<span className={`text-sm font-medium ${STATUS_COLOR[job.status] ?? 'text-gray-500'}`}>
|
||||
{STATUS_LABEL[job.status] ?? job.status}
|
||||
{job.label ? ` — ${job.label}` : ''}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{job.message && (
|
||||
<p className="text-sm text-gray-500">{job.message}</p>
|
||||
)}
|
||||
|
||||
{isRunning && (
|
||||
<div className="flex flex-col gap-1">
|
||||
<div className="flex justify-between text-xs text-gray-400">
|
||||
<span>{job.docs_done > 0 ? `Проиндексировано: ${job.docs_done.toLocaleString()} документов` : ''}</span>
|
||||
<span>{job.progress}%</span>
|
||||
</div>
|
||||
<div className="w-full h-2.5 bg-gray-200 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-blue-500 rounded-full transition-all duration-700"
|
||||
style={{ width: `${job.progress}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{job.status === 'done' && job.finished_at && (
|
||||
<p className="text-xs text-gray-400">
|
||||
Завершено: {new Date(job.finished_at).toLocaleString('ru-RU')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-xl p-3 text-red-600 text-sm">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Source cards */}
|
||||
<div className="flex flex-col gap-3">
|
||||
{sources.map((src) => (
|
||||
<div
|
||||
key={src.key}
|
||||
className="bg-white rounded-2xl shadow-sm border border-gray-100 p-5 flex items-center gap-4"
|
||||
>
|
||||
<div className="flex-1">
|
||||
<p className="font-semibold text-gray-800">{src.label}</p>
|
||||
<p className="text-sm text-gray-500">{SOURCE_DESC[src.key] ?? ''}</p>
|
||||
<p className="text-xs text-gray-400 mt-0.5">{SOURCE_SIZE[src.key] ?? ''}</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2 shrink-0">
|
||||
<button
|
||||
onClick={() => handleDownload(src.key)}
|
||||
disabled={isRunning}
|
||||
className="text-sm bg-blue-600 text-white px-4 py-2 rounded-xl hover:bg-blue-700 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
title="Скачать дамп и проиндексировать с нуля"
|
||||
>
|
||||
Скачать и индексировать
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleUpdate(src.key)}
|
||||
disabled={isRunning}
|
||||
className="text-sm border border-gray-200 px-4 py-2 rounded-xl hover:bg-gray-50 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
title="Добавить только новые документы (дамп уже скачан)"
|
||||
>
|
||||
Обновить базу
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="text-xs text-gray-400 bg-gray-100 rounded-xl p-4 flex flex-col gap-1">
|
||||
<p><strong>Скачать и индексировать</strong> — скачивает свежий дамп (~часы) и строит индекс с нуля.</p>
|
||||
<p><strong>Обновить базу</strong> — использует уже скачанный дамп, добавляет только документы, которых ещё нет в базе. Быстро после первой загрузки.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
85
frontend/src/pages/AuthPage.tsx
Normal file
85
frontend/src/pages/AuthPage.tsx
Normal file
@@ -0,0 +1,85 @@
|
||||
import { useState } from 'react'
|
||||
import { login, register } from '../api/client'
|
||||
|
||||
interface Props {
|
||||
onAuth: (token: string) => void
|
||||
}
|
||||
|
||||
export default function AuthPage({ onAuth }: Props) {
|
||||
const [mode, setMode] = useState<'login' | 'register'>('login')
|
||||
const [email, setEmail] = useState('')
|
||||
const [password, setPassword] = useState('')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handle = async () => {
|
||||
setError(null)
|
||||
setLoading(true)
|
||||
try {
|
||||
if (mode === 'register') {
|
||||
await register(email, password)
|
||||
}
|
||||
const token = await login(email, password)
|
||||
localStorage.setItem('token', token)
|
||||
onAuth(token)
|
||||
} catch (e: unknown) {
|
||||
const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
|
||||
setError(detail ?? 'Ошибка')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 flex items-center justify-center p-4">
|
||||
<div className="bg-white rounded-2xl shadow-md p-8 w-full max-w-sm flex flex-col gap-5">
|
||||
<h1 className="text-2xl font-bold text-center text-gray-800">Антиплагиат</h1>
|
||||
|
||||
<div className="flex rounded-xl overflow-hidden border text-sm font-medium">
|
||||
{(['login', 'register'] as const).map((m) => (
|
||||
<button
|
||||
key={m}
|
||||
onClick={() => { setMode(m); setError(null) }}
|
||||
className={`flex-1 py-2 transition-colors ${mode === m ? 'bg-blue-600 text-white' : 'text-gray-500 hover:bg-gray-50'}`}
|
||||
>
|
||||
{m === 'login' ? 'Вход' : 'Регистрация'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<input
|
||||
type="email"
|
||||
placeholder="Email"
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400"
|
||||
/>
|
||||
<input
|
||||
type="password"
|
||||
placeholder="Пароль"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
onKeyDown={(e) => e.key === 'Enter' && handle()}
|
||||
className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400"
|
||||
/>
|
||||
|
||||
{error && <p className="text-red-500 text-sm text-center">{error}</p>}
|
||||
|
||||
<button
|
||||
onClick={handle}
|
||||
disabled={loading || !email || !password}
|
||||
className="py-2.5 rounded-xl bg-blue-600 text-white font-semibold hover:bg-blue-700 disabled:opacity-40 transition-colors"
|
||||
>
|
||||
{loading ? '…' : mode === 'login' ? 'Войти' : 'Зарегистрироваться'}
|
||||
</button>
|
||||
|
||||
<button
|
||||
onClick={() => onAuth('')}
|
||||
className="text-xs text-gray-400 hover:text-gray-600 text-center"
|
||||
>
|
||||
Продолжить без входа
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
128
frontend/src/pages/HistoryPage.tsx
Normal file
128
frontend/src/pages/HistoryPage.tsx
Normal file
@@ -0,0 +1,128 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { listReports, downloadReportPdf, ReportSummary } from '../api/client'
|
||||
|
||||
interface Props {
|
||||
onOpen: (taskId: string) => void
|
||||
onNewCheck: () => void
|
||||
}
|
||||
|
||||
function statusBadge(status: string) {
|
||||
const map: Record<string, string> = {
|
||||
completed: 'bg-green-100 text-green-700',
|
||||
processing: 'bg-blue-100 text-blue-700',
|
||||
failed: 'bg-red-100 text-red-700',
|
||||
pending: 'bg-gray-100 text-gray-500',
|
||||
}
|
||||
const label: Record<string, string> = {
|
||||
completed: 'Готово',
|
||||
processing: 'Анализ…',
|
||||
failed: 'Ошибка',
|
||||
pending: 'Ожидание',
|
||||
}
|
||||
return (
|
||||
<span className={`text-xs font-medium px-2 py-0.5 rounded-full ${map[status] ?? 'bg-gray-100 text-gray-500'}`}>
|
||||
{label[status] ?? status}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
function similarityColor(v?: number | null) {
|
||||
if (v == null) return 'text-gray-400'
|
||||
if (v < 20) return 'text-green-600'
|
||||
if (v < 50) return 'text-yellow-600'
|
||||
return 'text-red-600'
|
||||
}
|
||||
|
||||
export default function HistoryPage({ onOpen, onNewCheck }: Props) {
|
||||
const [tasks, setTasks] = useState<ReportSummary[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
listReports()
|
||||
.then(setTasks)
|
||||
.catch(() => setError('Не удалось загрузить историю'))
|
||||
.finally(() => setLoading(false))
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 flex flex-col">
|
||||
<header className="bg-white border-b px-6 py-4 flex items-center justify-between">
|
||||
<h1 className="font-semibold text-gray-800">История проверок</h1>
|
||||
<button
|
||||
onClick={onNewCheck}
|
||||
className="text-sm bg-blue-600 text-white px-4 py-2 rounded-xl hover:bg-blue-700 transition-colors"
|
||||
>
|
||||
+ Новая проверка
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div className="flex-1 p-6 max-w-3xl mx-auto w-full">
|
||||
{loading && (
|
||||
<div className="text-center text-gray-400 mt-12 animate-pulse">Загрузка…</div>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-xl p-4 text-red-600 text-sm">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && !error && tasks.length === 0 && (
|
||||
<div className="text-center text-gray-400 mt-12">
|
||||
<p className="text-lg">Проверок пока нет</p>
|
||||
<button onClick={onNewCheck} className="mt-4 text-blue-600 hover:underline text-sm">
|
||||
Загрузить первый документ
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tasks.length > 0 && (
|
||||
<ul className="flex flex-col gap-3">
|
||||
{tasks.map((t) => (
|
||||
<li
|
||||
key={t.task_id}
|
||||
className="bg-white rounded-2xl shadow-sm border border-gray-100 px-5 py-4 flex items-center gap-4"
|
||||
>
|
||||
{/* Similarity circle */}
|
||||
<div className={`text-2xl font-bold w-16 text-center shrink-0 ${similarityColor(t.overall_similarity)}`}>
|
||||
{t.overall_similarity != null ? `${t.overall_similarity}%` : '—'}
|
||||
</div>
|
||||
|
||||
{/* Info */}
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
{statusBadge(t.status)}
|
||||
<span className="text-xs text-gray-400">
|
||||
{t.created_at ? new Date(t.created_at).toLocaleString('ru-RU') : ''}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-gray-400 truncate font-mono">{t.task_id}</p>
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className="flex gap-2 shrink-0">
|
||||
{t.status === 'completed' && (
|
||||
<button
|
||||
onClick={() => { downloadReportPdf(t.task_id).catch(console.error) }}
|
||||
className="text-xs border border-gray-200 rounded-lg px-3 py-1.5 hover:bg-gray-50 transition-colors"
|
||||
title="Скачать PDF"
|
||||
>
|
||||
PDF
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={() => onOpen(t.task_id)}
|
||||
className="text-xs bg-blue-50 text-blue-700 border border-blue-200 rounded-lg px-3 py-1.5 hover:bg-blue-100 transition-colors"
|
||||
>
|
||||
Открыть
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
53
frontend/src/pages/Home.tsx
Normal file
53
frontend/src/pages/Home.tsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { useState } from 'react'
|
||||
import DropZone from '../components/DropZone'
|
||||
import { uploadDocument } from '../api/client'
|
||||
|
||||
interface Props {
|
||||
onTaskCreated: (taskId: string) => void
|
||||
}
|
||||
|
||||
export default function Home({ onTaskCreated }: Props) {
|
||||
const [file, setFile] = useState<File | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const handleSubmit = async () => {
|
||||
if (!file) return
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const res = await uploadDocument(file)
|
||||
onTaskCreated(res.task_id)
|
||||
} catch (e: unknown) {
|
||||
const msg = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail ?? 'Ошибка загрузки'
|
||||
setError(msg)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 flex flex-col items-center justify-center p-6">
|
||||
<div className="bg-white rounded-2xl shadow-md p-8 w-full max-w-lg flex flex-col gap-6">
|
||||
<h1 className="text-2xl font-bold text-gray-800 text-center">Антиплагиат</h1>
|
||||
<p className="text-sm text-gray-500 text-center">
|
||||
Загрузите документ для проверки на заимствования
|
||||
</p>
|
||||
|
||||
<DropZone onFile={setFile} disabled={loading} />
|
||||
|
||||
{error && (
|
||||
<p className="text-red-500 text-sm text-center">{error}</p>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
disabled={!file || loading}
|
||||
className="w-full py-3 rounded-xl bg-blue-600 text-white font-semibold hover:bg-blue-700 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{loading ? 'Загрузка…' : 'Проверить'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
206
frontend/src/pages/Report.tsx
Normal file
206
frontend/src/pages/Report.tsx
Normal file
@@ -0,0 +1,206 @@
|
||||
import { useEffect, useState, useCallback, useRef } from 'react'
|
||||
import { getReport, openProgressSocket, downloadReportPdf, ReportResponse, MatchResult, ProgressEvent } from '../api/client'
|
||||
import SimilarityMeter from '../components/SimilarityMeter'
|
||||
import MatchCard from '../components/MatchCard'
|
||||
import HighlightedText from '../components/HighlightedText'
|
||||
|
||||
interface Props {
|
||||
taskId: string
|
||||
onBack: () => void
|
||||
}
|
||||
|
||||
const STAGE_LABEL: Record<string, string> = {
|
||||
fingerprint: 'Поиск точных совпадений…',
|
||||
fuzzy: 'Нечёткое сравнение…',
|
||||
semantic: 'Семантический анализ…',
|
||||
saving: 'Сохранение результатов…',
|
||||
done: 'Готово',
|
||||
failed: 'Ошибка',
|
||||
}
|
||||
|
||||
const POLL_INTERVAL = 4000
|
||||
|
||||
export default function Report({ taskId, onBack }: Props) {
|
||||
const [report, setReport] = useState<ReportResponse | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [progress, setProgress] = useState<ProgressEvent | null>(null)
|
||||
const [activeMatch, setActiveMatch] = useState<number | null>(null)
|
||||
const pollRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
||||
|
||||
const fetchReport = useCallback(async () => {
|
||||
try {
|
||||
const data = await getReport(taskId)
|
||||
setReport(data)
|
||||
return data.status
|
||||
} catch {
|
||||
setError('Не удалось загрузить отчёт')
|
||||
return 'failed'
|
||||
}
|
||||
}, [taskId])
|
||||
|
||||
// WebSocket progress + fallback polling
|
||||
useEffect(() => {
|
||||
const closeWs = openProgressSocket(taskId, (ev) => {
|
||||
setProgress(ev)
|
||||
if (ev.stage === 'done' || ev.stage === 'failed') {
|
||||
fetchReport()
|
||||
}
|
||||
})
|
||||
|
||||
// Polling fallback if WS doesn't deliver done
|
||||
const poll = async () => {
|
||||
const s = await fetchReport()
|
||||
if (s === 'processing') {
|
||||
pollRef.current = setTimeout(poll, POLL_INTERVAL)
|
||||
}
|
||||
}
|
||||
poll()
|
||||
|
||||
return () => {
|
||||
closeWs()
|
||||
if (pollRef.current) clearTimeout(pollRef.current)
|
||||
}
|
||||
}, [taskId, fetchReport])
|
||||
|
||||
const isProcessing = !report || report.status === 'processing'
|
||||
const matches: MatchResult[] = report?.matches ?? []
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 flex flex-col">
|
||||
{/* Header */}
|
||||
<header className="bg-white border-b px-6 py-4 flex items-center justify-between">
|
||||
<button onClick={onBack} className="text-sm text-blue-600 hover:underline">
|
||||
← Новая проверка
|
||||
</button>
|
||||
<h1 className="font-semibold text-gray-800">Отчёт о проверке</h1>
|
||||
<div className="flex items-center gap-3">
|
||||
{report?.status === 'completed' && (
|
||||
<button
|
||||
onClick={() => { downloadReportPdf(taskId).catch(console.error) }}
|
||||
className="text-xs border border-gray-200 rounded-lg px-3 py-1.5 hover:bg-gray-50 transition-colors"
|
||||
>
|
||||
Скачать PDF
|
||||
</button>
|
||||
)}
|
||||
<span className="text-xs text-gray-400">{taskId.slice(0, 8)}…</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* Progress bar */}
|
||||
{isProcessing && (
|
||||
<div className="bg-white border-b px-6 py-3">
|
||||
<div className="flex justify-between text-sm text-gray-600 mb-1">
|
||||
<span>{progress ? STAGE_LABEL[progress.stage] : 'Запуск анализа…'}</span>
|
||||
<span>{progress?.progress ?? 0}%</span>
|
||||
</div>
|
||||
<div className="w-full h-2 bg-gray-200 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-blue-500 rounded-full transition-all duration-500"
|
||||
style={{ width: `${progress?.progress ?? 0}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-gray-400 mt-1">
|
||||
{['fingerprint', 'fuzzy', 'semantic', 'saving', 'done'].map((stage) => (
|
||||
<span
|
||||
key={stage}
|
||||
className={progress && ['fingerprint','fuzzy','semantic','saving','done'].indexOf(stage) <=
|
||||
['fingerprint','fuzzy','semantic','saving','done'].indexOf(progress.stage)
|
||||
? 'text-blue-600 font-medium' : ''}
|
||||
>
|
||||
{STAGE_LABEL[stage].replace('…', '')}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<div className="m-4 bg-red-50 border border-red-200 rounded-xl p-4 text-red-600 text-sm">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Main content */}
|
||||
{report && report.status === 'completed' && (
|
||||
<>
|
||||
{/* Summary bar */}
|
||||
<div className="bg-white border-b px-6 py-4">
|
||||
<div className="max-w-4xl mx-auto">
|
||||
<SimilarityMeter value={report.overall_similarity ?? 0} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Legend */}
|
||||
<div className="bg-white border-b px-6 py-2 flex gap-4 text-xs text-gray-600">
|
||||
<span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-red-200 inline-block" /> Точное совпадение</span>
|
||||
<span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-orange-200 inline-block" /> Нечёткое</span>
|
||||
<span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-yellow-200 inline-block" /> Семантика</span>
|
||||
</div>
|
||||
|
||||
{/* Two-column layout */}
|
||||
<div className="flex-1 flex overflow-hidden">
|
||||
{/* Left: highlighted document text */}
|
||||
<div className="flex-1 overflow-y-auto p-6">
|
||||
<div className="max-w-3xl mx-auto bg-white rounded-2xl shadow-sm p-6 min-h-full">
|
||||
{report.source_text ? (
|
||||
<HighlightedText
|
||||
text={report.source_text}
|
||||
matches={matches}
|
||||
activeIndex={activeMatch}
|
||||
onMatchClick={setActiveMatch}
|
||||
/>
|
||||
) : (
|
||||
<p className="text-gray-400 text-sm">Текст документа недоступен</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Right: match list */}
|
||||
<aside className="w-80 shrink-0 overflow-y-auto border-l bg-gray-50 p-4 flex flex-col gap-3">
|
||||
<p className="text-xs font-semibold text-gray-500 uppercase tracking-wide">
|
||||
Совпадения ({matches.length})
|
||||
</p>
|
||||
|
||||
{matches.length === 0 && (
|
||||
<p className="text-sm text-gray-400 text-center mt-8">
|
||||
Заимствований не обнаружено
|
||||
</p>
|
||||
)}
|
||||
|
||||
{matches.map((m, i) => (
|
||||
<MatchCard
|
||||
key={i}
|
||||
match={m}
|
||||
index={i}
|
||||
active={activeMatch === i}
|
||||
onClick={() => setActiveMatch(activeMatch === i ? null : i)}
|
||||
/>
|
||||
))}
|
||||
</aside>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Still processing with no result yet */}
|
||||
{isProcessing && !error && (
|
||||
<div className="flex-1 flex items-center justify-center">
|
||||
<div className="text-center text-gray-400">
|
||||
<div className="text-4xl mb-3 animate-spin">⚙</div>
|
||||
<p className="text-sm">Анализируем документ…</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{report?.status === 'failed' && (
|
||||
<div className="flex-1 flex items-center justify-center">
|
||||
<div className="text-center text-red-500">
|
||||
<p className="text-lg font-semibold">Анализ завершился с ошибкой</p>
|
||||
<button onClick={onBack} className="mt-4 text-sm text-blue-600 hover:underline">
|
||||
Попробовать снова
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
8
frontend/tailwind.config.js
Normal file
8
frontend/tailwind.config.js
Normal file
@@ -0,0 +1,8 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: ['./index.html', './src/**/*.{ts,tsx}'],
|
||||
theme: {
|
||||
extend: {},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
||||
15
frontend/vite.config.ts
Normal file
15
frontend/vite.config.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8000',
|
||||
changeOrigin: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
37
requirements.txt
Normal file
37
requirements.txt
Normal file
@@ -0,0 +1,37 @@
|
||||
fastapi==0.104.1
|
||||
uvicorn==0.24.0
|
||||
sqlalchemy==2.0.23
|
||||
psycopg2-binary==2.9.9
|
||||
pydantic==2.5.0
|
||||
pydantic-settings==2.1.0
|
||||
python-multipart==0.0.6
|
||||
python-jose==3.3.0
|
||||
passlib==1.7.4
|
||||
bcrypt==4.1.1
|
||||
|
||||
elasticsearch==8.11.0
|
||||
faiss-cpu==1.7.4
|
||||
sentence-transformers==2.2.2
|
||||
datasketch==1.0.8
|
||||
xxhash==3.4.1
|
||||
|
||||
celery==5.3.4
|
||||
redis[asyncio]==5.0.1
|
||||
minio==7.2.0
|
||||
|
||||
numpy==1.24.3
|
||||
scipy==1.11.4
|
||||
scikit-learn==1.3.2
|
||||
|
||||
wikitextparser==0.31.9
|
||||
pymupdf==1.23.8
|
||||
pdfplumber==0.10.3
|
||||
python-docx==0.8.11
|
||||
|
||||
requests==2.31.0
|
||||
aiofiles==23.2.1
|
||||
websockets==12.0
|
||||
pydantic-extra-types==2.3.0
|
||||
|
||||
python-dotenv==1.0.0
|
||||
reportlab==4.1.0
|
||||
Reference in New Issue
Block a user