test build

.env.example

@@ -0,0 +1,34 @@
+# PostgreSQL
+DATABASE_URL=postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator
+
+# Elasticsearch
+ELASTICSEARCH_HOST=localhost
+ELASTICSEARCH_PORT=9200
+
+# Redis / Celery
+REDIS_URL=redis://localhost:6379/0
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+
+# MinIO
+MINIO_ENDPOINT=localhost:9000
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BUCKET=antiplagiator
+MINIO_SECURE=false
+
+# JWT — change in production!
+SECRET_KEY=change-me-in-production
+ALGORITHM=HS256
+ACCESS_TOKEN_EXPIRE_MINUTES=30
+
+# Analysis thresholds
+FINGERPRINT_THRESHOLD=0.5
+SEMANTIC_THRESHOLD=0.6
+TOP_K_RESULTS=10
+
+# Data directories
+DATA_DIR=./data
+RAW_DATA_DIR=./data/raw
+PROCESSED_DATA_DIR=./data/processed
+INDEX_DIR=./data/index

.github/workflows/deploy.yml

@@ -0,0 +1,52 @@
+name: Deploy
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build & push backend
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-backend:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build & push frontend
+        uses: docker/build-push-action@v5
+        with:
+          context: ./frontend
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-frontend:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Deploy to server via SSH
+        uses: appleboy/ssh-action@v1
+        with:
+          host: ${{ secrets.SERVER_HOST }}
+          username: ${{ secrets.SERVER_USER }}
+          key: ${{ secrets.SERVER_SSH_KEY }}
+          script: |
+            cd /opt/antiplagiator
+            docker compose pull
+            docker compose up -d --remove-orphans
+            docker image prune -f

backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application
+COPY . .
+
+# Create data directories
+RUN mkdir -p /app/data/{raw,processed,index}
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/app/api/admin.py

@@ -0,0 +1,115 @@
+from fastapi import APIRouter, HTTPException, status, WebSocket, WebSocketDisconnect
+from pydantic import BaseModel
+import asyncio
+import json as _json
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+SOURCES = {
+    "wikipedia_ru": "Wikipedia RU (~8 GB)",
+    "wikipedia_en": "Wikipedia EN (~22 GB)",
+}
+
+
+class JobResponse(BaseModel):
+    status: str
+    source: str | None = None
+    label: str | None = None
+    progress: int = 0
+    message: str = ""
+    started_at: str | None = None
+    finished_at: str | None = None
+    docs_done: int = 0
+
+
+@router.get("/index/sources")
+def list_sources():
+    """Список доступных источников для индексации."""
+    return [{"key": k, "label": v} for k, v in SOURCES.items()]
+
+
+@router.post("/index/download/{source_key}", response_model=JobResponse)
+def start_download(source_key: str):
+    """Скачать дамп и проиндексировать с нуля."""
+    if source_key not in SOURCES:
+        raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
+
+    from app.tasks.indexing import get_job_status, download_and_index
+    current = get_job_status()
+    if current.get("status") == "running":
+        raise HTTPException(status_code=409, detail="Задание уже выполняется")
+
+    download_and_index.delay(source_key)
+    return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск скачивания…")
+
+
+@router.post("/index/update/{source_key}", response_model=JobResponse)
+def start_update(source_key: str):
+    """Обновить индекс — только новые документы (дамп не скачивается заново если уже есть)."""
+    if source_key not in SOURCES:
+        raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
+
+    from app.tasks.indexing import get_job_status, update_index
+    current = get_job_status()
+    if current.get("status") == "running":
+        raise HTTPException(status_code=409, detail="Задание уже выполняется")
+
+    update_index.delay(source_key)
+    return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск обновления…")
+
+
+@router.get("/index/status", response_model=JobResponse)
+def job_status():
+    """Текущий статус задания индексации."""
+    from app.tasks.indexing import get_job_status
+    data = get_job_status()
+    return JobResponse(
+        status=data.get("status", "idle"),
+        source=data.get("source"),
+        label=data.get("label"),
+        progress=data.get("progress", 0),
+        message=data.get("message", ""),
+        started_at=data.get("started_at"),
+        finished_at=data.get("finished_at"),
+        docs_done=data.get("docs_done", 0),
+    )
+
+
+@router.websocket("/ws/index-progress")
+async def ws_index_progress(websocket: WebSocket):
+    """Stream indexing job progress via Redis pub/sub."""
+    await websocket.accept()
+
+    import redis.asyncio as aioredis
+    from app.config import settings
+
+    r = aioredis.from_url(settings.REDIS_URL)
+    pubsub = r.pubsub()
+    await pubsub.subscribe("index_job_progress")
+
+    # Send current state immediately
+    from app.tasks.indexing import get_job_status
+    await websocket.send_json(get_job_status())
+
+    try:
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            try:
+                data = _json.loads(message["data"])
+            except Exception:
+                continue
+            await websocket.send_json(data)
+            if data.get("status") in ("done", "error"):
+                break
+    except (WebSocketDisconnect, asyncio.CancelledError):
+        pass
+    finally:
+        try:
+            await pubsub.unsubscribe("index_job_progress")
+        except Exception:
+            pass
+        try:
+            await r.aclose()
+        except Exception:
+            pass

backend/app/api/auth.py

@@ -0,0 +1,115 @@
+from datetime import datetime, timedelta
+from typing import Optional
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+from pydantic import BaseModel, EmailStr
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models.db import get_db, User
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
+
+
+# ── Schemas ────────────────────────────────────────────────────────────────
+
+class RegisterRequest(BaseModel):
+    email: EmailStr
+    password: str
+
+
+class TokenResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+
+
+class UserOut(BaseModel):
+    id: str
+    email: str
+    created_at: datetime
+
+    class Config:
+        from_attributes = True
+
+
+# ── Helpers ────────────────────────────────────────────────────────────────
+
+def hash_password(password: str) -> str:
+    return pwd_context.hash(password)
+
+
+def verify_password(plain: str, hashed: str) -> bool:
+    return pwd_context.verify(plain, hashed)
+
+
+def create_token(user_id) -> str:
+    expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+    return jwt.encode(
+        {"sub": str(user_id), "exp": expire},
+        settings.SECRET_KEY,
+        algorithm=settings.ALGORITHM,
+    )
+
+
+def get_current_user(
+    token: Optional[str] = Depends(oauth2_scheme),
+    db: Session = Depends(get_db),
+) -> Optional[User]:
+    if not token:
+        return None
+    try:
+        payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
+        user_id_str: str = payload.get("sub")
+        if not user_id_str:
+            return None
+        import uuid as _uuid
+        user_id = _uuid.UUID(user_id_str)
+    except (JWTError, ValueError):
+        return None
+    return db.query(User).filter(User.id == user_id).first()
+
+
+def require_user(current_user: Optional[User] = Depends(get_current_user)) -> User:
+    if not current_user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return current_user
+
+
+# ── Endpoints ──────────────────────────────────────────────────────────────
+
+@router.post("/register", response_model=UserOut, status_code=status.HTTP_201_CREATED)
+def register(body: RegisterRequest, db: Session = Depends(get_db)):
+    if db.query(User).filter(User.email == body.email).first():
+        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered")
+    user = User(
+        email=body.email,
+        hashed_password=hash_password(body.password),
+    )
+    db.add(user)
+    db.commit()
+    db.refresh(user)
+    return user
+
+
+@router.post("/login", response_model=TokenResponse)
+def login(form: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):
+    user = db.query(User).filter(User.email == form.username).first()
+    if not user or not verify_password(form.password, user.hashed_password):
+        raise HTTPException(status_code=400, detail="Incorrect email or password")
+    return {"access_token": create_token(user.id)}
+
+
+@router.get("/me", response_model=UserOut)
+def me(current_user: User = Depends(require_user)):
+    return current_user

backend/app/celery_app.py

@@ -0,0 +1,18 @@
+from celery import Celery
+from app.config import settings
+
+celery_app = Celery(
+    "antiplagiator",
+    broker=settings.CELERY_BROKER_URL,
+    backend=settings.CELERY_RESULT_BACKEND,
+    include=["app.tasks", "app.tasks.indexing"],
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    result_serializer="json",
+    accept_content=["json"],
+    timezone="UTC",
+    enable_utc=True,
+    task_track_started=True,
+)

backend/app/config.py

@@ -0,0 +1,50 @@
+import os
+from pydantic_settings import BaseSettings
+
+class Settings(BaseSettings):
+    # Database
+    DATABASE_URL: str = "postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator"
+
+    # Elasticsearch
+    ELASTICSEARCH_HOST: str = "localhost"
+    ELASTICSEARCH_PORT: int = 9200
+
+    # Redis
+    REDIS_URL: str = "redis://localhost:6379/0"
+
+    # MinIO
+    MINIO_ENDPOINT: str = "localhost:9000"
+    MINIO_ACCESS_KEY: str = "minioadmin"
+    MINIO_SECRET_KEY: str = "minioadmin"
+    MINIO_BUCKET: str = "antiplagiator"
+    MINIO_SECURE: bool = False
+
+    # JWT
+    SECRET_KEY: str = "your-secret-key-change-in-prod"
+    ALGORITHM: str = "HS256"
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
+
+    # Celery
+    CELERY_BROKER_URL: str = "redis://localhost:6379/0"
+    CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
+
+    # API
+    API_TITLE: str = "Antiplagiator API"
+    API_VERSION: str = "0.1.0"
+    API_DESCRIPTION: str = "Text plagiarism detection system"
+
+    # Paths
+    DATA_DIR: str = "./data"
+    RAW_DATA_DIR: str = "./data/raw"
+    PROCESSED_DATA_DIR: str = "./data/processed"
+    INDEX_DIR: str = "./data/index"
+
+    # Analysis
+    FINGERPRINT_THRESHOLD: float = 0.5
+    SEMANTIC_THRESHOLD: float = 0.6
+    TOP_K_RESULTS: int = 10
+
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

backend/app/core/analyzer.py

@@ -0,0 +1,149 @@
+import re
+from typing import Dict, List, Any, Tuple
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+from app.config import settings
+
+FRAGMENT_WORDS = 40
+
+
+def _split_sentences(text: str) -> List[str]:
+    parts = re.split(r'(?<=[.!?])\s+', text.strip())
+    return [p for p in parts if p]
+
+
+def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]:
+    """Return (fragment_text, char_start, char_end) around the given sentence index."""
+    sentences = _split_sentences(text)
+    start_i = max(0, sentence_idx - context_sentences)
+    end_i = min(len(sentences), sentence_idx + context_sentences + 1)
+
+    fragment = " ".join(sentences[start_i:end_i])
+    char_start = text.find(sentences[start_i]) if sentences else 0
+    char_end = char_start + len(fragment)
+    return fragment, char_start, char_end
+
+
+def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str:
+    words = text.split()
+    if len(words) <= max_words:
+        return text
+    return " ".join(words[:max_words]) + "…"
+
+
+class DocumentAnalyzer:
+    def __init__(self):
+        self.winnowing = WinNowing()
+        self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD)
+        self.embeddings = EmbeddingIndex()
+        self.fulltext = FulltextSearch()
+
+    def analyze(self, text: str, task_id: str) -> Dict[str, Any]:
+        matches: List[Dict[str, Any]] = []
+        matches.extend(self._level_1_exact_match(text))
+        matches.extend(self._level_2_fuzzy_match(text))
+        matches.extend(self._level_3_semantic_match(text))
+
+        return {
+            "task_id": task_id,
+            "status": "completed",
+            "overall_similarity": self._calculate_overall_similarity(matches),
+            "matches": matches[:settings.TOP_K_RESULTS],
+        }
+
+    def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        sentences = _split_sentences(text)
+        if not sentences:
+            return []
+
+        results = []
+        for idx, sentence in enumerate(sentences[:20]):
+            if len(sentence.split()) < 6:
+                continue
+            try:
+                es_results = self.fulltext.search_phrase(sentence, top_k=2)
+                for r in es_results:
+                    r["_sentence_idx"] = idx
+                results.extend(es_results)
+            except Exception:
+                continue
+
+        matches = []
+        seen_docs: set = set()
+        for result in results:
+            if result["doc_id"] in seen_docs:
+                continue
+            idx = result.get("_sentence_idx", 0)
+            fragment, char_start, char_end = _extract_fragment(text, idx)
+            matches.append({
+                "method": "exact",
+                "similarity": min(result["score"] * 20, 100),
+                "source_id": str(result["doc_id"]),   # передаём doc_id напрямую
+                "source_title": result["title"],
+                "source_db": result["source"],
+                "url": result["url"],
+                "fragment_text": _truncate(fragment),
+                "fragment_start": char_start,
+                "fragment_end": char_end,
+            })
+            seen_docs.add(result["doc_id"])
+
+        return matches[:top_k]
+
+    def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            fuzzy_results = self.minhash_lsh.query(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in fuzzy_results:
+            if similarity >= settings.FINGERPRINT_THRESHOLD:
+                matches.append({
+                    "method": "fuzzy",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            embedding_results = self.embeddings.search(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in embedding_results:
+            if similarity >= settings.SEMANTIC_THRESHOLD:
+                matches.append({
+                    "method": "semantic",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float:
+        if not matches:
+            return 0.0
+
+        weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3}
+        total_weight = 0.0
+        total_similarity = 0.0
+
+        for match in matches:
+            w = weights.get(match["method"], 0.1)
+            total_similarity += match["similarity"] * w
+            total_weight += w
+
+        return round(total_similarity / total_weight if total_weight > 0 else 0, 1)

backend/app/core/embeddings.py

@@ -0,0 +1,76 @@
+import numpy as np
+import faiss
+from sentence_transformers import SentenceTransformer
+from typing import List, Tuple
+import os
+from app.config import settings
+
+class EmbeddingIndex:
+    def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"):
+        self.model = SentenceTransformer(model_name)
+        self.embedding_dim = self.model.get_sentence_embedding_dimension()
+        self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32)
+        self.doc_ids = []
+        self.embeddings = None
+        
+        # Try to load existing index
+        self._load_index()
+
+    def add_document(self, doc_id: str, text: str):
+        embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        self.index.add(np.array([embedding], dtype=np.float32))
+        self.doc_ids.append(doc_id)
+        
+        if self.embeddings is None:
+            self.embeddings = np.array([embedding])
+        else:
+            self.embeddings = np.vstack([self.embeddings, embedding])
+
+    def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32):
+        doc_ids = [doc[0] for doc in docs]
+        texts = [doc[1] for doc in docs]
+        
+        for i in range(0, len(texts), batch_size):
+            batch_texts = texts[i:i+batch_size]
+            embeddings = self.model.encode(batch_texts, convert_to_numpy=True)
+            
+            self.index.add(embeddings.astype(np.float32))
+            self.doc_ids.extend(doc_ids[i:i+batch_size])
+            
+            if self.embeddings is None:
+                self.embeddings = embeddings
+            else:
+                self.embeddings = np.vstack([self.embeddings, embeddings])
+
+    def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        query_embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        query_embedding = np.array([query_embedding], dtype=np.float32)
+        
+        distances, indices = self.index.search(query_embedding, top_k)
+        
+        results = []
+        for idx, distance in zip(indices[0], distances[0]):
+            if idx < len(self.doc_ids):
+                similarity = 1.0 / (1.0 + distance)
+                results.append((self.doc_ids[idx], similarity))
+        
+        return results
+
+    def _load_index(self):
+        index_path = os.path.join(settings.INDEX_DIR, "faiss_index")
+        ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy")
+        
+        if os.path.exists(index_path) and os.path.exists(ids_path):
+            try:
+                self.index = faiss.read_index(index_path)
+                self.doc_ids = list(np.load(ids_path, allow_pickle=True))
+            except Exception as e:
+                print(f"Failed to load index: {e}")
+
+    def save_index(self):
+        os.makedirs(settings.INDEX_DIR, exist_ok=True)
+        faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index"))
+        np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object))
+
+    def get_index_size(self) -> int:
+        return self.index.ntotal

backend/app/core/fingerprint.py

@@ -0,0 +1,82 @@
+import xxhash
+from typing import Set, Tuple, List
+from datasketch import MinHash, MinHashLSH
+import numpy as np
+
+class WinNowing:
+    def __init__(self, k: int = 5, window_size: int = 4):
+        self.k = k
+        self.window_size = window_size
+
+    def get_kgrams(self, text: str) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(len(words) - self.k + 1):
+            kgram = ' '.join(words[i:i+self.k])
+            kgrams.append(kgram)
+        return kgrams
+
+    def fingerprint(self, text: str) -> Set[int]:
+        kgrams = self.get_kgrams(text)
+        hashes = []
+        for kgram in kgrams:
+            h = int(xxhash.xxh64(kgram).hexdigest(), 16)
+            hashes.append(h)
+        
+        if not hashes:
+            return set()
+        
+        fingerprints = set()
+        for i in range(len(hashes) - self.window_size + 1):
+            window = hashes[i:i+self.window_size]
+            min_hash = min(window)
+            fingerprints.add(min_hash)
+        
+        return fingerprints
+
+    def compare(self, fp1: Set[int], fp2: Set[int]) -> float:
+        if not fp1 or not fp2:
+            return 0.0
+        intersection = len(fp1 & fp2)
+        union = len(fp1 | fp2)
+        return intersection / union if union > 0 else 0.0
+
+class MinHashLSHIndex:
+    def __init__(self, num_perm: int = 128, threshold: float = 0.5):
+        self.num_perm = num_perm
+        self.threshold = threshold
+        self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
+        self.documents = {}
+
+    def add_document(self, doc_id: str, text: str):
+        kgrams = self._get_kgrams(text)
+        m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            m.update(kgram.encode())
+        
+        self.lsh.insert(doc_id, m)
+        self.documents[doc_id] = m
+
+    def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        kgrams = self._get_kgrams(text)
+        query_m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            query_m.update(kgram.encode())
+        
+        candidates = self.lsh.query(query_m)
+        
+        results = []
+        for doc_id in candidates:
+            similarity = query_m.jaccard(self.documents[doc_id])
+            results.append((doc_id, similarity))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def _get_kgrams(self, text: str, k: int = 5) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(max(1, len(words) - k + 1)):
+            kgram = ' '.join(words[i:i+k])
+            kgrams.append(kgram)
+        return kgrams

backend/app/core/fulltext.py

@@ -0,0 +1,135 @@
+from elasticsearch import Elasticsearch
+from typing import List, Tuple, Dict, Any
+from app.config import settings
+
+class FulltextSearch:
+    def __init__(self):
+        self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"])
+        self.index_name = "documents"
+        self._create_index()
+
+    def _create_index(self):
+        if not self.es.indices.exists(index=self.index_name):
+            self.es.indices.create(
+                index=self.index_name,
+                body={
+                    "settings": {
+                        "analysis": {
+                            "analyzer": {
+                                "ru_en": {
+                                    "type": "custom",
+                                    "tokenizer": "standard",
+                                    "filter": ["lowercase", "ru_stop", "ru_stem"]
+                                }
+                            },
+                            "filter": {
+                                "ru_stop": {
+                                    "type": "stop",
+                                    "stopwords": "_russian_"
+                                },
+                                "ru_stem": {
+                                    "type": "stemmer",
+                                    "language": "russian"
+                                }
+                            }
+                        }
+                    },
+                    "mappings": {
+                        "properties": {
+                            "doc_id": {"type": "keyword"},
+                            "text": {"type": "text", "analyzer": "ru_en"},
+                            "title": {"type": "text", "analyzer": "ru_en"},
+                            "source": {"type": "keyword"},
+                            "lang": {"type": "keyword"},
+                            "url": {"type": "keyword"}
+                        }
+                    }
+                }
+            )
+
+    def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""):
+        self.es.index(
+            index=self.index_name,
+            id=doc_id,
+            body={
+                "doc_id": doc_id,
+                "title": title,
+                "text": text,
+                "source": source,
+                "lang": lang,
+                "url": url
+            }
+        )
+
+    def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]:
+        must = {
+            "multi_match": {
+                "query": query,
+                "fields": ["text^2", "title^3"],
+                "type": "best_fields",
+                "analyzer": "ru_en",
+            }
+        }
+
+        if lang:
+            search_body = {
+                "size": top_k,
+                "query": {
+                    "bool": {
+                        "must": must,
+                        "filter": {"term": {"lang": lang}},
+                    }
+                },
+            }
+        else:
+            search_body = {"size": top_k, "query": must}
+
+        results = self.es.search(index=self.index_name, body=search_body)
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]:
+        results = self.es.search(
+            index=self.index_name,
+            body={
+                "size": top_k,
+                "query": {
+                    "match_phrase": {
+                        "text": phrase
+                    }
+                }
+            }
+        )
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def bulk_index(self, documents: List[Dict[str, Any]]):
+        actions = []
+        for doc in documents:
+            actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}})
+            actions.append(doc)
+        
+        from elasticsearch.helpers import bulk
+        bulk(self.es, actions)

backend/app/main.py

@@ -0,0 +1,354 @@
+from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+import uuid
+import io
+import asyncio
+import json as _json
+from sqlalchemy.orm import Session
+from app.config import settings
+from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User
+from app.schemas import (
+    CheckResponse, ReportResponse, ReportSummary,
+    ReportsListResponse, StatsResponse, HealthResponse, MatchResult,
+)
+from app.core.embeddings import EmbeddingIndex
+from app.api.auth import router as auth_router, get_current_user
+from app.api.admin import router as admin_router
+import pdfplumber
+import docx as python_docx
+from minio import Minio
+from minio.error import S3Error
+from typing import Optional
+
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(
+    title=settings.API_TITLE,
+    version=settings.API_VERSION,
+    description=settings.API_DESCRIPTION,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(auth_router)
+app.include_router(admin_router)
+
+_minio: Minio | None = None
+_embedding_index: EmbeddingIndex | None = None
+
+
+def get_minio() -> Minio:
+    global _minio
+    if _minio is None:
+        _minio = Minio(
+            settings.MINIO_ENDPOINT,
+            access_key=settings.MINIO_ACCESS_KEY,
+            secret_key=settings.MINIO_SECRET_KEY,
+            secure=settings.MINIO_SECURE,
+        )
+        try:
+            if not _minio.bucket_exists(settings.MINIO_BUCKET):
+                _minio.make_bucket(settings.MINIO_BUCKET)
+        except S3Error:
+            pass
+    return _minio
+
+
+def get_embedding_index() -> EmbeddingIndex:
+    global _embedding_index
+    if _embedding_index is None:
+        _embedding_index = EmbeddingIndex()
+    return _embedding_index
+
+
+def extract_text(content: bytes, filename: str) -> str:
+    if filename.endswith(".txt"):
+        return content.decode("utf-8", errors="replace")
+
+    if filename.endswith(".pdf"):
+        with pdfplumber.open(io.BytesIO(content)) as pdf:
+            return "\n".join(page.extract_text() or "" for page in pdf.pages)
+
+    if filename.endswith(".docx"):
+        doc = python_docx.Document(io.BytesIO(content))
+        return "\n".join(p.text for p in doc.paragraphs)
+
+    return content.decode("utf-8", errors="replace")
+
+
+@app.get("/api/health", response_model=HealthResponse)
+async def health_check():
+    return {"status": "ok", "version": settings.API_VERSION}
+
+
+@app.post("/api/check", response_model=CheckResponse)
+async def check_document(
+    file: UploadFile = File(...),
+    db: Session = Depends(get_db),
+    current_user: Optional[User] = Depends(get_current_user),
+):
+    if not file.filename:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename")
+
+    allowed = {".txt", ".pdf", ".docx"}
+    ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
+    if ext not in allowed:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Unsupported file type. Allowed: {', '.join(allowed)}",
+        )
+
+    task_id = uuid.uuid4()
+    content = await file.read()
+
+    minio_key = f"uploads/{task_id}/{file.filename}"
+    try:
+        minio = get_minio()
+        minio.put_object(
+            settings.MINIO_BUCKET,
+            minio_key,
+            io.BytesIO(content),
+            length=len(content),
+            content_type=file.content_type or "application/octet-stream",
+        )
+    except Exception:
+        minio_key = None
+
+    try:
+        text = extract_text(content, file.filename)
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}")
+
+    if not text.strip():
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable")
+
+    task = AnalysisTask(
+        id=task_id,
+        status="processing",
+        source_text=text,
+        user_id=current_user.id if current_user else None,
+    )
+    db.add(task)
+    db.commit()
+
+    from app.tasks import analyze_document
+    analyze_document.delay(text, str(task_id))
+
+    return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"}
+
+
+@app.get("/api/report/{task_id}", response_model=ReportResponse)
+async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)):
+    task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+    if not task:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
+
+    matches = []
+    if task.status == "completed":
+        db_matches = db.query(Match).filter(Match.task_id == task_id).all()
+        for m in db_matches:
+            doc = db.query(Document).filter(Document.id == m.doc_id).first()
+            matches.append(
+                MatchResult(
+                    method=m.method,
+                    similarity=float(m.similarity),
+                    source_title=doc.title if doc else None,
+                    source_db=doc.source if doc else None,
+                    url=doc.url if doc else None,
+                    fragment_text=m.fragment_text,
+                    fragment_start=m.fragment_start,
+                    fragment_end=m.fragment_end,
+                )
+            )
+
+    return ReportResponse(
+        task_id=str(task.id),
+        status=task.status,
+        overall_similarity=task.overall_similarity,
+        matches=matches or None,
+        source_text=task.source_text,
+        created_at=task.created_at,
+        completed_at=task.completed_at,
+    )
+
+
+@app.get("/api/report/{task_id}/pdf")
+async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)):
+    """Generate and return a PDF report."""
+    task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+    if not task:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
+    if task.status != "completed":
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet")
+
+    db_matches = db.query(Match).filter(Match.task_id == task_id).all()
+    matches_data = []
+    for m in db_matches:
+        doc = db.query(Document).filter(Document.id == m.doc_id).first()
+        matches_data.append({
+            "method": m.method,
+            "similarity": m.similarity,
+            "source_title": doc.title if doc else "—",
+            "source_db": doc.source if doc else "—",
+            "url": doc.url if doc else "",
+            "fragment_text": m.fragment_text or "",
+        })
+
+    pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data)
+
+    return StreamingResponse(
+        io.BytesIO(pdf_bytes),
+        media_type="application/pdf",
+        headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'},
+    )
+
+
+def _build_pdf(task_id: str, overall: float, matches: list) -> bytes:
+    from reportlab.lib.pagesizes import A4
+    from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
+    from reportlab.lib.units import cm
+    from reportlab.lib import colors
+    from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable
+    from reportlab.pdfbase import pdfmetrics
+    from reportlab.pdfbase.ttfonts import TTFont
+    import os
+
+    buf = io.BytesIO()
+    doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
+    styles = getSampleStyleSheet()
+
+    METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"}
+    METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")}
+
+    title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6)
+    h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4)
+    body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14)
+    small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey)
+
+    story = [
+        Paragraph("Отчёт о проверке на плагиат", title_style),
+        Paragraph(f"ID задачи: {task_id}", small_style),
+        Spacer(1, 0.3*cm),
+        HRFlowable(width="100%", thickness=1, color=colors.lightgrey),
+        Spacer(1, 0.3*cm),
+        Paragraph(f"Общий процент заимствований: <b>{overall:.1f}%</b>", body_style),
+        Spacer(1, 0.5*cm),
+    ]
+
+    if not matches:
+        story.append(Paragraph("Совпадений не обнаружено.", body_style))
+    else:
+        story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style))
+        for i, m in enumerate(matches, 1):
+            bg = METHOD_COLOR.get(m["method"], colors.whitesmoke)
+            data = [
+                [Paragraph(f"<b>#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%</b>", body_style), ""],
+                ["Источник:", Paragraph(m["source_title"], body_style)],
+                ["База:", m["source_db"]],
+            ]
+            if m["url"]:
+                data.append(["URL:", Paragraph(f'<link href="{m["url"]}">{m["url"]}</link>', small_style)])
+            if m["fragment_text"]:
+                data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)])
+
+            tbl = Table(data, colWidths=[3.5*cm, None])
+            tbl.setStyle(TableStyle([
+                ("BACKGROUND", (0, 0), (-1, 0), bg),
+                ("SPAN", (0, 0), (-1, 0)),
+                ("FONTSIZE", (0, 0), (-1, -1), 9),
+                ("VALIGN", (0, 0), (-1, -1), "TOP"),
+                ("TOPPADDING", (0, 0), (-1, -1), 3),
+                ("BOTTOMPADDING", (0, 0), (-1, -1), 3),
+                ("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey),
+            ]))
+            story.append(tbl)
+            story.append(Spacer(1, 0.3*cm))
+
+    doc.build(story)
+    return buf.getvalue()
+
+
+@app.get("/api/reports", response_model=ReportsListResponse)
+async def list_reports(
+    user_id: str | None = Query(default=None),
+    db: Session = Depends(get_db),
+    current_user: Optional[User] = Depends(get_current_user),
+):
+    query = db.query(AnalysisTask)
+    # Авторизованный пользователь видит только свои задачи
+    if current_user:
+        query = query.filter(AnalysisTask.user_id == current_user.id)
+    elif user_id:
+        query = query.filter(AnalysisTask.user_id == user_id)
+
+    tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all()
+
+    return ReportsListResponse(
+        tasks=[
+            ReportSummary(
+                task_id=str(t.id),
+                status=t.status,
+                overall_similarity=t.overall_similarity,
+                created_at=t.created_at,
+            )
+            for t in tasks
+        ]
+    )
+
+
+@app.get("/api/stats", response_model=StatsResponse)
+async def get_stats(db: Session = Depends(get_db)):
+    total_documents = db.query(Document).count()
+    index = get_embedding_index()
+
+    return StatsResponse(
+        total_documents_indexed=total_documents,
+        faiss_index_size=index.get_index_size(),
+        elasticsearch_available=True,
+        api_version=settings.API_VERSION,
+    )
+
+
+@app.websocket("/ws/progress/{task_id}")
+async def ws_progress(websocket: WebSocket, task_id: str):
+    await websocket.accept()
+    import redis.asyncio as aioredis
+
+    r = aioredis.from_url(settings.REDIS_URL)
+    pubsub = r.pubsub()
+    await pubsub.subscribe(f"progress:{task_id}")
+
+    try:
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            try:
+                data = _json.loads(message["data"])
+            except Exception:
+                continue
+            await websocket.send_json(data)
+            if data.get("stage") in ("done", "failed"):
+                break
+    except (WebSocketDisconnect, asyncio.CancelledError):
+        pass
+    finally:
+        try:
+            await pubsub.unsubscribe(f"progress:{task_id}")
+        except Exception:
+            pass
+        try:
+            await r.aclose()
+        except Exception:
+            pass
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

backend/app/models/db.py

@@ -0,0 +1,82 @@
+import uuid
+from sqlalchemy import Column, String, Text, DateTime, Integer, Float, ForeignKey, Index, create_engine, Boolean
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from datetime import datetime
+from app.config import settings
+
+Base = declarative_base()
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    email = Column(String(255), unique=True, nullable=False, index=True)
+    hashed_password = Column(String(255), nullable=False)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class Document(Base):
+    __tablename__ = "documents"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    source = Column(String(50), nullable=False)
+    ext_id = Column(String(200), nullable=True)
+    title = Column(Text, nullable=False)
+    url = Column(Text, nullable=True)
+    lang = Column(String(2), nullable=True)
+    minio_key = Column(Text, nullable=True)
+    indexed_at = Column(DateTime, default=datetime.utcnow)
+
+
+class Fingerprint(Base):
+    __tablename__ = "fingerprints"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False)
+    hash = Column(String(64), nullable=False)   # xxhash hex string
+    position = Column(Integer, nullable=True)
+
+
+Index("ix_fingerprints_hash", Fingerprint.hash)
+Index("ix_fingerprints_doc_id", Fingerprint.doc_id)
+
+
+class AnalysisTask(Base):
+    __tablename__ = "analysis_tasks"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
+    status = Column(String(20), default="pending")
+    overall_similarity = Column(Float, nullable=True)
+    source_text = Column(Text, nullable=True)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    completed_at = Column(DateTime, nullable=True)
+
+
+class Match(Base):
+    __tablename__ = "matches"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    task_id = Column(UUID(as_uuid=True), ForeignKey("analysis_tasks.id"), nullable=False)
+    doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=True)
+    similarity = Column(Float, nullable=False)
+    method = Column(String(20), nullable=False)
+    fragment_start = Column(Integer, nullable=True)
+    fragment_end = Column(Integer, nullable=True)
+    fragment_text = Column(Text, nullable=True)
+
+
+engine = create_engine(settings.DATABASE_URL, echo=False)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/schemas/__init__.py

@@ -0,0 +1,54 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List
+from datetime import datetime
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str
+
+
+class CheckResponse(BaseModel):
+    task_id: str
+    status: str
+    message: str
+
+
+class MatchResult(BaseModel):
+    method: str = Field(..., description="exact | fuzzy | semantic")
+    similarity: float = Field(..., ge=0, le=100)
+    source_title: Optional[str] = None
+    source_db: Optional[str] = None
+    url: Optional[str] = None
+    source_id: Optional[str] = None
+    fragment_text: Optional[str] = None
+    fragment_start: Optional[int] = None
+    fragment_end: Optional[int] = None
+
+
+class ReportResponse(BaseModel):
+    task_id: str
+    status: str
+    overall_similarity: Optional[float] = None
+    matches: Optional[List[MatchResult]] = None
+    source_text: Optional[str] = None
+    created_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+
+
+class ReportSummary(BaseModel):
+    task_id: str
+    status: str
+    overall_similarity: Optional[float] = None
+    created_at: Optional[datetime] = None
+
+
+class ReportsListResponse(BaseModel):
+    tasks: List[ReportSummary]
+
+
+class StatsResponse(BaseModel):
+    total_documents_indexed: int
+    faiss_index_size: int
+    elasticsearch_available: bool
+    api_version: str

backend/app/tasks/__init__.py

@@ -0,0 +1,95 @@
+import json
+import uuid
+import redis
+from app.config import settings
+from app.models.db import SessionLocal, AnalysisTask, Match
+from app.core.analyzer import DocumentAnalyzer
+from app.celery_app import celery_app
+from datetime import datetime
+
+_analyzer: DocumentAnalyzer | None = None
+_redis: redis.Redis | None = None
+
+
+def get_analyzer() -> DocumentAnalyzer:
+    global _analyzer
+    if _analyzer is None:
+        _analyzer = DocumentAnalyzer()
+    return _analyzer
+
+
+def get_redis() -> redis.Redis:
+    global _redis
+    if _redis is None:
+        _redis = redis.from_url(settings.REDIS_URL)
+    return _redis
+
+
+def _publish(task_id: str, stage: str, progress: int):
+    try:
+        get_redis().publish(
+            f"progress:{task_id}",
+            json.dumps({"stage": stage, "progress": progress}),
+        )
+    except Exception:
+        pass
+
+
+@celery_app.task(bind=True, name="tasks.analyze_document")
+def analyze_document(self, text: str, task_id: str):
+    db = SessionLocal()
+    try:
+        analyzer = get_analyzer()
+
+        _publish(task_id, "fingerprint", 10)
+        exact = analyzer._level_1_exact_match(text)
+
+        _publish(task_id, "fuzzy", 40)
+        fuzzy = analyzer._level_2_fuzzy_match(text)
+
+        _publish(task_id, "semantic", 70)
+        semantic = analyzer._level_3_semantic_match(text)
+
+        matches = exact + fuzzy + semantic
+        overall_similarity = analyzer._calculate_overall_similarity(matches)
+
+        _publish(task_id, "saving", 90)
+
+        task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+        if not task:
+            return
+
+        task.status = "completed"
+        task.overall_similarity = overall_similarity
+        task.completed_at = datetime.utcnow()
+
+        for match in matches[:settings.TOP_K_RESULTS]:
+            raw_id = match.get("source_id")
+            try:
+                doc_id = uuid.UUID(str(raw_id)) if raw_id is not None else None
+            except (ValueError, TypeError, AttributeError):
+                doc_id = None
+
+            db.add(Match(
+                task_id=task_id,
+                doc_id=doc_id,
+                similarity=float(match["similarity"]),
+                method=match["method"],
+                fragment_start=match.get("fragment_start"),
+                fragment_end=match.get("fragment_end"),
+                fragment_text=match.get("fragment_text"),
+            ))
+
+        db.commit()
+        _publish(task_id, "done", 100)
+
+    except Exception as exc:
+        task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+        if task:
+            task.status = "failed"
+            db.commit()
+        _publish(task_id, "failed", 0)
+        raise self.retry(exc=exc, countdown=5, max_retries=2)
+
+    finally:
+        db.close()

backend/app/tasks/indexing.py

@@ -0,0 +1,282 @@
+"""
+Celery tasks for downloading and indexing document sources.
+
+Redis key  "index_job"  stores current job state as JSON:
+  { status, source, progress, message, started_at, finished_at, docs_total, docs_done }
+"""
+import bz2
+import json
+import os
+import time
+import requests
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from pathlib import Path
+
+import redis
+from app.celery_app import celery_app
+
+from app.config import settings
+from app.models.db import SessionLocal, Document, Fingerprint
+
+JOB_KEY = "index_job"
+
+WIKIPEDIA_SOURCES = {
+    "wikipedia_ru": {
+        "url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-pages-articles.xml.bz2",
+        "meta_url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-md5sums.txt",
+        "lang": "ru",
+        "label": "Wikipedia RU",
+    },
+    "wikipedia_en": {
+        "url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2",
+        "meta_url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-md5sums.txt",
+        "lang": "en",
+        "label": "Wikipedia EN",
+    },
+}
+
+
+# ── Redis helpers ──────────────────────────────────────────────────────────
+
+def _r() -> redis.Redis:
+    return redis.from_url(settings.REDIS_URL)
+
+
+def _set_status(data: dict):
+    _r().set(JOB_KEY, json.dumps(data, default=str))
+
+
+def _get_status() -> dict:
+    raw = _r().get(JOB_KEY)
+    if not raw:
+        return {"status": "idle"}
+    return json.loads(raw)
+
+
+def _pub(message: str, progress: int = 0, **extra):
+    state = _get_status()
+    state.update({"message": message, "progress": progress, **extra})
+    _set_status(state)
+    _r().publish("index_job_progress", json.dumps(state, default=str))
+
+
+# ── Text helpers ───────────────────────────────────────────────────────────
+
+import re
+
+
+def _clean_wiki(text: str) -> str:
+    text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
+    text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
+    text = re.sub(r'\{\{[^}]*\}\}', '', text)
+    text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
+    text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
+    text = re.sub(r'<[^>]+>', '', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r' +', ' ', text)
+    return text.strip()
+
+
+def _normalize(text: str) -> str:
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
+    text = re.sub(r'[—–‒―]', '-', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
+
+
+# ── Download helper ────────────────────────────────────────────────────────
+
+def _download_file(url: str, dest: Path, label: str) -> Path:
+    """Stream-download url → dest, publishing progress."""
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    r = requests.get(url, stream=True, timeout=30)
+    r.raise_for_status()
+    total = int(r.headers.get("Content-Length", 0))
+    done = 0
+    last_pct = -1
+
+    with open(dest, "wb") as f:
+        for chunk in r.iter_content(chunk_size=1 << 20):  # 1 MB
+            f.write(chunk)
+            done += len(chunk)
+            pct = int(done / total * 40) if total else 0  # 0-40% of overall
+            if pct != last_pct:
+                _pub(f"Скачивание {label}: {done >> 20} MB", progress=pct)
+                last_pct = pct
+
+    return dest
+
+
+# ── Indexing helper ────────────────────────────────────────────────────────
+
+def _index_wikipedia_bz2(bz2_path: Path, source: str, lang: str, update_only: bool) -> int:
+    """
+    Parse Wikipedia bz2 dump and index documents.
+    update_only=True → skip ext_ids already in DB.
+    Returns count of newly indexed docs.
+    """
+    from app.core.fingerprint import WinNowing, MinHashLSHIndex
+    from app.core.embeddings import EmbeddingIndex
+    from app.core.fulltext import FulltextSearch
+
+    db = SessionLocal()
+    winnowing = WinNowing()
+    minhash = MinHashLSHIndex()
+    emb = EmbeddingIndex()
+    fulltext = FulltextSearch()
+
+    # Pre-load known ext_ids for fast skip
+    known: set[str] = set()
+    if update_only:
+        for (ext_id,) in db.query(Document.ext_id).filter(Document.source == source).yield_per(10_000):
+            if ext_id:
+                known.add(ext_id)
+
+    ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
+
+    embed_batch: list[tuple[str, str]] = []
+    BATCH = 32
+    count = 0
+    parsed = 0
+
+    with bz2.BZ2File(str(bz2_path), 'r') as f:
+        context = ET.iterparse(f, events=('end',))
+        for event, elem in context:
+            if not elem.tag.endswith('page'):
+                continue
+
+            title_el = elem.find('wiki:title', ns) or elem.find('.//title')
+            id_el = elem.find('wiki:id', ns) or elem.find('.//id')
+            rev = elem.find('.//revision')
+            text_el = rev.find('wiki:text', ns) if rev is not None else None
+            if rev is None:
+                text_el = elem.find('.//text')
+
+            if title_el is None or text_el is None:
+                elem.clear(); continue
+
+            title = (title_el.text or '').strip()
+            page_id = (id_el.text or '').strip() if id_el is not None else ''
+            raw_text = text_el.text or ''
+            ext_id = f"wiki_{page_id}"
+
+            if raw_text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
+                elem.clear(); continue
+
+            text = _normalize(_clean_wiki(raw_text))
+            if len(text.split()) < 50:
+                elem.clear(); continue
+
+            parsed += 1
+
+            if update_only and ext_id in known:
+                elem.clear(); continue
+
+            url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}"
+
+            # PostgreSQL
+            doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
+            db.add(doc)
+            db.flush()
+
+            # Fingerprints
+            fp = winnowing.fingerprint(text)
+            for pos, h in enumerate(list(fp)[:500]):
+                db.add(Fingerprint(doc_id=doc.id, hash=format(h, '016x'), position=pos))
+
+            # MinHash
+            try:
+                minhash.add_document(str(doc.id), text)
+            except ValueError:
+                pass
+
+            # Elasticsearch
+            fulltext.index_document(str(doc.id), title, text, source, lang, url)
+
+            embed_batch.append((str(doc.id), text))
+            if len(embed_batch) >= BATCH:
+                emb.batch_add(embed_batch, batch_size=BATCH)
+                embed_batch.clear()
+
+            count += 1
+            if count % 500 == 0:
+                db.commit()
+                pct = 40 + min(int(count / 100000 * 55), 55)  # 40-95%
+                _pub(f"Проиндексировано: {count} документов", progress=pct, docs_done=count)
+
+            elem.clear()
+
+    if embed_batch:
+        emb.batch_add(embed_batch, batch_size=BATCH)
+
+    db.commit()
+    db.close()
+    emb.save_index()
+
+    return count
+
+
+# ── Celery tasks ───────────────────────────────────────────────────────────
+
+def _run_indexing(source_key: str, update_only: bool):
+    src = WIKIPEDIA_SOURCES[source_key]
+    label = src["label"]
+    lang = src["lang"]
+
+    _set_status({
+        "status": "running",
+        "source": source_key,
+        "label": label,
+        "progress": 0,
+        "message": "Запуск…",
+        "started_at": datetime.now(timezone.utc).isoformat(),
+        "docs_done": 0,
+    })
+
+    data_dir = Path(settings.RAW_DATA_DIR) / source_key
+    bz2_path = data_dir / "dump.xml.bz2"
+
+    # --- Download if needed ---
+    if not bz2_path.exists() or not update_only:
+        _pub(f"Скачивание дампа {label}…", progress=1)
+        try:
+            _download_file(src["url"], bz2_path, label)
+        except Exception as e:
+            _set_status({"status": "error", "message": f"Ошибка скачивания: {e}"})
+            return
+
+    _pub("Индексация…", progress=41)
+
+    try:
+        count = _index_wikipedia_bz2(bz2_path, source_key, lang, update_only)
+    except Exception as e:
+        _set_status({"status": "error", "message": f"Ошибка индексации: {e}"})
+        return
+
+    action = "обновлено" if update_only else "проиндексировано"
+    _set_status({
+        "status": "done",
+        "source": source_key,
+        "label": label,
+        "progress": 100,
+        "message": f"Готово — {action} {count} документов",
+        "finished_at": datetime.now(timezone.utc).isoformat(),
+        "docs_done": count,
+    })
+    _r().publish("index_job_progress", _r().get(JOB_KEY))
+
+
+@celery_app.task(name="tasks.download_and_index", bind=True)
+def download_and_index(self, source_key: str):
+    _run_indexing(source_key, update_only=False)
+
+
+@celery_app.task(name="tasks.update_index", bind=True)
+def update_index(self, source_key: str):
+    _run_indexing(source_key, update_only=True)
+
+
+def get_job_status() -> dict:
+    return _get_status()

backend/scripts/index_documents.py

@@ -0,0 +1,119 @@
+"""
+Index pre-parsed documents (JSON files from parse_pdf.py / parse_wikipedia.py)
+into PostgreSQL, Elasticsearch, FAISS, and MinHash LSH.
+
+Usage:
+    python scripts/index_documents.py <dir_with_json_files> [--limit N]
+"""
+import sys
+import os
+import json
+import argparse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from app.config import settings
+from app.models.db import SessionLocal, Document, Fingerprint, engine, Base
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+
+
+Base.metadata.create_all(bind=engine)
+
+
+def load_json_docs(directory: str, limit: int | None = None):
+    docs = []
+    for path in sorted(Path(directory).glob("*.json")):
+        try:
+            with open(path, encoding="utf-8") as f:
+                doc = json.load(f)
+            if doc.get("text") and doc.get("title"):
+                docs.append(doc)
+        except Exception as e:
+            print(f"Skip {path.name}: {e}")
+        if limit and len(docs) >= limit:
+            break
+    return docs
+
+
+def index_batch(docs: list, batch_size: int = 32):
+    db = SessionLocal()
+    winnowing = WinNowing()
+    minhash_lsh = MinHashLSHIndex()
+    embeddings = EmbeddingIndex()
+    fulltext = FulltextSearch()
+
+    print(f"Indexing {len(docs)} documents …")
+
+    embed_batch: list[tuple[str, str]] = []
+
+    for i, doc in enumerate(docs):
+        source = doc.get("source", "unknown")
+        ext_id = doc.get("id", "")
+        title = doc["title"]
+        text = doc["text"]
+        url = doc.get("url", "")
+        lang = doc.get("lang", "ru")
+
+        # 1. PostgreSQL
+        db_doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
+        db.add(db_doc)
+        db.flush()
+
+        # 2. Fingerprints → PostgreSQL
+        fp = winnowing.fingerprint(text)
+        for pos, h in enumerate(list(fp)[:500]):
+            db.add(Fingerprint(doc_id=db_doc.id, hash=format(h, '016x'), position=pos))
+
+        # 3. MinHash LSH (in-memory; call save separately if needed)
+        try:
+            minhash_lsh.add_document(str(db_doc.id), text)
+        except ValueError:
+            pass
+
+        # 4. Elasticsearch
+        fulltext.index_document(str(db_doc.id), title, text, source, lang, url)
+
+        # 5. Accumulate for batch embedding
+        embed_batch.append((str(db_doc.id), text))
+
+        if len(embed_batch) >= batch_size:
+            embeddings.batch_add(embed_batch, batch_size=batch_size)
+            embed_batch.clear()
+
+        if (i + 1) % 100 == 0:
+            db.commit()
+            print(f"  {i + 1}/{len(docs)}")
+
+    # Flush remaining embeddings
+    if embed_batch:
+        embeddings.batch_add(embed_batch, batch_size=batch_size)
+
+    db.commit()
+    db.close()
+
+    embeddings.save_index()
+    print("Done. FAISS index saved.")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", help="Directory with parsed JSON files")
+    parser.add_argument("--limit", type=int, default=None, help="Max documents to index")
+    parser.add_argument("--batch-size", type=int, default=32)
+    args = parser.parse_args()
+
+    docs = load_json_docs(args.directory, args.limit)
+    print(f"Loaded {len(docs)} documents from {args.directory}")
+
+    if not docs:
+        print("No documents found, exiting.")
+        sys.exit(0)
+
+    index_batch(docs, batch_size=args.batch_size)
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/normalize_text.py

@@ -0,0 +1,83 @@
+"""
+Normalize extracted text before indexing.
+
+Usage:
+    python scripts/normalize_text.py <input_dir> <output_dir>
+
+Reads JSON files produced by parse_wikipedia.py / parse_pdf.py,
+cleans the text field, writes cleaned files to output_dir.
+"""
+import sys
+import re
+import json
+import argparse
+from pathlib import Path
+
+
+def normalize(text: str) -> str:
+    # Collapse unicode whitespace and control characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
+
+    # Normalize various dash/hyphen variants to a regular hyphen
+    text = re.sub(r'[—–‒―]', '-', text)
+
+    # Remove repeated punctuation (e.g. "!!!" → "!")
+    text = re.sub(r'([!?.]){2,}', r'\1', text)
+
+    # Collapse multiple spaces / tabs to a single space
+    text = re.sub(r'[ \t]+', ' ', text)
+
+    # Collapse more than two consecutive newlines
+    text = re.sub(r'\n{3,}', '\n\n', text)
+
+    # Strip leading/trailing whitespace per line
+    lines = [line.strip() for line in text.splitlines()]
+    text = '\n'.join(lines)
+
+    return text.strip()
+
+
+def process_directory(input_dir: str, output_dir: str) -> int:
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    count = 0
+    for src in sorted(Path(input_dir).glob('*.json')):
+        try:
+            with open(src, encoding='utf-8') as f:
+                doc = json.load(f)
+        except Exception as e:
+            print(f'Skip {src.name}: {e}')
+            continue
+
+        if 'text' not in doc:
+            continue
+
+        doc['text'] = normalize(doc['text'])
+
+        if len(doc['text'].split()) < 30:
+            continue
+
+        dest = out_path / src.name
+        with open(dest, 'w', encoding='utf-8') as f:
+            json.dump(doc, f, ensure_ascii=False, indent=2)
+
+        count += 1
+        if count % 1000 == 0:
+            print(f'Normalized {count} documents')
+
+    return count
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_dir')
+    parser.add_argument('output_dir')
+    args = parser.parse_args()
+
+    count = process_directory(args.input_dir, args.output_dir)
+    print(f'Done: {count} documents written to {args.output_dir}')
+
+
+if __name__ == '__main__':
+    main()

backend/scripts/parse_pdf.py

@@ -0,0 +1,90 @@
+import os
+import json
+import pymupdf
+from pathlib import Path
+from typing import Dict, Any, List
+
+def extract_text_from_pdf(pdf_path: str) -> Dict[str, Any]:
+    """Extract text and metadata from PDF"""
+    try:
+        doc = pymupdf.open(pdf_path)
+        
+        text = ""
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text += page.get_text()
+        
+        metadata = doc.metadata
+        doc.close()
+        
+        # Basic cleaning
+        text = clean_text(text)
+        
+        return {
+            "title": metadata.get('title', Path(pdf_path).stem),
+            "text": text,
+            "author": metadata.get('author', ''),
+            "subject": metadata.get('subject', ''),
+            "num_pages": len(doc)
+        }
+    except Exception as e:
+        print(f"Error processing {pdf_path}: {e}")
+        return None
+
+def process_pdf_directory(input_dir: str, output_dir: str) -> int:
+    """Process all PDFs in a directory"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    for pdf_file in Path(input_dir).glob("*.pdf"):
+        result = extract_text_from_pdf(str(pdf_file))
+        
+        if result and result.get('text'):
+            output_file = os.path.join(output_dir, f"{pdf_file.stem}.json")
+            
+            doc = {
+                "id": f"pdf_{pdf_file.stem}",
+                "title": result['title'],
+                "text": result['text'],
+                "source": "pdf",
+                "metadata": {
+                    "author": result.get('author'),
+                    "subject": result.get('subject'),
+                    "pages": result.get('num_pages')
+                }
+            }
+            
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(doc, f, ensure_ascii=False, indent=2)
+            
+            count += 1
+            if count % 100 == 0:
+                print(f"Processed {count} PDFs")
+    
+    return count
+
+def clean_text(text: str) -> str:
+    """Clean extracted PDF text"""
+    import re
+    
+    # Remove multiple spaces
+    text = re.sub(r'  +', ' ', text)
+    # Remove multiple newlines
+    text = re.sub(r'\n\n+', '\n', text)
+    # Remove special characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_pdf.py <input_dir> <output_dir>")
+        sys.exit(1)
+    
+    input_dir = sys.argv[1]
+    output_dir = sys.argv[2]
+    
+    count = process_pdf_directory(input_dir, output_dir)
+    print(f"Successfully processed {count} PDFs")

backend/scripts/parse_wikipedia.py

@@ -0,0 +1,110 @@
+import bz2
+import xml.etree.ElementTree as ET
+import json
+import os
+from pathlib import Path
+from typing import Generator, Dict, Any
+
+def extract_text_from_wikipedia_dump(bz2_path: str, output_dir: str, limit: int = None) -> int:
+    """Extract text from Wikipedia XML dump"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    with bz2.BZ2File(bz2_path, 'r') as f:
+        context = ET.iterparse(f, events=('start', 'end'))
+        context = iter(context)
+        
+        event, root = next(context)
+        
+        for event, elem in context:
+            if event == 'end' and elem.tag.endswith('page'):
+                doc = parse_page(elem)
+                if doc and doc.get('text'):
+                    filename = f"{output_dir}/wiki_{doc['id']}.json"
+                    with open(filename, 'w', encoding='utf-8') as out:
+                        json.dump(doc, out, ensure_ascii=False)
+                    
+                    count += 1
+                    if count % 1000 == 0:
+                        print(f"Processed {count} documents")
+                    
+                    if limit and count >= limit:
+                        break
+                
+                root.clear()
+    
+    return count
+
+def parse_page(page_elem) -> Dict[str, Any]:
+    """Parse a single Wikipedia page"""
+    ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
+    
+    title_elem = page_elem.find('wiki:title', ns) or page_elem.find('.//title')
+    page_id_elem = page_elem.find('wiki:id', ns) or page_elem.find('.//id')
+    revision_elem = page_elem.find('.//revision')
+    
+    if revision_elem is None:
+        return None
+    
+    text_elem = revision_elem.find('wiki:text', ns) or revision_elem.find('.//text')
+    
+    if title_elem is None or text_elem is None:
+        return None
+    
+    title = title_elem.text or ""
+    page_id = page_id_elem.text if page_id_elem is not None else ""
+    text = text_elem.text or ""
+    
+    # Skip redirects and special pages
+    if text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
+        return None
+    
+    # Basic text cleaning
+    text = clean_wikipedia_markup(text)
+    
+    if len(text.split()) < 50:
+        return None
+    
+    return {
+        "id": f"wiki_{page_id}",
+        "title": title,
+        "text": text,
+        "source": "wikipedia",
+        "lang": "ru"  # Change for English
+    }
+
+def clean_wikipedia_markup(text: str) -> str:
+    """Remove Wikipedia markup"""
+    import re
+    
+    # Remove wiki links
+    text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
+    # Remove external links
+    text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
+    # Remove templates
+    text = re.sub(r'\{\{[^}]*\}\}', '', text)
+    # Remove categories
+    text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
+    # Remove refs
+    text = re.sub(r'<ref[^>]*>[^<]*</ref>', '', text)
+    # Remove HTML tags
+    text = re.sub(r'<[^>]*>', '', text)
+    # Clean whitespace
+    text = re.sub(r'\n\n+', '\n', text)
+    text = re.sub(r' +', ' ', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_wikipedia.py <input_bz2> <output_dir> [limit]")
+        sys.exit(1)
+    
+    input_file = sys.argv[1]
+    output_dir = sys.argv[2]
+    limit = int(sys.argv[3]) if len(sys.argv) > 3 else None
+    
+    count = extract_text_from_wikipedia_dump(input_file, output_dir, limit)
+    print(f"Successfully processed {count} documents")

docker-compose.yml

@@ -0,0 +1,141 @@
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: antiplagiator
+      POSTGRES_USER: antiplagiator
+      POSTGRES_PASSWORD: antiplagiator_dev
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U antiplagiator"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  elasticsearch:
+    image: elasticsearch:8.13.0
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    ports:
+      - "9200:9200"
+    volumes:
+      - elasticsearch_data:/usr/share/elasticsearch/data
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:9200 | grep -q cluster_name"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  minio:
+    image: minio/minio
+    command: server /data --console-address ":9001"
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - minio_data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  backend:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile
+    ports:
+      - "8000:8000"
+    environment:
+      DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator
+      ELASTICSEARCH_HOST: elasticsearch
+      ELASTICSEARCH_PORT: 9200
+      REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MINIO_ENDPOINT: minio:9000
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+      MINIO_BUCKET: antiplagiator
+      MINIO_SECURE: "false"
+    volumes:
+      - backend_data:/app/data
+    depends_on:
+      postgres:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    restart: unless-stopped
+
+  frontend:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+    ports:
+      - "80:80"
+    depends_on:
+      - backend
+    restart: unless-stopped
+
+  celery-worker:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile
+    command: celery -A app.celery_app worker --loglevel=info --concurrency=2
+    environment:
+      DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator
+      ELASTICSEARCH_HOST: elasticsearch
+      ELASTICSEARCH_PORT: 9200
+      REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+      MINIO_ENDPOINT: minio:9000
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+      MINIO_BUCKET: antiplagiator
+      MINIO_SECURE: "false"
+    volumes:
+      - backend_data:/app/data
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  postgres_data:
+  elasticsearch_data:
+  redis_data:
+  minio_data:
+  backend_data:
+
+networks:
+  default:
+    name: antiplagiator_network

frontend/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:20-alpine AS builder
+
+WORKDIR /app
+COPY package.json ./
+RUN npm install
+
+COPY . .
+RUN npm run build
+
+FROM nginx:alpine
+COPY --from=builder /app/dist /usr/share/nginx/html
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+EXPOSE 80

frontend/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="ru">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Антиплагиат</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

frontend/nginx.conf

@@ -0,0 +1,52 @@
+server {
+    listen 80;
+    server_name _;
+
+    root /usr/share/nginx/html;
+    index index.html;
+
+    # Gzip
+    gzip on;
+    gzip_types text/plain text/css application/javascript application/json image/svg+xml;
+    gzip_min_length 1024;
+    gzip_vary on;
+
+    # Cache static assets (JS/CSS с хешами в имени — кешируем надолго)
+    location ~* \.(js|css|woff2?|ttf|svg|png|jpg|ico)$ {
+        expires 1y;
+        add_header Cache-Control "public, immutable";
+        try_files $uri =404;
+    }
+
+    # React SPA — все неизвестные пути → index.html (не кешируем)
+    location / {
+        add_header Cache-Control "no-store";
+        try_files $uri $uri/ /index.html;
+    }
+
+    # Proxy REST API → FastAPI
+    location /api/ {
+        proxy_pass http://backend:8000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_read_timeout 120s;
+        proxy_send_timeout 120s;
+    }
+
+    # Proxy WebSocket → FastAPI (/ws/ и /api/admin/ws/)
+    location ~ ^/(ws/|api/admin/ws/) {
+        proxy_pass http://backend:8000;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_read_timeout 3600s;
+    }
+
+    # MinIO console
+    location /minio/ {
+        proxy_pass http://minio:9001/;
+        proxy_set_header Host $host;
+    }
+}

frontend/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "antiplagiator-frontend",
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview",
+    "lint": "eslint . --ext .ts,.tsx"
+  },
+  "dependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-dropzone": "^14.2.3",
+    "zustand": "^4.4.0",
+    "axios": "^1.6.0",
+    "react-query": "^3.39.3",
+    "tailwindcss": "^3.3.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.0",
+    "@types/react-dom": "^18.2.0",
+    "@vitejs/plugin-react": "^4.2.0",
+    "vite": "^5.0.0",
+    "typescript": "^5.3.0",
+    "autoprefixer": "^10.4.16",
+    "postcss": "^8.4.31"
+  }
+}

frontend/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

frontend/src/App.tsx

@@ -0,0 +1,98 @@
+import { useEffect, useState } from 'react'
+import AuthPage from './pages/AuthPage'
+import Home from './pages/Home'
+import Report from './pages/Report'
+import HistoryPage from './pages/HistoryPage'
+import AdminPage from './pages/AdminPage'
+import { getMe } from './api/client'
+
+type Screen = 'home' | 'report' | 'history' | 'admin'
+
+export default function App() {
+  const [authed, setAuthed] = useState<boolean | null>(null) // null = checking
+  const [screen, setScreen] = useState<Screen>('home')
+  const [taskId, setTaskId] = useState<string | null>(null)
+
+  // Check stored token on mount
+  useEffect(() => {
+    const token = localStorage.getItem('token')
+    if (!token) { setAuthed(false); return }
+    getMe()
+      .then(() => setAuthed(true))
+      .catch(() => { localStorage.removeItem('token'); setAuthed(false) })
+  }, [])
+
+  const handleAuth = (token: string) => {
+    if (token) localStorage.setItem('token', token)
+    setAuthed(true)
+  }
+
+  const handleLogout = () => {
+    localStorage.removeItem('token')
+    setAuthed(false)
+    setScreen('home')
+    setTaskId(null)
+  }
+
+  const openReport = (id: string) => {
+    setTaskId(id)
+    setScreen('report')
+  }
+
+  // Loading state while checking token
+  if (authed === null) {
+    return (
+      <div className="min-h-screen bg-gray-50 flex items-center justify-center text-gray-400">
+        Загрузка…
+      </div>
+    )
+  }
+
+  if (!authed) {
+    return <AuthPage onAuth={handleAuth} />
+  }
+
+  return (
+    <div className="min-h-screen flex flex-col">
+      {/* Top nav — only on home/history */}
+      {screen !== 'report' && (
+        <nav className="bg-white border-b px-6 py-3 flex items-center justify-between">
+          <div className="flex gap-4 text-sm font-medium">
+            {([['home', 'Проверка'], ['history', 'История'], ['admin', 'База']] as const).map(([s, label]) => (
+              <button
+                key={s}
+                onClick={() => setScreen(s)}
+                className={screen === s ? 'text-blue-600' : 'text-gray-500 hover:text-gray-800'}
+              >
+                {label}
+              </button>
+            ))}
+          </div>
+          <button onClick={handleLogout} className="text-xs text-gray-400 hover:text-gray-700">
+            Выйти
+          </button>
+        </nav>
+      )}
+
+      {screen === 'home' && (
+        <Home onTaskCreated={openReport} />
+      )}
+
+      {screen === 'history' && (
+        <HistoryPage
+          onOpen={openReport}
+          onNewCheck={() => setScreen('home')}
+        />
+      )}
+
+      {screen === 'admin' && <AdminPage />}
+
+      {screen === 'report' && taskId && (
+        <Report
+          taskId={taskId}
+          onBack={() => setScreen('home')}
+        />
+      )}
+    </div>
+  )
+}

frontend/src/api/client.ts

@@ -0,0 +1,160 @@
+import axios from 'axios'
+
+const api = axios.create({ baseURL: '/api' })
+
+// Attach token from localStorage on every request
+api.interceptors.request.use((config) => {
+  const token = localStorage.getItem('token')
+  if (token) config.headers.Authorization = `Bearer ${token}`
+  return config
+})
+
+export interface CheckResponse {
+  task_id: string
+  status: string
+  message: string
+}
+
+export interface MatchResult {
+  method: string
+  similarity: number
+  source_title?: string
+  source_db?: string
+  url?: string
+  source_id?: string
+}
+
+export interface ReportResponse {
+  task_id: string
+  status: string
+  overall_similarity?: number
+  matches?: MatchResult[]
+  source_text?: string
+  created_at?: string
+  completed_at?: string
+}
+
+export interface ProgressEvent {
+  stage: 'fingerprint' | 'fuzzy' | 'semantic' | 'saving' | 'done' | 'failed'
+  progress: number
+}
+
+export function openProgressSocket(taskId: string, onEvent: (e: ProgressEvent) => void): () => void {
+  const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'
+  const ws = new WebSocket(`${protocol}://${window.location.host}/ws/progress/${taskId}`)
+  ws.onmessage = (e) => {
+    try { onEvent(JSON.parse(e.data)) } catch {}
+  }
+  return () => ws.close()
+}
+
+export interface ReportSummary {
+  task_id: string
+  status: string
+  overall_similarity?: number
+  created_at?: string
+}
+
+export const uploadDocument = async (file: File): Promise<CheckResponse> => {
+  const form = new FormData()
+  form.append('file', file)
+  const res = await api.post<CheckResponse>('/check', form)
+  return res.data
+}
+
+export const getReport = async (taskId: string): Promise<ReportResponse> => {
+  const res = await api.get<ReportResponse>(`/report/${taskId}`)
+  return res.data
+}
+
+export const listReports = async (): Promise<ReportSummary[]> => {
+  const res = await api.get<{ tasks: ReportSummary[] }>('/reports')
+  return res.data.tasks
+}
+
+export interface UserInfo {
+  id: string
+  email: string
+  created_at: string
+}
+
+export const register = async (email: string, password: string): Promise<void> => {
+  await api.post('/auth/register', { email, password })
+}
+
+export const login = async (email: string, password: string): Promise<string> => {
+  const form = new URLSearchParams({ username: email, password })
+  const res = await api.post<{ access_token: string }>('/auth/login', form, {
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+  })
+  return res.data.access_token
+}
+
+export const getMe = async (): Promise<UserInfo> => {
+  const res = await api.get<UserInfo>('/auth/me')
+  return res.data
+}
+
+// ── Admin / Index ──────────────────────────────────────────────────────────
+
+export interface IndexSource {
+  key: string
+  label: string
+}
+
+export interface JobStatus {
+  status: string
+  source?: string
+  label?: string
+  progress: number
+  message: string
+  started_at?: string
+  finished_at?: string
+  docs_done: number
+}
+
+export const getIndexSources = async (): Promise<IndexSource[]> => {
+  const res = await api.get<IndexSource[]>('/admin/index/sources')
+  return res.data
+}
+
+export const startDownload = async (sourceKey: string): Promise<JobStatus> => {
+  const res = await api.post<JobStatus>(`/admin/index/download/${sourceKey}`)
+  return res.data
+}
+
+export const startUpdate = async (sourceKey: string): Promise<JobStatus> => {
+  const res = await api.post<JobStatus>(`/admin/index/update/${sourceKey}`)
+  return res.data
+}
+
+export const getJobStatus = async (): Promise<JobStatus> => {
+  const res = await api.get<JobStatus>('/admin/index/status')
+  return res.data
+}
+
+export function openIndexSocket(onEvent: (e: JobStatus) => void): () => void {
+  const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'
+  const ws = new WebSocket(`${protocol}://${window.location.host}/api/admin/ws/index-progress`)
+  ws.onmessage = (e) => {
+    try { onEvent(JSON.parse(e.data)) } catch {}
+  }
+  return () => ws.close()
+}
+
+export const downloadReportPdf = async (taskId: string): Promise<void> => {
+  const token = localStorage.getItem('token')
+  const res = await fetch(`/api/report/${taskId}/pdf`, {
+    headers: token ? { Authorization: `Bearer ${token}` } : {},
+  })
+  if (!res.ok) throw new Error('PDF download failed')
+  const blob = await res.blob()
+  const url = URL.createObjectURL(blob)
+  const a = document.createElement('a')
+  a.href = url
+  a.download = `report-${taskId.slice(0, 8)}.pdf`
+  document.body.appendChild(a)
+  a.click()
+  document.body.removeChild(a)
+  URL.revokeObjectURL(url)
+}

frontend/src/components/DropZone.tsx

@@ -0,0 +1,56 @@
+import { useCallback, useState } from 'react'
+import { useDropzone } from 'react-dropzone'
+
+interface Props {
+  onFile: (file: File) => void
+  disabled?: boolean
+}
+
+const ACCEPTED = { 'text/plain': ['.txt'], 'application/pdf': ['.pdf'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'] }
+
+export default function DropZone({ onFile, disabled }: Props) {
+  const [error, setError] = useState<string | null>(null)
+
+  const onDrop = useCallback((accepted: File[], rejected: { errors: { message: string }[] }[]) => {
+    setError(null)
+    if (rejected.length) {
+      setError('Поддерживаются только .txt, .pdf, .docx')
+      return
+    }
+    if (accepted[0]) onFile(accepted[0])
+  }, [onFile])
+
+  const { getRootProps, getInputProps, isDragActive, acceptedFiles } = useDropzone({
+    onDrop,
+    accept: ACCEPTED,
+    maxFiles: 1,
+    disabled,
+  })
+
+  const file = acceptedFiles[0]
+
+  return (
+    <div className="flex flex-col items-center gap-3">
+      <div
+        {...getRootProps()}
+        className={[
+          'w-full max-w-lg border-2 border-dashed rounded-xl p-10 text-center cursor-pointer transition-colors',
+          isDragActive ? 'border-blue-500 bg-blue-50' : 'border-gray-300 hover:border-blue-400',
+          disabled ? 'opacity-50 cursor-not-allowed' : '',
+        ].join(' ')}
+      >
+        <input {...getInputProps()} />
+        <p className="text-gray-500 text-sm">
+          {isDragActive
+            ? 'Отпустите файл здесь…'
+            : 'Перетащите файл или нажмите для выбора'}
+        </p>
+        <p className="text-xs text-gray-400 mt-1">.txt, .pdf, .docx</p>
+        {file && (
+          <p className="mt-3 text-sm font-medium text-blue-700">{file.name}</p>
+        )}
+      </div>
+      {error && <p className="text-red-500 text-sm">{error}</p>}
+    </div>
+  )
+}

frontend/src/components/HighlightedText.tsx

@@ -0,0 +1,109 @@
+import { useEffect, useRef } from 'react'
+import { MatchResult } from '../api/client'
+import { HIGHLIGHT_COLOR } from './MatchCard'
+
+interface Span {
+  start: number
+  end: number
+  method: string
+  matchIndex: number
+}
+
+interface Props {
+  text: string
+  matches: MatchResult[]
+  activeIndex: number | null
+  onMatchClick: (index: number) => void
+}
+
+function buildSegments(text: string, spans: Span[]) {
+  // Sort by start, then merge/flatten overlapping spans keeping the one with highest priority
+  const priority: Record<string, number> = { exact: 3, fuzzy: 2, semantic: 1 }
+
+  // Build a char-level annotation array (sparse)
+  const events: Array<{ pos: number; type: 'open' | 'close'; span: Span }> = []
+  for (const s of spans) {
+    events.push({ pos: s.start, type: 'open', span: s })
+    events.push({ pos: s.end, type: 'close', span: s })
+  }
+  events.sort((a, b) => a.pos - b.pos || (a.type === 'close' ? -1 : 1))
+
+  const segments: Array<{ text: string; span: Span | null }> = []
+  let cursor = 0
+  const active = new Map<Span, true>()
+
+  const dominant = () => {
+    let best: Span | null = null
+    for (const s of active.keys()) {
+      if (!best || (priority[s.method] ?? 0) > (priority[best.method] ?? 0)) best = s
+    }
+    return best
+  }
+
+  for (const ev of events) {
+    if (ev.pos > cursor) {
+      segments.push({ text: text.slice(cursor, ev.pos), span: dominant() })
+      cursor = ev.pos
+    }
+    if (ev.type === 'open') active.set(ev.span, true)
+    else active.delete(ev.span)
+  }
+
+  if (cursor < text.length) {
+    segments.push({ text: text.slice(cursor), span: null })
+  }
+
+  return segments
+}
+
+export default function HighlightedText({ text, matches, activeIndex, onMatchClick }: Props) {
+  const spanRefs = useRef<Map<number, HTMLElement>>(new Map())
+
+  // Scroll to active match
+  useEffect(() => {
+    if (activeIndex === null) return
+    const el = spanRefs.current.get(activeIndex)
+    el?.scrollIntoView({ behavior: 'smooth', block: 'center' })
+  }, [activeIndex])
+
+  const spans: Span[] = matches
+    .map((m, i) => ({
+      start: m.fragment_start ?? 0,
+      end: m.fragment_end ?? 0,
+      method: m.method,
+      matchIndex: i,
+    }))
+    .filter((s) => s.end > s.start)
+
+  const segments = buildSegments(text, spans)
+
+  return (
+    <div className="font-mono text-sm leading-7 whitespace-pre-wrap break-words">
+      {segments.map((seg, i) => {
+        if (!seg.span) {
+          return <span key={i}>{seg.text}</span>
+        }
+
+        const mi = seg.span.matchIndex
+        const isActive = mi === activeIndex
+        const color = HIGHLIGHT_COLOR[seg.span.method] ?? 'bg-gray-200'
+
+        return (
+          <mark
+            key={i}
+            ref={(el) => { if (el) spanRefs.current.set(mi, el) }}
+            onClick={() => onMatchClick(mi)}
+            className={[
+              color,
+              'cursor-pointer rounded px-0.5 transition-all',
+              isActive ? 'ring-2 ring-blue-500 brightness-90' : 'hover:brightness-90',
+            ].join(' ')}
+            title={`${matches[mi]?.source_title ?? ''} — ${matches[mi]?.similarity.toFixed(1)}%`}
+          >
+            {seg.text}
+          </mark>
+        )
+      })}
+    </div>
+  )
+}

frontend/src/components/MatchCard.tsx

@@ -0,0 +1,76 @@
+import { MatchResult } from '../api/client'
+
+const METHOD_LABEL: Record<string, string> = {
+  exact: 'Точное',
+  fuzzy: 'Нечёткое',
+  semantic: 'Семантика',
+}
+
+const METHOD_COLOR: Record<string, string> = {
+  exact: 'bg-red-100 text-red-700 border-red-200',
+  fuzzy: 'bg-orange-100 text-orange-700 border-orange-200',
+  semantic: 'bg-yellow-100 text-yellow-700 border-yellow-200',
+}
+
+const HIGHLIGHT_COLOR: Record<string, string> = {
+  exact: 'bg-red-200',
+  fuzzy: 'bg-orange-200',
+  semantic: 'bg-yellow-200',
+}
+
+interface Props {
+  match: MatchResult
+  index: number
+  active: boolean
+  onClick: () => void
+}
+
+export { HIGHLIGHT_COLOR }
+
+export default function MatchCard({ match, index, active, onClick }: Props) {
+  const colorClass = METHOD_COLOR[match.method] ?? 'bg-gray-100 text-gray-700 border-gray-200'
+
+  return (
+    <button
+      onClick={onClick}
+      className={[
+        'w-full text-left border rounded-xl p-3 transition-all',
+        active ? 'ring-2 ring-blue-400 shadow-md' : 'hover:shadow-sm',
+        colorClass,
+      ].join(' ')}
+    >
+      <div className="flex items-center justify-between gap-2 mb-1">
+        <span className="text-xs font-semibold uppercase tracking-wide">
+          {METHOD_LABEL[match.method] ?? match.method}
+        </span>
+        <span className="text-sm font-bold">{match.similarity.toFixed(1)}%</span>
+      </div>
+
+      <p className="text-sm font-medium truncate">
+        {match.source_title ?? match.source_id ?? '—'}
+      </p>
+
+      {match.source_db && (
+        <p className="text-xs opacity-70">{match.source_db}</p>
+      )}
+
+      {match.fragment_text && (
+        <p className="mt-1 text-xs italic opacity-80 line-clamp-2">
+          «{match.fragment_text}»
+        </p>
+      )}
+
+      {match.url && (
+        <a
+          href={match.url}
+          target="_blank"
+          rel="noopener noreferrer"
+          onClick={(e) => e.stopPropagation()}
+          className="mt-1 text-xs underline opacity-60 block truncate"
+        >
+          {match.url}
+        </a>
+      )}
+    </button>
+  )
+}

frontend/src/components/SimilarityMeter.tsx

@@ -0,0 +1,33 @@
+interface Props {
+  value: number
+}
+
+function color(v: number) {
+  if (v < 20) return 'bg-green-500'
+  if (v < 50) return 'bg-yellow-400'
+  return 'bg-red-500'
+}
+
+function label(v: number) {
+  if (v < 20) return 'Оригинально'
+  if (v < 50) return 'Умеренное сходство'
+  return 'Высокое заимствование'
+}
+
+export default function SimilarityMeter({ value }: Props) {
+  const pct = Math.min(Math.max(value, 0), 100)
+  return (
+    <div className="flex flex-col gap-2">
+      <div className="flex justify-between text-sm font-medium">
+        <span>{label(pct)}</span>
+        <span className="text-gray-700">{pct.toFixed(1)}%</span>
+      </div>
+      <div className="w-full h-3 rounded-full bg-gray-200 overflow-hidden">
+        <div
+          className={`h-full rounded-full transition-all ${color(pct)}`}
+          style={{ width: `${pct}%` }}
+        />
+      </div>
+    </div>
+  )
+}

frontend/src/index.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/src/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+)

frontend/src/pages/AdminPage.tsx

@@ -0,0 +1,172 @@
+import { useEffect, useState } from 'react'
+import {
+  getIndexSources, startDownload, startUpdate,
+  getJobStatus, openIndexSocket,
+  IndexSource, JobStatus,
+} from '../api/client'
+
+const STATUS_LABEL: Record<string, string> = {
+  idle: 'Ожидание',
+  running: 'Выполняется',
+  done: 'Готово',
+  error: 'Ошибка',
+  started: 'Запуск…',
+}
+
+const STATUS_COLOR: Record<string, string> = {
+  idle: 'text-gray-400',
+  running: 'text-blue-600',
+  done: 'text-green-600',
+  error: 'text-red-600',
+  started: 'text-blue-500',
+}
+
+const SOURCE_SIZE: Record<string, string> = {
+  wikipedia_ru: '~8 GB',
+  wikipedia_en: '~22 GB',
+}
+
+const SOURCE_DESC: Record<string, string> = {
+  wikipedia_ru: 'Русскоязычные статьи Википедии',
+  wikipedia_en: 'Англоязычные статьи Википедии',
+}
+
+export default function AdminPage() {
+  const [sources, setSources] = useState<IndexSource[]>([])
+  const [job, setJob] = useState<JobStatus>({ status: 'idle', progress: 0, message: '', docs_done: 0 })
+  const [busy, setBusy] = useState(false)
+  const [error, setError] = useState<string | null>(null)
+
+  // Load sources + current status
+  useEffect(() => {
+    getIndexSources().then(setSources).catch(() => {})
+    getJobStatus().then(setJob).catch(() => {})
+  }, [])
+
+  // Subscribe to WebSocket progress
+  useEffect(() => {
+    const close = openIndexSocket((ev) => {
+      setJob(ev)
+      if (ev.status !== 'running') setBusy(false)
+    })
+    return close
+  }, [])
+
+  const isRunning = job.status === 'running' || busy
+
+  const handleDownload = async (key: string) => {
+    setError(null)
+    setBusy(true)
+    try {
+      const j = await startDownload(key)
+      setJob(j)
+    } catch (e: unknown) {
+      const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
+      setError(detail ?? 'Ошибка запуска')
+      setBusy(false)
+    }
+  }
+
+  const handleUpdate = async (key: string) => {
+    setError(null)
+    setBusy(true)
+    try {
+      const j = await startUpdate(key)
+      setJob(j)
+    } catch (e: unknown) {
+      const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
+      setError(detail ?? 'Ошибка запуска')
+      setBusy(false)
+    }
+  }
+
+  return (
+    <div className="min-h-screen bg-gray-50 p-6">
+      <div className="max-w-2xl mx-auto flex flex-col gap-6">
+        <h1 className="text-xl font-bold text-gray-800">Управление базой документов</h1>
+
+        {/* Current job status */}
+        <div className="bg-white rounded-2xl shadow-sm border border-gray-100 p-5 flex flex-col gap-3">
+          <div className="flex items-center justify-between">
+            <span className="text-sm font-semibold text-gray-700">Статус задания</span>
+            <span className={`text-sm font-medium ${STATUS_COLOR[job.status] ?? 'text-gray-500'}`}>
+              {STATUS_LABEL[job.status] ?? job.status}
+              {job.label ? ` — ${job.label}` : ''}
+            </span>
+          </div>
+
+          {job.message && (
+            <p className="text-sm text-gray-500">{job.message}</p>
+          )}
+
+          {isRunning && (
+            <div className="flex flex-col gap-1">
+              <div className="flex justify-between text-xs text-gray-400">
+                <span>{job.docs_done > 0 ? `Проиндексировано: ${job.docs_done.toLocaleString()} документов` : ''}</span>
+                <span>{job.progress}%</span>
+              </div>
+              <div className="w-full h-2.5 bg-gray-200 rounded-full overflow-hidden">
+                <div
+                  className="h-full bg-blue-500 rounded-full transition-all duration-700"
+                  style={{ width: `${job.progress}%` }}
+                />
+              </div>
+            </div>
+          )}
+
+          {job.status === 'done' && job.finished_at && (
+            <p className="text-xs text-gray-400">
+              Завершено: {new Date(job.finished_at).toLocaleString('ru-RU')}
+            </p>
+          )}
+        </div>
+
+        {error && (
+          <div className="bg-red-50 border border-red-200 rounded-xl p-3 text-red-600 text-sm">
+            {error}
+          </div>
+        )}
+
+        {/* Source cards */}
+        <div className="flex flex-col gap-3">
+          {sources.map((src) => (
+            <div
+              key={src.key}
+              className="bg-white rounded-2xl shadow-sm border border-gray-100 p-5 flex items-center gap-4"
+            >
+              <div className="flex-1">
+                <p className="font-semibold text-gray-800">{src.label}</p>
+                <p className="text-sm text-gray-500">{SOURCE_DESC[src.key] ?? ''}</p>
+                <p className="text-xs text-gray-400 mt-0.5">{SOURCE_SIZE[src.key] ?? ''}</p>
+              </div>
+
+              <div className="flex flex-col gap-2 shrink-0">
+                <button
+                  onClick={() => handleDownload(src.key)}
+                  disabled={isRunning}
+                  className="text-sm bg-blue-600 text-white px-4 py-2 rounded-xl hover:bg-blue-700 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
+                  title="Скачать дамп и проиндексировать с нуля"
+                >
+                  Скачать и индексировать
+                </button>
+                <button
+                  onClick={() => handleUpdate(src.key)}
+                  disabled={isRunning}
+                  className="text-sm border border-gray-200 px-4 py-2 rounded-xl hover:bg-gray-50 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
+                  title="Добавить только новые документы (дамп уже скачан)"
+                >
+                  Обновить базу
+                </button>
+              </div>
+            </div>
+          ))}
+        </div>
+
+        <div className="text-xs text-gray-400 bg-gray-100 rounded-xl p-4 flex flex-col gap-1">
+          <p><strong>Скачать и индексировать</strong> — скачивает свежий дамп (~часы) и строит индекс с нуля.</p>
+          <p><strong>Обновить базу</strong> — использует уже скачанный дамп, добавляет только документы, которых ещё нет в базе. Быстро после первой загрузки.</p>
+        </div>
+      </div>
+    </div>
+  )
+}

frontend/src/pages/AuthPage.tsx

@@ -0,0 +1,85 @@
+import { useState } from 'react'
+import { login, register } from '../api/client'
+
+interface Props {
+  onAuth: (token: string) => void
+}
+
+export default function AuthPage({ onAuth }: Props) {
+  const [mode, setMode] = useState<'login' | 'register'>('login')
+  const [email, setEmail] = useState('')
+  const [password, setPassword] = useState('')
+  const [error, setError] = useState<string | null>(null)
+  const [loading, setLoading] = useState(false)
+
+  const handle = async () => {
+    setError(null)
+    setLoading(true)
+    try {
+      if (mode === 'register') {
+        await register(email, password)
+      }
+      const token = await login(email, password)
+      localStorage.setItem('token', token)
+      onAuth(token)
+    } catch (e: unknown) {
+      const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail
+      setError(detail ?? 'Ошибка')
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <div className="min-h-screen bg-gray-50 flex items-center justify-center p-4">
+      <div className="bg-white rounded-2xl shadow-md p-8 w-full max-w-sm flex flex-col gap-5">
+        <h1 className="text-2xl font-bold text-center text-gray-800">Антиплагиат</h1>
+
+        <div className="flex rounded-xl overflow-hidden border text-sm font-medium">
+          {(['login', 'register'] as const).map((m) => (
+            <button
+              key={m}
+              onClick={() => { setMode(m); setError(null) }}
+              className={`flex-1 py-2 transition-colors ${mode === m ? 'bg-blue-600 text-white' : 'text-gray-500 hover:bg-gray-50'}`}
+            >
+              {m === 'login' ? 'Вход' : 'Регистрация'}
+            </button>
+          ))}
+        </div>
+
+        <input
+          type="email"
+          placeholder="Email"
+          value={email}
+          onChange={(e) => setEmail(e.target.value)}
+          className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400"
+        />
+        <input
+          type="password"
+          placeholder="Пароль"
+          value={password}
+          onChange={(e) => setPassword(e.target.value)}
+          onKeyDown={(e) => e.key === 'Enter' && handle()}
+          className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400"
+        />
+
+        {error && <p className="text-red-500 text-sm text-center">{error}</p>}
+
+        <button
+          onClick={handle}
+          disabled={loading || !email || !password}
+          className="py-2.5 rounded-xl bg-blue-600 text-white font-semibold hover:bg-blue-700 disabled:opacity-40 transition-colors"
+        >
+          {loading ? '…' : mode === 'login' ? 'Войти' : 'Зарегистрироваться'}
+        </button>
+
+        <button
+          onClick={() => onAuth('')}
+          className="text-xs text-gray-400 hover:text-gray-600 text-center"
+        >
+          Продолжить без входа
+        </button>
+      </div>
+    </div>
+  )
+}

frontend/src/pages/HistoryPage.tsx

@@ -0,0 +1,128 @@
+import { useEffect, useState } from 'react'
+import { listReports, downloadReportPdf, ReportSummary } from '../api/client'
+
+interface Props {
+  onOpen: (taskId: string) => void
+  onNewCheck: () => void
+}
+
+function statusBadge(status: string) {
+  const map: Record<string, string> = {
+    completed: 'bg-green-100 text-green-700',
+    processing: 'bg-blue-100 text-blue-700',
+    failed: 'bg-red-100 text-red-700',
+    pending: 'bg-gray-100 text-gray-500',
+  }
+  const label: Record<string, string> = {
+    completed: 'Готово',
+    processing: 'Анализ…',
+    failed: 'Ошибка',
+    pending: 'Ожидание',
+  }
+  return (
+    <span className={`text-xs font-medium px-2 py-0.5 rounded-full ${map[status] ?? 'bg-gray-100 text-gray-500'}`}>
+      {label[status] ?? status}
+    </span>
+  )
+}
+
+function similarityColor(v?: number | null) {
+  if (v == null) return 'text-gray-400'
+  if (v < 20) return 'text-green-600'
+  if (v < 50) return 'text-yellow-600'
+  return 'text-red-600'
+}
+
+export default function HistoryPage({ onOpen, onNewCheck }: Props) {
+  const [tasks, setTasks] = useState<ReportSummary[]>([])
+  const [loading, setLoading] = useState(true)
+  const [error, setError] = useState<string | null>(null)
+
+  useEffect(() => {
+    listReports()
+      .then(setTasks)
+      .catch(() => setError('Не удалось загрузить историю'))
+      .finally(() => setLoading(false))
+  }, [])
+
+  return (
+    <div className="min-h-screen bg-gray-50 flex flex-col">
+      <header className="bg-white border-b px-6 py-4 flex items-center justify-between">
+        <h1 className="font-semibold text-gray-800">История проверок</h1>
+        <button
+          onClick={onNewCheck}
+          className="text-sm bg-blue-600 text-white px-4 py-2 rounded-xl hover:bg-blue-700 transition-colors"
+        >
+          + Новая проверка
+        </button>
+      </header>
+
+      <div className="flex-1 p-6 max-w-3xl mx-auto w-full">
+        {loading && (
+          <div className="text-center text-gray-400 mt-12 animate-pulse">Загрузка…</div>
+        )}
+
+        {error && (
+          <div className="bg-red-50 border border-red-200 rounded-xl p-4 text-red-600 text-sm">
+            {error}
+          </div>
+        )}
+
+        {!loading && !error && tasks.length === 0 && (
+          <div className="text-center text-gray-400 mt-12">
+            <p className="text-lg">Проверок пока нет</p>
+            <button onClick={onNewCheck} className="mt-4 text-blue-600 hover:underline text-sm">
+              Загрузить первый документ
+            </button>
+          </div>
+        )}
+
+        {tasks.length > 0 && (
+          <ul className="flex flex-col gap-3">
+            {tasks.map((t) => (
+              <li
+                key={t.task_id}
+                className="bg-white rounded-2xl shadow-sm border border-gray-100 px-5 py-4 flex items-center gap-4"
+              >
+                {/* Similarity circle */}
+                <div className={`text-2xl font-bold w-16 text-center shrink-0 ${similarityColor(t.overall_similarity)}`}>
+                  {t.overall_similarity != null ? `${t.overall_similarity}%` : '—'}
+                </div>
+
+                {/* Info */}
+                <div className="flex-1 min-w-0">
+                  <div className="flex items-center gap-2 mb-1">
+                    {statusBadge(t.status)}
+                    <span className="text-xs text-gray-400">
+                      {t.created_at ? new Date(t.created_at).toLocaleString('ru-RU') : ''}
+                    </span>
+                  </div>
+                  <p className="text-xs text-gray-400 truncate font-mono">{t.task_id}</p>
+                </div>
+
+                {/* Actions */}
+                <div className="flex gap-2 shrink-0">
+                  {t.status === 'completed' && (
+                    <button
+                      onClick={() => { downloadReportPdf(t.task_id).catch(console.error) }}
+                      className="text-xs border border-gray-200 rounded-lg px-3 py-1.5 hover:bg-gray-50 transition-colors"
+                      title="Скачать PDF"
+                    >
+                      PDF
+                    </button>
+                  )}
+                  <button
+                    onClick={() => onOpen(t.task_id)}
+                    className="text-xs bg-blue-50 text-blue-700 border border-blue-200 rounded-lg px-3 py-1.5 hover:bg-blue-100 transition-colors"
+                  >
+                    Открыть
+                  </button>
+                </div>
+              </li>
+            ))}
+          </ul>
+        )}
+      </div>
+    </div>
+  )
+}

frontend/src/pages/Home.tsx

@@ -0,0 +1,53 @@
+import { useState } from 'react'
+import DropZone from '../components/DropZone'
+import { uploadDocument } from '../api/client'
+
+interface Props {
+  onTaskCreated: (taskId: string) => void
+}
+
+export default function Home({ onTaskCreated }: Props) {
+  const [file, setFile] = useState<File | null>(null)
+  const [loading, setLoading] = useState(false)
+  const [error, setError] = useState<string | null>(null)
+
+  const handleSubmit = async () => {
+    if (!file) return
+    setLoading(true)
+    setError(null)
+    try {
+      const res = await uploadDocument(file)
+      onTaskCreated(res.task_id)
+    } catch (e: unknown) {
+      const msg = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail ?? 'Ошибка загрузки'
+      setError(msg)
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <div className="min-h-screen bg-gray-50 flex flex-col items-center justify-center p-6">
+      <div className="bg-white rounded-2xl shadow-md p-8 w-full max-w-lg flex flex-col gap-6">
+        <h1 className="text-2xl font-bold text-gray-800 text-center">Антиплагиат</h1>
+        <p className="text-sm text-gray-500 text-center">
+          Загрузите документ для проверки на заимствования
+        </p>
+
+        <DropZone onFile={setFile} disabled={loading} />
+
+        {error && (
+          <p className="text-red-500 text-sm text-center">{error}</p>
+        )}
+
+        <button
+          onClick={handleSubmit}
+          disabled={!file || loading}
+          className="w-full py-3 rounded-xl bg-blue-600 text-white font-semibold hover:bg-blue-700 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
+        >
+          {loading ? 'Загрузка…' : 'Проверить'}
+        </button>
+      </div>
+    </div>
+  )
+}

frontend/src/pages/Report.tsx

@@ -0,0 +1,206 @@
+import { useEffect, useState, useCallback, useRef } from 'react'
+import { getReport, openProgressSocket, downloadReportPdf, ReportResponse, MatchResult, ProgressEvent } from '../api/client'
+import SimilarityMeter from '../components/SimilarityMeter'
+import MatchCard from '../components/MatchCard'
+import HighlightedText from '../components/HighlightedText'
+
+interface Props {
+  taskId: string
+  onBack: () => void
+}
+
+const STAGE_LABEL: Record<string, string> = {
+  fingerprint: 'Поиск точных совпадений…',
+  fuzzy: 'Нечёткое сравнение…',
+  semantic: 'Семантический анализ…',
+  saving: 'Сохранение результатов…',
+  done: 'Готово',
+  failed: 'Ошибка',
+}
+
+const POLL_INTERVAL = 4000
+
+export default function Report({ taskId, onBack }: Props) {
+  const [report, setReport] = useState<ReportResponse | null>(null)
+  const [error, setError] = useState<string | null>(null)
+  const [progress, setProgress] = useState<ProgressEvent | null>(null)
+  const [activeMatch, setActiveMatch] = useState<number | null>(null)
+  const pollRef = useRef<ReturnType<typeof setTimeout> | null>(null)
+
+  const fetchReport = useCallback(async () => {
+    try {
+      const data = await getReport(taskId)
+      setReport(data)
+      return data.status
+    } catch {
+      setError('Не удалось загрузить отчёт')
+      return 'failed'
+    }
+  }, [taskId])
+
+  // WebSocket progress + fallback polling
+  useEffect(() => {
+    const closeWs = openProgressSocket(taskId, (ev) => {
+      setProgress(ev)
+      if (ev.stage === 'done' || ev.stage === 'failed') {
+        fetchReport()
+      }
+    })
+
+    // Polling fallback if WS doesn't deliver done
+    const poll = async () => {
+      const s = await fetchReport()
+      if (s === 'processing') {
+        pollRef.current = setTimeout(poll, POLL_INTERVAL)
+      }
+    }
+    poll()
+
+    return () => {
+      closeWs()
+      if (pollRef.current) clearTimeout(pollRef.current)
+    }
+  }, [taskId, fetchReport])
+
+  const isProcessing = !report || report.status === 'processing'
+  const matches: MatchResult[] = report?.matches ?? []
+
+  return (
+    <div className="min-h-screen bg-gray-50 flex flex-col">
+      {/* Header */}
+      <header className="bg-white border-b px-6 py-4 flex items-center justify-between">
+        <button onClick={onBack} className="text-sm text-blue-600 hover:underline">
+          ← Новая проверка
+        </button>
+        <h1 className="font-semibold text-gray-800">Отчёт о проверке</h1>
+        <div className="flex items-center gap-3">
+          {report?.status === 'completed' && (
+            <button
+              onClick={() => { downloadReportPdf(taskId).catch(console.error) }}
+              className="text-xs border border-gray-200 rounded-lg px-3 py-1.5 hover:bg-gray-50 transition-colors"
+            >
+              Скачать PDF
+            </button>
+          )}
+          <span className="text-xs text-gray-400">{taskId.slice(0, 8)}…</span>
+        </div>
+      </header>
+
+      {/* Progress bar */}
+      {isProcessing && (
+        <div className="bg-white border-b px-6 py-3">
+          <div className="flex justify-between text-sm text-gray-600 mb-1">
+            <span>{progress ? STAGE_LABEL[progress.stage] : 'Запуск анализа…'}</span>
+            <span>{progress?.progress ?? 0}%</span>
+          </div>
+          <div className="w-full h-2 bg-gray-200 rounded-full overflow-hidden">
+            <div
+              className="h-full bg-blue-500 rounded-full transition-all duration-500"
+              style={{ width: `${progress?.progress ?? 0}%` }}
+            />
+          </div>
+          <div className="flex justify-between text-xs text-gray-400 mt-1">
+            {['fingerprint', 'fuzzy', 'semantic', 'saving', 'done'].map((stage) => (
+              <span
+                key={stage}
+                className={progress && ['fingerprint','fuzzy','semantic','saving','done'].indexOf(stage) <=
+                  ['fingerprint','fuzzy','semantic','saving','done'].indexOf(progress.stage)
+                  ? 'text-blue-600 font-medium' : ''}
+              >
+                {STAGE_LABEL[stage].replace('…', '')}
+              </span>
+            ))}
+          </div>
+        </div>
+      )}
+
+      {error && (
+        <div className="m-4 bg-red-50 border border-red-200 rounded-xl p-4 text-red-600 text-sm">
+          {error}
+        </div>
+      )}
+
+      {/* Main content */}
+      {report && report.status === 'completed' && (
+        <>
+          {/* Summary bar */}
+          <div className="bg-white border-b px-6 py-4">
+            <div className="max-w-4xl mx-auto">
+              <SimilarityMeter value={report.overall_similarity ?? 0} />
+            </div>
+          </div>
+
+          {/* Legend */}
+          <div className="bg-white border-b px-6 py-2 flex gap-4 text-xs text-gray-600">
+            <span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-red-200 inline-block" /> Точное совпадение</span>
+            <span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-orange-200 inline-block" /> Нечёткое</span>
+            <span className="flex items-center gap-1"><span className="w-3 h-3 rounded bg-yellow-200 inline-block" /> Семантика</span>
+          </div>
+
+          {/* Two-column layout */}
+          <div className="flex-1 flex overflow-hidden">
+            {/* Left: highlighted document text */}
+            <div className="flex-1 overflow-y-auto p-6">
+              <div className="max-w-3xl mx-auto bg-white rounded-2xl shadow-sm p-6 min-h-full">
+                {report.source_text ? (
+                  <HighlightedText
+                    text={report.source_text}
+                    matches={matches}
+                    activeIndex={activeMatch}
+                    onMatchClick={setActiveMatch}
+                  />
+                ) : (
+                  <p className="text-gray-400 text-sm">Текст документа недоступен</p>
+                )}
+              </div>
+            </div>
+
+            {/* Right: match list */}
+            <aside className="w-80 shrink-0 overflow-y-auto border-l bg-gray-50 p-4 flex flex-col gap-3">
+              <p className="text-xs font-semibold text-gray-500 uppercase tracking-wide">
+                Совпадения ({matches.length})
+              </p>
+
+              {matches.length === 0 && (
+                <p className="text-sm text-gray-400 text-center mt-8">
+                  Заимствований не обнаружено
+                </p>
+              )}
+
+              {matches.map((m, i) => (
+                <MatchCard
+                  key={i}
+                  match={m}
+                  index={i}
+                  active={activeMatch === i}
+                  onClick={() => setActiveMatch(activeMatch === i ? null : i)}
+                />
+              ))}
+            </aside>
+          </div>
+        </>
+      )}
+
+      {/* Still processing with no result yet */}
+      {isProcessing && !error && (
+        <div className="flex-1 flex items-center justify-center">
+          <div className="text-center text-gray-400">
+            <div className="text-4xl mb-3 animate-spin">⚙</div>
+            <p className="text-sm">Анализируем документ…</p>
+          </div>
+        </div>
+      )}
+
+      {report?.status === 'failed' && (
+        <div className="flex-1 flex items-center justify-center">
+          <div className="text-center text-red-500">
+            <p className="text-lg font-semibold">Анализ завершился с ошибкой</p>
+            <button onClick={onBack} className="mt-4 text-sm text-blue-600 hover:underline">
+              Попробовать снова
+            </button>
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}

frontend/tailwind.config.js

@@ -0,0 +1,8 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: ['./index.html', './src/**/*.{ts,tsx}'],
+  theme: {
+    extend: {},
+  },
+  plugins: [],
+}

frontend/vite.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+
+export default defineConfig({
+  plugins: [react()],
+  server: {
+    port: 5173,
+    proxy: {
+      '/api': {
+        target: 'http://localhost:8000',
+        changeOrigin: true,
+      }
+    }
+  }
+})

requirements.txt

@@ -0,0 +1,37 @@
+fastapi==0.104.1
+uvicorn==0.24.0
+sqlalchemy==2.0.23
+psycopg2-binary==2.9.9
+pydantic==2.5.0
+pydantic-settings==2.1.0
+python-multipart==0.0.6
+python-jose==3.3.0
+passlib==1.7.4
+bcrypt==4.1.1
+
+elasticsearch==8.11.0
+faiss-cpu==1.7.4
+sentence-transformers==2.2.2
+datasketch==1.0.8
+xxhash==3.4.1
+
+celery==5.3.4
+redis[asyncio]==5.0.1
+minio==7.2.0
+
+numpy==1.24.3
+scipy==1.11.4
+scikit-learn==1.3.2
+
+wikitextparser==0.31.9
+pymupdf==1.23.8
+pdfplumber==0.10.3
+python-docx==0.8.11
+
+requests==2.31.0
+aiofiles==23.2.1
+websockets==12.0
+pydantic-extra-types==2.3.0
+
+python-dotenv==1.0.0
+reportlab==4.1.0