test build

backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application
+COPY . .
+
+# Create data directories
+RUN mkdir -p /app/data/{raw,processed,index}
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/app/api/admin.py

@@ -0,0 +1,115 @@
+from fastapi import APIRouter, HTTPException, status, WebSocket, WebSocketDisconnect
+from pydantic import BaseModel
+import asyncio
+import json as _json
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+SOURCES = {
+    "wikipedia_ru": "Wikipedia RU (~8 GB)",
+    "wikipedia_en": "Wikipedia EN (~22 GB)",
+}
+
+
+class JobResponse(BaseModel):
+    status: str
+    source: str | None = None
+    label: str | None = None
+    progress: int = 0
+    message: str = ""
+    started_at: str | None = None
+    finished_at: str | None = None
+    docs_done: int = 0
+
+
+@router.get("/index/sources")
+def list_sources():
+    """Список доступных источников для индексации."""
+    return [{"key": k, "label": v} for k, v in SOURCES.items()]
+
+
+@router.post("/index/download/{source_key}", response_model=JobResponse)
+def start_download(source_key: str):
+    """Скачать дамп и проиндексировать с нуля."""
+    if source_key not in SOURCES:
+        raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
+
+    from app.tasks.indexing import get_job_status, download_and_index
+    current = get_job_status()
+    if current.get("status") == "running":
+        raise HTTPException(status_code=409, detail="Задание уже выполняется")
+
+    download_and_index.delay(source_key)
+    return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск скачивания…")
+
+
+@router.post("/index/update/{source_key}", response_model=JobResponse)
+def start_update(source_key: str):
+    """Обновить индекс — только новые документы (дамп не скачивается заново если уже есть)."""
+    if source_key not in SOURCES:
+        raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
+
+    from app.tasks.indexing import get_job_status, update_index
+    current = get_job_status()
+    if current.get("status") == "running":
+        raise HTTPException(status_code=409, detail="Задание уже выполняется")
+
+    update_index.delay(source_key)
+    return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск обновления…")
+
+
+@router.get("/index/status", response_model=JobResponse)
+def job_status():
+    """Текущий статус задания индексации."""
+    from app.tasks.indexing import get_job_status
+    data = get_job_status()
+    return JobResponse(
+        status=data.get("status", "idle"),
+        source=data.get("source"),
+        label=data.get("label"),
+        progress=data.get("progress", 0),
+        message=data.get("message", ""),
+        started_at=data.get("started_at"),
+        finished_at=data.get("finished_at"),
+        docs_done=data.get("docs_done", 0),
+    )
+
+
+@router.websocket("/ws/index-progress")
+async def ws_index_progress(websocket: WebSocket):
+    """Stream indexing job progress via Redis pub/sub."""
+    await websocket.accept()
+
+    import redis.asyncio as aioredis
+    from app.config import settings
+
+    r = aioredis.from_url(settings.REDIS_URL)
+    pubsub = r.pubsub()
+    await pubsub.subscribe("index_job_progress")
+
+    # Send current state immediately
+    from app.tasks.indexing import get_job_status
+    await websocket.send_json(get_job_status())
+
+    try:
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            try:
+                data = _json.loads(message["data"])
+            except Exception:
+                continue
+            await websocket.send_json(data)
+            if data.get("status") in ("done", "error"):
+                break
+    except (WebSocketDisconnect, asyncio.CancelledError):
+        pass
+    finally:
+        try:
+            await pubsub.unsubscribe("index_job_progress")
+        except Exception:
+            pass
+        try:
+            await r.aclose()
+        except Exception:
+            pass

backend/app/api/auth.py

@@ -0,0 +1,115 @@
+from datetime import datetime, timedelta
+from typing import Optional
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+from pydantic import BaseModel, EmailStr
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models.db import get_db, User
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
+
+
+# ── Schemas ────────────────────────────────────────────────────────────────
+
+class RegisterRequest(BaseModel):
+    email: EmailStr
+    password: str
+
+
+class TokenResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+
+
+class UserOut(BaseModel):
+    id: str
+    email: str
+    created_at: datetime
+
+    class Config:
+        from_attributes = True
+
+
+# ── Helpers ────────────────────────────────────────────────────────────────
+
+def hash_password(password: str) -> str:
+    return pwd_context.hash(password)
+
+
+def verify_password(plain: str, hashed: str) -> bool:
+    return pwd_context.verify(plain, hashed)
+
+
+def create_token(user_id) -> str:
+    expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+    return jwt.encode(
+        {"sub": str(user_id), "exp": expire},
+        settings.SECRET_KEY,
+        algorithm=settings.ALGORITHM,
+    )
+
+
+def get_current_user(
+    token: Optional[str] = Depends(oauth2_scheme),
+    db: Session = Depends(get_db),
+) -> Optional[User]:
+    if not token:
+        return None
+    try:
+        payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
+        user_id_str: str = payload.get("sub")
+        if not user_id_str:
+            return None
+        import uuid as _uuid
+        user_id = _uuid.UUID(user_id_str)
+    except (JWTError, ValueError):
+        return None
+    return db.query(User).filter(User.id == user_id).first()
+
+
+def require_user(current_user: Optional[User] = Depends(get_current_user)) -> User:
+    if not current_user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return current_user
+
+
+# ── Endpoints ──────────────────────────────────────────────────────────────
+
+@router.post("/register", response_model=UserOut, status_code=status.HTTP_201_CREATED)
+def register(body: RegisterRequest, db: Session = Depends(get_db)):
+    if db.query(User).filter(User.email == body.email).first():
+        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered")
+    user = User(
+        email=body.email,
+        hashed_password=hash_password(body.password),
+    )
+    db.add(user)
+    db.commit()
+    db.refresh(user)
+    return user
+
+
+@router.post("/login", response_model=TokenResponse)
+def login(form: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):
+    user = db.query(User).filter(User.email == form.username).first()
+    if not user or not verify_password(form.password, user.hashed_password):
+        raise HTTPException(status_code=400, detail="Incorrect email or password")
+    return {"access_token": create_token(user.id)}
+
+
+@router.get("/me", response_model=UserOut)
+def me(current_user: User = Depends(require_user)):
+    return current_user

backend/app/celery_app.py

@@ -0,0 +1,18 @@
+from celery import Celery
+from app.config import settings
+
+celery_app = Celery(
+    "antiplagiator",
+    broker=settings.CELERY_BROKER_URL,
+    backend=settings.CELERY_RESULT_BACKEND,
+    include=["app.tasks", "app.tasks.indexing"],
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    result_serializer="json",
+    accept_content=["json"],
+    timezone="UTC",
+    enable_utc=True,
+    task_track_started=True,
+)

backend/app/config.py

@@ -0,0 +1,50 @@
+import os
+from pydantic_settings import BaseSettings
+
+class Settings(BaseSettings):
+    # Database
+    DATABASE_URL: str = "postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator"
+
+    # Elasticsearch
+    ELASTICSEARCH_HOST: str = "localhost"
+    ELASTICSEARCH_PORT: int = 9200
+
+    # Redis
+    REDIS_URL: str = "redis://localhost:6379/0"
+
+    # MinIO
+    MINIO_ENDPOINT: str = "localhost:9000"
+    MINIO_ACCESS_KEY: str = "minioadmin"
+    MINIO_SECRET_KEY: str = "minioadmin"
+    MINIO_BUCKET: str = "antiplagiator"
+    MINIO_SECURE: bool = False
+
+    # JWT
+    SECRET_KEY: str = "your-secret-key-change-in-prod"
+    ALGORITHM: str = "HS256"
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
+
+    # Celery
+    CELERY_BROKER_URL: str = "redis://localhost:6379/0"
+    CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
+
+    # API
+    API_TITLE: str = "Antiplagiator API"
+    API_VERSION: str = "0.1.0"
+    API_DESCRIPTION: str = "Text plagiarism detection system"
+
+    # Paths
+    DATA_DIR: str = "./data"
+    RAW_DATA_DIR: str = "./data/raw"
+    PROCESSED_DATA_DIR: str = "./data/processed"
+    INDEX_DIR: str = "./data/index"
+
+    # Analysis
+    FINGERPRINT_THRESHOLD: float = 0.5
+    SEMANTIC_THRESHOLD: float = 0.6
+    TOP_K_RESULTS: int = 10
+
+    class Config:
+        env_file = ".env"
+
+settings = Settings()

backend/app/core/analyzer.py

@@ -0,0 +1,149 @@
+import re
+from typing import Dict, List, Any, Tuple
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+from app.config import settings
+
+FRAGMENT_WORDS = 40
+
+
+def _split_sentences(text: str) -> List[str]:
+    parts = re.split(r'(?<=[.!?])\s+', text.strip())
+    return [p for p in parts if p]
+
+
+def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]:
+    """Return (fragment_text, char_start, char_end) around the given sentence index."""
+    sentences = _split_sentences(text)
+    start_i = max(0, sentence_idx - context_sentences)
+    end_i = min(len(sentences), sentence_idx + context_sentences + 1)
+
+    fragment = " ".join(sentences[start_i:end_i])
+    char_start = text.find(sentences[start_i]) if sentences else 0
+    char_end = char_start + len(fragment)
+    return fragment, char_start, char_end
+
+
+def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str:
+    words = text.split()
+    if len(words) <= max_words:
+        return text
+    return " ".join(words[:max_words]) + "…"
+
+
+class DocumentAnalyzer:
+    def __init__(self):
+        self.winnowing = WinNowing()
+        self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD)
+        self.embeddings = EmbeddingIndex()
+        self.fulltext = FulltextSearch()
+
+    def analyze(self, text: str, task_id: str) -> Dict[str, Any]:
+        matches: List[Dict[str, Any]] = []
+        matches.extend(self._level_1_exact_match(text))
+        matches.extend(self._level_2_fuzzy_match(text))
+        matches.extend(self._level_3_semantic_match(text))
+
+        return {
+            "task_id": task_id,
+            "status": "completed",
+            "overall_similarity": self._calculate_overall_similarity(matches),
+            "matches": matches[:settings.TOP_K_RESULTS],
+        }
+
+    def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        sentences = _split_sentences(text)
+        if not sentences:
+            return []
+
+        results = []
+        for idx, sentence in enumerate(sentences[:20]):
+            if len(sentence.split()) < 6:
+                continue
+            try:
+                es_results = self.fulltext.search_phrase(sentence, top_k=2)
+                for r in es_results:
+                    r["_sentence_idx"] = idx
+                results.extend(es_results)
+            except Exception:
+                continue
+
+        matches = []
+        seen_docs: set = set()
+        for result in results:
+            if result["doc_id"] in seen_docs:
+                continue
+            idx = result.get("_sentence_idx", 0)
+            fragment, char_start, char_end = _extract_fragment(text, idx)
+            matches.append({
+                "method": "exact",
+                "similarity": min(result["score"] * 20, 100),
+                "source_id": str(result["doc_id"]),   # передаём doc_id напрямую
+                "source_title": result["title"],
+                "source_db": result["source"],
+                "url": result["url"],
+                "fragment_text": _truncate(fragment),
+                "fragment_start": char_start,
+                "fragment_end": char_end,
+            })
+            seen_docs.add(result["doc_id"])
+
+        return matches[:top_k]
+
+    def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            fuzzy_results = self.minhash_lsh.query(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in fuzzy_results:
+            if similarity >= settings.FINGERPRINT_THRESHOLD:
+                matches.append({
+                    "method": "fuzzy",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            embedding_results = self.embeddings.search(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in embedding_results:
+            if similarity >= settings.SEMANTIC_THRESHOLD:
+                matches.append({
+                    "method": "semantic",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float:
+        if not matches:
+            return 0.0
+
+        weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3}
+        total_weight = 0.0
+        total_similarity = 0.0
+
+        for match in matches:
+            w = weights.get(match["method"], 0.1)
+            total_similarity += match["similarity"] * w
+            total_weight += w
+
+        return round(total_similarity / total_weight if total_weight > 0 else 0, 1)

backend/app/core/embeddings.py

@@ -0,0 +1,76 @@
+import numpy as np
+import faiss
+from sentence_transformers import SentenceTransformer
+from typing import List, Tuple
+import os
+from app.config import settings
+
+class EmbeddingIndex:
+    def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"):
+        self.model = SentenceTransformer(model_name)
+        self.embedding_dim = self.model.get_sentence_embedding_dimension()
+        self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32)
+        self.doc_ids = []
+        self.embeddings = None
+        
+        # Try to load existing index
+        self._load_index()
+
+    def add_document(self, doc_id: str, text: str):
+        embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        self.index.add(np.array([embedding], dtype=np.float32))
+        self.doc_ids.append(doc_id)
+        
+        if self.embeddings is None:
+            self.embeddings = np.array([embedding])
+        else:
+            self.embeddings = np.vstack([self.embeddings, embedding])
+
+    def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32):
+        doc_ids = [doc[0] for doc in docs]
+        texts = [doc[1] for doc in docs]
+        
+        for i in range(0, len(texts), batch_size):
+            batch_texts = texts[i:i+batch_size]
+            embeddings = self.model.encode(batch_texts, convert_to_numpy=True)
+            
+            self.index.add(embeddings.astype(np.float32))
+            self.doc_ids.extend(doc_ids[i:i+batch_size])
+            
+            if self.embeddings is None:
+                self.embeddings = embeddings
+            else:
+                self.embeddings = np.vstack([self.embeddings, embeddings])
+
+    def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        query_embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        query_embedding = np.array([query_embedding], dtype=np.float32)
+        
+        distances, indices = self.index.search(query_embedding, top_k)
+        
+        results = []
+        for idx, distance in zip(indices[0], distances[0]):
+            if idx < len(self.doc_ids):
+                similarity = 1.0 / (1.0 + distance)
+                results.append((self.doc_ids[idx], similarity))
+        
+        return results
+
+    def _load_index(self):
+        index_path = os.path.join(settings.INDEX_DIR, "faiss_index")
+        ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy")
+        
+        if os.path.exists(index_path) and os.path.exists(ids_path):
+            try:
+                self.index = faiss.read_index(index_path)
+                self.doc_ids = list(np.load(ids_path, allow_pickle=True))
+            except Exception as e:
+                print(f"Failed to load index: {e}")
+
+    def save_index(self):
+        os.makedirs(settings.INDEX_DIR, exist_ok=True)
+        faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index"))
+        np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object))
+
+    def get_index_size(self) -> int:
+        return self.index.ntotal

backend/app/core/fingerprint.py

@@ -0,0 +1,82 @@
+import xxhash
+from typing import Set, Tuple, List
+from datasketch import MinHash, MinHashLSH
+import numpy as np
+
+class WinNowing:
+    def __init__(self, k: int = 5, window_size: int = 4):
+        self.k = k
+        self.window_size = window_size
+
+    def get_kgrams(self, text: str) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(len(words) - self.k + 1):
+            kgram = ' '.join(words[i:i+self.k])
+            kgrams.append(kgram)
+        return kgrams
+
+    def fingerprint(self, text: str) -> Set[int]:
+        kgrams = self.get_kgrams(text)
+        hashes = []
+        for kgram in kgrams:
+            h = int(xxhash.xxh64(kgram).hexdigest(), 16)
+            hashes.append(h)
+        
+        if not hashes:
+            return set()
+        
+        fingerprints = set()
+        for i in range(len(hashes) - self.window_size + 1):
+            window = hashes[i:i+self.window_size]
+            min_hash = min(window)
+            fingerprints.add(min_hash)
+        
+        return fingerprints
+
+    def compare(self, fp1: Set[int], fp2: Set[int]) -> float:
+        if not fp1 or not fp2:
+            return 0.0
+        intersection = len(fp1 & fp2)
+        union = len(fp1 | fp2)
+        return intersection / union if union > 0 else 0.0
+
+class MinHashLSHIndex:
+    def __init__(self, num_perm: int = 128, threshold: float = 0.5):
+        self.num_perm = num_perm
+        self.threshold = threshold
+        self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
+        self.documents = {}
+
+    def add_document(self, doc_id: str, text: str):
+        kgrams = self._get_kgrams(text)
+        m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            m.update(kgram.encode())
+        
+        self.lsh.insert(doc_id, m)
+        self.documents[doc_id] = m
+
+    def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        kgrams = self._get_kgrams(text)
+        query_m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            query_m.update(kgram.encode())
+        
+        candidates = self.lsh.query(query_m)
+        
+        results = []
+        for doc_id in candidates:
+            similarity = query_m.jaccard(self.documents[doc_id])
+            results.append((doc_id, similarity))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def _get_kgrams(self, text: str, k: int = 5) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(max(1, len(words) - k + 1)):
+            kgram = ' '.join(words[i:i+k])
+            kgrams.append(kgram)
+        return kgrams

backend/app/core/fulltext.py

@@ -0,0 +1,135 @@
+from elasticsearch import Elasticsearch
+from typing import List, Tuple, Dict, Any
+from app.config import settings
+
+class FulltextSearch:
+    def __init__(self):
+        self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"])
+        self.index_name = "documents"
+        self._create_index()
+
+    def _create_index(self):
+        if not self.es.indices.exists(index=self.index_name):
+            self.es.indices.create(
+                index=self.index_name,
+                body={
+                    "settings": {
+                        "analysis": {
+                            "analyzer": {
+                                "ru_en": {
+                                    "type": "custom",
+                                    "tokenizer": "standard",
+                                    "filter": ["lowercase", "ru_stop", "ru_stem"]
+                                }
+                            },
+                            "filter": {
+                                "ru_stop": {
+                                    "type": "stop",
+                                    "stopwords": "_russian_"
+                                },
+                                "ru_stem": {
+                                    "type": "stemmer",
+                                    "language": "russian"
+                                }
+                            }
+                        }
+                    },
+                    "mappings": {
+                        "properties": {
+                            "doc_id": {"type": "keyword"},
+                            "text": {"type": "text", "analyzer": "ru_en"},
+                            "title": {"type": "text", "analyzer": "ru_en"},
+                            "source": {"type": "keyword"},
+                            "lang": {"type": "keyword"},
+                            "url": {"type": "keyword"}
+                        }
+                    }
+                }
+            )
+
+    def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""):
+        self.es.index(
+            index=self.index_name,
+            id=doc_id,
+            body={
+                "doc_id": doc_id,
+                "title": title,
+                "text": text,
+                "source": source,
+                "lang": lang,
+                "url": url
+            }
+        )
+
+    def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]:
+        must = {
+            "multi_match": {
+                "query": query,
+                "fields": ["text^2", "title^3"],
+                "type": "best_fields",
+                "analyzer": "ru_en",
+            }
+        }
+
+        if lang:
+            search_body = {
+                "size": top_k,
+                "query": {
+                    "bool": {
+                        "must": must,
+                        "filter": {"term": {"lang": lang}},
+                    }
+                },
+            }
+        else:
+            search_body = {"size": top_k, "query": must}
+
+        results = self.es.search(index=self.index_name, body=search_body)
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]:
+        results = self.es.search(
+            index=self.index_name,
+            body={
+                "size": top_k,
+                "query": {
+                    "match_phrase": {
+                        "text": phrase
+                    }
+                }
+            }
+        )
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def bulk_index(self, documents: List[Dict[str, Any]]):
+        actions = []
+        for doc in documents:
+            actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}})
+            actions.append(doc)
+        
+        from elasticsearch.helpers import bulk
+        bulk(self.es, actions)

backend/app/main.py

@@ -0,0 +1,354 @@
+from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+import uuid
+import io
+import asyncio
+import json as _json
+from sqlalchemy.orm import Session
+from app.config import settings
+from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User
+from app.schemas import (
+    CheckResponse, ReportResponse, ReportSummary,
+    ReportsListResponse, StatsResponse, HealthResponse, MatchResult,
+)
+from app.core.embeddings import EmbeddingIndex
+from app.api.auth import router as auth_router, get_current_user
+from app.api.admin import router as admin_router
+import pdfplumber
+import docx as python_docx
+from minio import Minio
+from minio.error import S3Error
+from typing import Optional
+
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(
+    title=settings.API_TITLE,
+    version=settings.API_VERSION,
+    description=settings.API_DESCRIPTION,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(auth_router)
+app.include_router(admin_router)
+
+_minio: Minio | None = None
+_embedding_index: EmbeddingIndex | None = None
+
+
+def get_minio() -> Minio:
+    global _minio
+    if _minio is None:
+        _minio = Minio(
+            settings.MINIO_ENDPOINT,
+            access_key=settings.MINIO_ACCESS_KEY,
+            secret_key=settings.MINIO_SECRET_KEY,
+            secure=settings.MINIO_SECURE,
+        )
+        try:
+            if not _minio.bucket_exists(settings.MINIO_BUCKET):
+                _minio.make_bucket(settings.MINIO_BUCKET)
+        except S3Error:
+            pass
+    return _minio
+
+
+def get_embedding_index() -> EmbeddingIndex:
+    global _embedding_index
+    if _embedding_index is None:
+        _embedding_index = EmbeddingIndex()
+    return _embedding_index
+
+
+def extract_text(content: bytes, filename: str) -> str:
+    if filename.endswith(".txt"):
+        return content.decode("utf-8", errors="replace")
+
+    if filename.endswith(".pdf"):
+        with pdfplumber.open(io.BytesIO(content)) as pdf:
+            return "\n".join(page.extract_text() or "" for page in pdf.pages)
+
+    if filename.endswith(".docx"):
+        doc = python_docx.Document(io.BytesIO(content))
+        return "\n".join(p.text for p in doc.paragraphs)
+
+    return content.decode("utf-8", errors="replace")
+
+
+@app.get("/api/health", response_model=HealthResponse)
+async def health_check():
+    return {"status": "ok", "version": settings.API_VERSION}
+
+
+@app.post("/api/check", response_model=CheckResponse)
+async def check_document(
+    file: UploadFile = File(...),
+    db: Session = Depends(get_db),
+    current_user: Optional[User] = Depends(get_current_user),
+):
+    if not file.filename:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename")
+
+    allowed = {".txt", ".pdf", ".docx"}
+    ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
+    if ext not in allowed:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Unsupported file type. Allowed: {', '.join(allowed)}",
+        )
+
+    task_id = uuid.uuid4()
+    content = await file.read()
+
+    minio_key = f"uploads/{task_id}/{file.filename}"
+    try:
+        minio = get_minio()
+        minio.put_object(
+            settings.MINIO_BUCKET,
+            minio_key,
+            io.BytesIO(content),
+            length=len(content),
+            content_type=file.content_type or "application/octet-stream",
+        )
+    except Exception:
+        minio_key = None
+
+    try:
+        text = extract_text(content, file.filename)
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}")
+
+    if not text.strip():
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable")
+
+    task = AnalysisTask(
+        id=task_id,
+        status="processing",
+        source_text=text,
+        user_id=current_user.id if current_user else None,
+    )
+    db.add(task)
+    db.commit()
+
+    from app.tasks import analyze_document
+    analyze_document.delay(text, str(task_id))
+
+    return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"}
+
+
+@app.get("/api/report/{task_id}", response_model=ReportResponse)
+async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)):
+    task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+    if not task:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
+
+    matches = []
+    if task.status == "completed":
+        db_matches = db.query(Match).filter(Match.task_id == task_id).all()
+        for m in db_matches:
+            doc = db.query(Document).filter(Document.id == m.doc_id).first()
+            matches.append(
+                MatchResult(
+                    method=m.method,
+                    similarity=float(m.similarity),
+                    source_title=doc.title if doc else None,
+                    source_db=doc.source if doc else None,
+                    url=doc.url if doc else None,
+                    fragment_text=m.fragment_text,
+                    fragment_start=m.fragment_start,
+                    fragment_end=m.fragment_end,
+                )
+            )
+
+    return ReportResponse(
+        task_id=str(task.id),
+        status=task.status,
+        overall_similarity=task.overall_similarity,
+        matches=matches or None,
+        source_text=task.source_text,
+        created_at=task.created_at,
+        completed_at=task.completed_at,
+    )
+
+
+@app.get("/api/report/{task_id}/pdf")
+async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)):
+    """Generate and return a PDF report."""
+    task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+    if not task:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
+    if task.status != "completed":
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet")
+
+    db_matches = db.query(Match).filter(Match.task_id == task_id).all()
+    matches_data = []
+    for m in db_matches:
+        doc = db.query(Document).filter(Document.id == m.doc_id).first()
+        matches_data.append({
+            "method": m.method,
+            "similarity": m.similarity,
+            "source_title": doc.title if doc else "—",
+            "source_db": doc.source if doc else "—",
+            "url": doc.url if doc else "",
+            "fragment_text": m.fragment_text or "",
+        })
+
+    pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data)
+
+    return StreamingResponse(
+        io.BytesIO(pdf_bytes),
+        media_type="application/pdf",
+        headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'},
+    )
+
+
+def _build_pdf(task_id: str, overall: float, matches: list) -> bytes:
+    from reportlab.lib.pagesizes import A4
+    from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
+    from reportlab.lib.units import cm
+    from reportlab.lib import colors
+    from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable
+    from reportlab.pdfbase import pdfmetrics
+    from reportlab.pdfbase.ttfonts import TTFont
+    import os
+
+    buf = io.BytesIO()
+    doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
+    styles = getSampleStyleSheet()
+
+    METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"}
+    METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")}
+
+    title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6)
+    h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4)
+    body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14)
+    small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey)
+
+    story = [
+        Paragraph("Отчёт о проверке на плагиат", title_style),
+        Paragraph(f"ID задачи: {task_id}", small_style),
+        Spacer(1, 0.3*cm),
+        HRFlowable(width="100%", thickness=1, color=colors.lightgrey),
+        Spacer(1, 0.3*cm),
+        Paragraph(f"Общий процент заимствований: <b>{overall:.1f}%</b>", body_style),
+        Spacer(1, 0.5*cm),
+    ]
+
+    if not matches:
+        story.append(Paragraph("Совпадений не обнаружено.", body_style))
+    else:
+        story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style))
+        for i, m in enumerate(matches, 1):
+            bg = METHOD_COLOR.get(m["method"], colors.whitesmoke)
+            data = [
+                [Paragraph(f"<b>#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%</b>", body_style), ""],
+                ["Источник:", Paragraph(m["source_title"], body_style)],
+                ["База:", m["source_db"]],
+            ]
+            if m["url"]:
+                data.append(["URL:", Paragraph(f'<link href="{m["url"]}">{m["url"]}</link>', small_style)])
+            if m["fragment_text"]:
+                data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)])
+
+            tbl = Table(data, colWidths=[3.5*cm, None])
+            tbl.setStyle(TableStyle([
+                ("BACKGROUND", (0, 0), (-1, 0), bg),
+                ("SPAN", (0, 0), (-1, 0)),
+                ("FONTSIZE", (0, 0), (-1, -1), 9),
+                ("VALIGN", (0, 0), (-1, -1), "TOP"),
+                ("TOPPADDING", (0, 0), (-1, -1), 3),
+                ("BOTTOMPADDING", (0, 0), (-1, -1), 3),
+                ("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey),
+            ]))
+            story.append(tbl)
+            story.append(Spacer(1, 0.3*cm))
+
+    doc.build(story)
+    return buf.getvalue()
+
+
+@app.get("/api/reports", response_model=ReportsListResponse)
+async def list_reports(
+    user_id: str | None = Query(default=None),
+    db: Session = Depends(get_db),
+    current_user: Optional[User] = Depends(get_current_user),
+):
+    query = db.query(AnalysisTask)
+    # Авторизованный пользователь видит только свои задачи
+    if current_user:
+        query = query.filter(AnalysisTask.user_id == current_user.id)
+    elif user_id:
+        query = query.filter(AnalysisTask.user_id == user_id)
+
+    tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all()
+
+    return ReportsListResponse(
+        tasks=[
+            ReportSummary(
+                task_id=str(t.id),
+                status=t.status,
+                overall_similarity=t.overall_similarity,
+                created_at=t.created_at,
+            )
+            for t in tasks
+        ]
+    )
+
+
+@app.get("/api/stats", response_model=StatsResponse)
+async def get_stats(db: Session = Depends(get_db)):
+    total_documents = db.query(Document).count()
+    index = get_embedding_index()
+
+    return StatsResponse(
+        total_documents_indexed=total_documents,
+        faiss_index_size=index.get_index_size(),
+        elasticsearch_available=True,
+        api_version=settings.API_VERSION,
+    )
+
+
+@app.websocket("/ws/progress/{task_id}")
+async def ws_progress(websocket: WebSocket, task_id: str):
+    await websocket.accept()
+    import redis.asyncio as aioredis
+
+    r = aioredis.from_url(settings.REDIS_URL)
+    pubsub = r.pubsub()
+    await pubsub.subscribe(f"progress:{task_id}")
+
+    try:
+        async for message in pubsub.listen():
+            if message["type"] != "message":
+                continue
+            try:
+                data = _json.loads(message["data"])
+            except Exception:
+                continue
+            await websocket.send_json(data)
+            if data.get("stage") in ("done", "failed"):
+                break
+    except (WebSocketDisconnect, asyncio.CancelledError):
+        pass
+    finally:
+        try:
+            await pubsub.unsubscribe(f"progress:{task_id}")
+        except Exception:
+            pass
+        try:
+            await r.aclose()
+        except Exception:
+            pass
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

backend/app/models/db.py

@@ -0,0 +1,82 @@
+import uuid
+from sqlalchemy import Column, String, Text, DateTime, Integer, Float, ForeignKey, Index, create_engine, Boolean
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from datetime import datetime
+from app.config import settings
+
+Base = declarative_base()
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    email = Column(String(255), unique=True, nullable=False, index=True)
+    hashed_password = Column(String(255), nullable=False)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class Document(Base):
+    __tablename__ = "documents"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    source = Column(String(50), nullable=False)
+    ext_id = Column(String(200), nullable=True)
+    title = Column(Text, nullable=False)
+    url = Column(Text, nullable=True)
+    lang = Column(String(2), nullable=True)
+    minio_key = Column(Text, nullable=True)
+    indexed_at = Column(DateTime, default=datetime.utcnow)
+
+
+class Fingerprint(Base):
+    __tablename__ = "fingerprints"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False)
+    hash = Column(String(64), nullable=False)   # xxhash hex string
+    position = Column(Integer, nullable=True)
+
+
+Index("ix_fingerprints_hash", Fingerprint.hash)
+Index("ix_fingerprints_doc_id", Fingerprint.doc_id)
+
+
+class AnalysisTask(Base):
+    __tablename__ = "analysis_tasks"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
+    status = Column(String(20), default="pending")
+    overall_similarity = Column(Float, nullable=True)
+    source_text = Column(Text, nullable=True)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    completed_at = Column(DateTime, nullable=True)
+
+
+class Match(Base):
+    __tablename__ = "matches"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    task_id = Column(UUID(as_uuid=True), ForeignKey("analysis_tasks.id"), nullable=False)
+    doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=True)
+    similarity = Column(Float, nullable=False)
+    method = Column(String(20), nullable=False)
+    fragment_start = Column(Integer, nullable=True)
+    fragment_end = Column(Integer, nullable=True)
+    fragment_text = Column(Text, nullable=True)
+
+
+engine = create_engine(settings.DATABASE_URL, echo=False)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/schemas/__init__.py

@@ -0,0 +1,54 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List
+from datetime import datetime
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str
+
+
+class CheckResponse(BaseModel):
+    task_id: str
+    status: str
+    message: str
+
+
+class MatchResult(BaseModel):
+    method: str = Field(..., description="exact | fuzzy | semantic")
+    similarity: float = Field(..., ge=0, le=100)
+    source_title: Optional[str] = None
+    source_db: Optional[str] = None
+    url: Optional[str] = None
+    source_id: Optional[str] = None
+    fragment_text: Optional[str] = None
+    fragment_start: Optional[int] = None
+    fragment_end: Optional[int] = None
+
+
+class ReportResponse(BaseModel):
+    task_id: str
+    status: str
+    overall_similarity: Optional[float] = None
+    matches: Optional[List[MatchResult]] = None
+    source_text: Optional[str] = None
+    created_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+
+
+class ReportSummary(BaseModel):
+    task_id: str
+    status: str
+    overall_similarity: Optional[float] = None
+    created_at: Optional[datetime] = None
+
+
+class ReportsListResponse(BaseModel):
+    tasks: List[ReportSummary]
+
+
+class StatsResponse(BaseModel):
+    total_documents_indexed: int
+    faiss_index_size: int
+    elasticsearch_available: bool
+    api_version: str

backend/app/tasks/__init__.py

@@ -0,0 +1,95 @@
+import json
+import uuid
+import redis
+from app.config import settings
+from app.models.db import SessionLocal, AnalysisTask, Match
+from app.core.analyzer import DocumentAnalyzer
+from app.celery_app import celery_app
+from datetime import datetime
+
+_analyzer: DocumentAnalyzer | None = None
+_redis: redis.Redis | None = None
+
+
+def get_analyzer() -> DocumentAnalyzer:
+    global _analyzer
+    if _analyzer is None:
+        _analyzer = DocumentAnalyzer()
+    return _analyzer
+
+
+def get_redis() -> redis.Redis:
+    global _redis
+    if _redis is None:
+        _redis = redis.from_url(settings.REDIS_URL)
+    return _redis
+
+
+def _publish(task_id: str, stage: str, progress: int):
+    try:
+        get_redis().publish(
+            f"progress:{task_id}",
+            json.dumps({"stage": stage, "progress": progress}),
+        )
+    except Exception:
+        pass
+
+
+@celery_app.task(bind=True, name="tasks.analyze_document")
+def analyze_document(self, text: str, task_id: str):
+    db = SessionLocal()
+    try:
+        analyzer = get_analyzer()
+
+        _publish(task_id, "fingerprint", 10)
+        exact = analyzer._level_1_exact_match(text)
+
+        _publish(task_id, "fuzzy", 40)
+        fuzzy = analyzer._level_2_fuzzy_match(text)
+
+        _publish(task_id, "semantic", 70)
+        semantic = analyzer._level_3_semantic_match(text)
+
+        matches = exact + fuzzy + semantic
+        overall_similarity = analyzer._calculate_overall_similarity(matches)
+
+        _publish(task_id, "saving", 90)
+
+        task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+        if not task:
+            return
+
+        task.status = "completed"
+        task.overall_similarity = overall_similarity
+        task.completed_at = datetime.utcnow()
+
+        for match in matches[:settings.TOP_K_RESULTS]:
+            raw_id = match.get("source_id")
+            try:
+                doc_id = uuid.UUID(str(raw_id)) if raw_id is not None else None
+            except (ValueError, TypeError, AttributeError):
+                doc_id = None
+
+            db.add(Match(
+                task_id=task_id,
+                doc_id=doc_id,
+                similarity=float(match["similarity"]),
+                method=match["method"],
+                fragment_start=match.get("fragment_start"),
+                fragment_end=match.get("fragment_end"),
+                fragment_text=match.get("fragment_text"),
+            ))
+
+        db.commit()
+        _publish(task_id, "done", 100)
+
+    except Exception as exc:
+        task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
+        if task:
+            task.status = "failed"
+            db.commit()
+        _publish(task_id, "failed", 0)
+        raise self.retry(exc=exc, countdown=5, max_retries=2)
+
+    finally:
+        db.close()

backend/app/tasks/indexing.py

@@ -0,0 +1,282 @@
+"""
+Celery tasks for downloading and indexing document sources.
+
+Redis key  "index_job"  stores current job state as JSON:
+  { status, source, progress, message, started_at, finished_at, docs_total, docs_done }
+"""
+import bz2
+import json
+import os
+import time
+import requests
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from pathlib import Path
+
+import redis
+from app.celery_app import celery_app
+
+from app.config import settings
+from app.models.db import SessionLocal, Document, Fingerprint
+
+JOB_KEY = "index_job"
+
+WIKIPEDIA_SOURCES = {
+    "wikipedia_ru": {
+        "url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-pages-articles.xml.bz2",
+        "meta_url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-md5sums.txt",
+        "lang": "ru",
+        "label": "Wikipedia RU",
+    },
+    "wikipedia_en": {
+        "url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2",
+        "meta_url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-md5sums.txt",
+        "lang": "en",
+        "label": "Wikipedia EN",
+    },
+}
+
+
+# ── Redis helpers ──────────────────────────────────────────────────────────
+
+def _r() -> redis.Redis:
+    return redis.from_url(settings.REDIS_URL)
+
+
+def _set_status(data: dict):
+    _r().set(JOB_KEY, json.dumps(data, default=str))
+
+
+def _get_status() -> dict:
+    raw = _r().get(JOB_KEY)
+    if not raw:
+        return {"status": "idle"}
+    return json.loads(raw)
+
+
+def _pub(message: str, progress: int = 0, **extra):
+    state = _get_status()
+    state.update({"message": message, "progress": progress, **extra})
+    _set_status(state)
+    _r().publish("index_job_progress", json.dumps(state, default=str))
+
+
+# ── Text helpers ───────────────────────────────────────────────────────────
+
+import re
+
+
+def _clean_wiki(text: str) -> str:
+    text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
+    text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
+    text = re.sub(r'\{\{[^}]*\}\}', '', text)
+    text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
+    text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
+    text = re.sub(r'<[^>]+>', '', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r' +', ' ', text)
+    return text.strip()
+
+
+def _normalize(text: str) -> str:
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
+    text = re.sub(r'[—–‒―]', '-', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
+
+
+# ── Download helper ────────────────────────────────────────────────────────
+
+def _download_file(url: str, dest: Path, label: str) -> Path:
+    """Stream-download url → dest, publishing progress."""
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    r = requests.get(url, stream=True, timeout=30)
+    r.raise_for_status()
+    total = int(r.headers.get("Content-Length", 0))
+    done = 0
+    last_pct = -1
+
+    with open(dest, "wb") as f:
+        for chunk in r.iter_content(chunk_size=1 << 20):  # 1 MB
+            f.write(chunk)
+            done += len(chunk)
+            pct = int(done / total * 40) if total else 0  # 0-40% of overall
+            if pct != last_pct:
+                _pub(f"Скачивание {label}: {done >> 20} MB", progress=pct)
+                last_pct = pct
+
+    return dest
+
+
+# ── Indexing helper ────────────────────────────────────────────────────────
+
+def _index_wikipedia_bz2(bz2_path: Path, source: str, lang: str, update_only: bool) -> int:
+    """
+    Parse Wikipedia bz2 dump and index documents.
+    update_only=True → skip ext_ids already in DB.
+    Returns count of newly indexed docs.
+    """
+    from app.core.fingerprint import WinNowing, MinHashLSHIndex
+    from app.core.embeddings import EmbeddingIndex
+    from app.core.fulltext import FulltextSearch
+
+    db = SessionLocal()
+    winnowing = WinNowing()
+    minhash = MinHashLSHIndex()
+    emb = EmbeddingIndex()
+    fulltext = FulltextSearch()
+
+    # Pre-load known ext_ids for fast skip
+    known: set[str] = set()
+    if update_only:
+        for (ext_id,) in db.query(Document.ext_id).filter(Document.source == source).yield_per(10_000):
+            if ext_id:
+                known.add(ext_id)
+
+    ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
+
+    embed_batch: list[tuple[str, str]] = []
+    BATCH = 32
+    count = 0
+    parsed = 0
+
+    with bz2.BZ2File(str(bz2_path), 'r') as f:
+        context = ET.iterparse(f, events=('end',))
+        for event, elem in context:
+            if not elem.tag.endswith('page'):
+                continue
+
+            title_el = elem.find('wiki:title', ns) or elem.find('.//title')
+            id_el = elem.find('wiki:id', ns) or elem.find('.//id')
+            rev = elem.find('.//revision')
+            text_el = rev.find('wiki:text', ns) if rev is not None else None
+            if rev is None:
+                text_el = elem.find('.//text')
+
+            if title_el is None or text_el is None:
+                elem.clear(); continue
+
+            title = (title_el.text or '').strip()
+            page_id = (id_el.text or '').strip() if id_el is not None else ''
+            raw_text = text_el.text or ''
+            ext_id = f"wiki_{page_id}"
+
+            if raw_text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
+                elem.clear(); continue
+
+            text = _normalize(_clean_wiki(raw_text))
+            if len(text.split()) < 50:
+                elem.clear(); continue
+
+            parsed += 1
+
+            if update_only and ext_id in known:
+                elem.clear(); continue
+
+            url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}"
+
+            # PostgreSQL
+            doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
+            db.add(doc)
+            db.flush()
+
+            # Fingerprints
+            fp = winnowing.fingerprint(text)
+            for pos, h in enumerate(list(fp)[:500]):
+                db.add(Fingerprint(doc_id=doc.id, hash=format(h, '016x'), position=pos))
+
+            # MinHash
+            try:
+                minhash.add_document(str(doc.id), text)
+            except ValueError:
+                pass
+
+            # Elasticsearch
+            fulltext.index_document(str(doc.id), title, text, source, lang, url)
+
+            embed_batch.append((str(doc.id), text))
+            if len(embed_batch) >= BATCH:
+                emb.batch_add(embed_batch, batch_size=BATCH)
+                embed_batch.clear()
+
+            count += 1
+            if count % 500 == 0:
+                db.commit()
+                pct = 40 + min(int(count / 100000 * 55), 55)  # 40-95%
+                _pub(f"Проиндексировано: {count} документов", progress=pct, docs_done=count)
+
+            elem.clear()
+
+    if embed_batch:
+        emb.batch_add(embed_batch, batch_size=BATCH)
+
+    db.commit()
+    db.close()
+    emb.save_index()
+
+    return count
+
+
+# ── Celery tasks ───────────────────────────────────────────────────────────
+
+def _run_indexing(source_key: str, update_only: bool):
+    src = WIKIPEDIA_SOURCES[source_key]
+    label = src["label"]
+    lang = src["lang"]
+
+    _set_status({
+        "status": "running",
+        "source": source_key,
+        "label": label,
+        "progress": 0,
+        "message": "Запуск…",
+        "started_at": datetime.now(timezone.utc).isoformat(),
+        "docs_done": 0,
+    })
+
+    data_dir = Path(settings.RAW_DATA_DIR) / source_key
+    bz2_path = data_dir / "dump.xml.bz2"
+
+    # --- Download if needed ---
+    if not bz2_path.exists() or not update_only:
+        _pub(f"Скачивание дампа {label}…", progress=1)
+        try:
+            _download_file(src["url"], bz2_path, label)
+        except Exception as e:
+            _set_status({"status": "error", "message": f"Ошибка скачивания: {e}"})
+            return
+
+    _pub("Индексация…", progress=41)
+
+    try:
+        count = _index_wikipedia_bz2(bz2_path, source_key, lang, update_only)
+    except Exception as e:
+        _set_status({"status": "error", "message": f"Ошибка индексации: {e}"})
+        return
+
+    action = "обновлено" if update_only else "проиндексировано"
+    _set_status({
+        "status": "done",
+        "source": source_key,
+        "label": label,
+        "progress": 100,
+        "message": f"Готово — {action} {count} документов",
+        "finished_at": datetime.now(timezone.utc).isoformat(),
+        "docs_done": count,
+    })
+    _r().publish("index_job_progress", _r().get(JOB_KEY))
+
+
+@celery_app.task(name="tasks.download_and_index", bind=True)
+def download_and_index(self, source_key: str):
+    _run_indexing(source_key, update_only=False)
+
+
+@celery_app.task(name="tasks.update_index", bind=True)
+def update_index(self, source_key: str):
+    _run_indexing(source_key, update_only=True)
+
+
+def get_job_status() -> dict:
+    return _get_status()

backend/scripts/index_documents.py

@@ -0,0 +1,119 @@
+"""
+Index pre-parsed documents (JSON files from parse_pdf.py / parse_wikipedia.py)
+into PostgreSQL, Elasticsearch, FAISS, and MinHash LSH.
+
+Usage:
+    python scripts/index_documents.py <dir_with_json_files> [--limit N]
+"""
+import sys
+import os
+import json
+import argparse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from app.config import settings
+from app.models.db import SessionLocal, Document, Fingerprint, engine, Base
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+
+
+Base.metadata.create_all(bind=engine)
+
+
+def load_json_docs(directory: str, limit: int | None = None):
+    docs = []
+    for path in sorted(Path(directory).glob("*.json")):
+        try:
+            with open(path, encoding="utf-8") as f:
+                doc = json.load(f)
+            if doc.get("text") and doc.get("title"):
+                docs.append(doc)
+        except Exception as e:
+            print(f"Skip {path.name}: {e}")
+        if limit and len(docs) >= limit:
+            break
+    return docs
+
+
+def index_batch(docs: list, batch_size: int = 32):
+    db = SessionLocal()
+    winnowing = WinNowing()
+    minhash_lsh = MinHashLSHIndex()
+    embeddings = EmbeddingIndex()
+    fulltext = FulltextSearch()
+
+    print(f"Indexing {len(docs)} documents …")
+
+    embed_batch: list[tuple[str, str]] = []
+
+    for i, doc in enumerate(docs):
+        source = doc.get("source", "unknown")
+        ext_id = doc.get("id", "")
+        title = doc["title"]
+        text = doc["text"]
+        url = doc.get("url", "")
+        lang = doc.get("lang", "ru")
+
+        # 1. PostgreSQL
+        db_doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
+        db.add(db_doc)
+        db.flush()
+
+        # 2. Fingerprints → PostgreSQL
+        fp = winnowing.fingerprint(text)
+        for pos, h in enumerate(list(fp)[:500]):
+            db.add(Fingerprint(doc_id=db_doc.id, hash=format(h, '016x'), position=pos))
+
+        # 3. MinHash LSH (in-memory; call save separately if needed)
+        try:
+            minhash_lsh.add_document(str(db_doc.id), text)
+        except ValueError:
+            pass
+
+        # 4. Elasticsearch
+        fulltext.index_document(str(db_doc.id), title, text, source, lang, url)
+
+        # 5. Accumulate for batch embedding
+        embed_batch.append((str(db_doc.id), text))
+
+        if len(embed_batch) >= batch_size:
+            embeddings.batch_add(embed_batch, batch_size=batch_size)
+            embed_batch.clear()
+
+        if (i + 1) % 100 == 0:
+            db.commit()
+            print(f"  {i + 1}/{len(docs)}")
+
+    # Flush remaining embeddings
+    if embed_batch:
+        embeddings.batch_add(embed_batch, batch_size=batch_size)
+
+    db.commit()
+    db.close()
+
+    embeddings.save_index()
+    print("Done. FAISS index saved.")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", help="Directory with parsed JSON files")
+    parser.add_argument("--limit", type=int, default=None, help="Max documents to index")
+    parser.add_argument("--batch-size", type=int, default=32)
+    args = parser.parse_args()
+
+    docs = load_json_docs(args.directory, args.limit)
+    print(f"Loaded {len(docs)} documents from {args.directory}")
+
+    if not docs:
+        print("No documents found, exiting.")
+        sys.exit(0)
+
+    index_batch(docs, batch_size=args.batch_size)
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/normalize_text.py

@@ -0,0 +1,83 @@
+"""
+Normalize extracted text before indexing.
+
+Usage:
+    python scripts/normalize_text.py <input_dir> <output_dir>
+
+Reads JSON files produced by parse_wikipedia.py / parse_pdf.py,
+cleans the text field, writes cleaned files to output_dir.
+"""
+import sys
+import re
+import json
+import argparse
+from pathlib import Path
+
+
+def normalize(text: str) -> str:
+    # Collapse unicode whitespace and control characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
+
+    # Normalize various dash/hyphen variants to a regular hyphen
+    text = re.sub(r'[—–‒―]', '-', text)
+
+    # Remove repeated punctuation (e.g. "!!!" → "!")
+    text = re.sub(r'([!?.]){2,}', r'\1', text)
+
+    # Collapse multiple spaces / tabs to a single space
+    text = re.sub(r'[ \t]+', ' ', text)
+
+    # Collapse more than two consecutive newlines
+    text = re.sub(r'\n{3,}', '\n\n', text)
+
+    # Strip leading/trailing whitespace per line
+    lines = [line.strip() for line in text.splitlines()]
+    text = '\n'.join(lines)
+
+    return text.strip()
+
+
+def process_directory(input_dir: str, output_dir: str) -> int:
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    count = 0
+    for src in sorted(Path(input_dir).glob('*.json')):
+        try:
+            with open(src, encoding='utf-8') as f:
+                doc = json.load(f)
+        except Exception as e:
+            print(f'Skip {src.name}: {e}')
+            continue
+
+        if 'text' not in doc:
+            continue
+
+        doc['text'] = normalize(doc['text'])
+
+        if len(doc['text'].split()) < 30:
+            continue
+
+        dest = out_path / src.name
+        with open(dest, 'w', encoding='utf-8') as f:
+            json.dump(doc, f, ensure_ascii=False, indent=2)
+
+        count += 1
+        if count % 1000 == 0:
+            print(f'Normalized {count} documents')
+
+    return count
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_dir')
+    parser.add_argument('output_dir')
+    args = parser.parse_args()
+
+    count = process_directory(args.input_dir, args.output_dir)
+    print(f'Done: {count} documents written to {args.output_dir}')
+
+
+if __name__ == '__main__':
+    main()

backend/scripts/parse_pdf.py

@@ -0,0 +1,90 @@
+import os
+import json
+import pymupdf
+from pathlib import Path
+from typing import Dict, Any, List
+
+def extract_text_from_pdf(pdf_path: str) -> Dict[str, Any]:
+    """Extract text and metadata from PDF"""
+    try:
+        doc = pymupdf.open(pdf_path)
+        
+        text = ""
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text += page.get_text()
+        
+        metadata = doc.metadata
+        doc.close()
+        
+        # Basic cleaning
+        text = clean_text(text)
+        
+        return {
+            "title": metadata.get('title', Path(pdf_path).stem),
+            "text": text,
+            "author": metadata.get('author', ''),
+            "subject": metadata.get('subject', ''),
+            "num_pages": len(doc)
+        }
+    except Exception as e:
+        print(f"Error processing {pdf_path}: {e}")
+        return None
+
+def process_pdf_directory(input_dir: str, output_dir: str) -> int:
+    """Process all PDFs in a directory"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    for pdf_file in Path(input_dir).glob("*.pdf"):
+        result = extract_text_from_pdf(str(pdf_file))
+        
+        if result and result.get('text'):
+            output_file = os.path.join(output_dir, f"{pdf_file.stem}.json")
+            
+            doc = {
+                "id": f"pdf_{pdf_file.stem}",
+                "title": result['title'],
+                "text": result['text'],
+                "source": "pdf",
+                "metadata": {
+                    "author": result.get('author'),
+                    "subject": result.get('subject'),
+                    "pages": result.get('num_pages')
+                }
+            }
+            
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(doc, f, ensure_ascii=False, indent=2)
+            
+            count += 1
+            if count % 100 == 0:
+                print(f"Processed {count} PDFs")
+    
+    return count
+
+def clean_text(text: str) -> str:
+    """Clean extracted PDF text"""
+    import re
+    
+    # Remove multiple spaces
+    text = re.sub(r'  +', ' ', text)
+    # Remove multiple newlines
+    text = re.sub(r'\n\n+', '\n', text)
+    # Remove special characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_pdf.py <input_dir> <output_dir>")
+        sys.exit(1)
+    
+    input_dir = sys.argv[1]
+    output_dir = sys.argv[2]
+    
+    count = process_pdf_directory(input_dir, output_dir)
+    print(f"Successfully processed {count} PDFs")

backend/scripts/parse_wikipedia.py

@@ -0,0 +1,110 @@
+import bz2
+import xml.etree.ElementTree as ET
+import json
+import os
+from pathlib import Path
+from typing import Generator, Dict, Any
+
+def extract_text_from_wikipedia_dump(bz2_path: str, output_dir: str, limit: int = None) -> int:
+    """Extract text from Wikipedia XML dump"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    with bz2.BZ2File(bz2_path, 'r') as f:
+        context = ET.iterparse(f, events=('start', 'end'))
+        context = iter(context)
+        
+        event, root = next(context)
+        
+        for event, elem in context:
+            if event == 'end' and elem.tag.endswith('page'):
+                doc = parse_page(elem)
+                if doc and doc.get('text'):
+                    filename = f"{output_dir}/wiki_{doc['id']}.json"
+                    with open(filename, 'w', encoding='utf-8') as out:
+                        json.dump(doc, out, ensure_ascii=False)
+                    
+                    count += 1
+                    if count % 1000 == 0:
+                        print(f"Processed {count} documents")
+                    
+                    if limit and count >= limit:
+                        break
+                
+                root.clear()
+    
+    return count
+
+def parse_page(page_elem) -> Dict[str, Any]:
+    """Parse a single Wikipedia page"""
+    ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
+    
+    title_elem = page_elem.find('wiki:title', ns) or page_elem.find('.//title')
+    page_id_elem = page_elem.find('wiki:id', ns) or page_elem.find('.//id')
+    revision_elem = page_elem.find('.//revision')
+    
+    if revision_elem is None:
+        return None
+    
+    text_elem = revision_elem.find('wiki:text', ns) or revision_elem.find('.//text')
+    
+    if title_elem is None or text_elem is None:
+        return None
+    
+    title = title_elem.text or ""
+    page_id = page_id_elem.text if page_id_elem is not None else ""
+    text = text_elem.text or ""
+    
+    # Skip redirects and special pages
+    if text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
+        return None
+    
+    # Basic text cleaning
+    text = clean_wikipedia_markup(text)
+    
+    if len(text.split()) < 50:
+        return None
+    
+    return {
+        "id": f"wiki_{page_id}",
+        "title": title,
+        "text": text,
+        "source": "wikipedia",
+        "lang": "ru"  # Change for English
+    }
+
+def clean_wikipedia_markup(text: str) -> str:
+    """Remove Wikipedia markup"""
+    import re
+    
+    # Remove wiki links
+    text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
+    # Remove external links
+    text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
+    # Remove templates
+    text = re.sub(r'\{\{[^}]*\}\}', '', text)
+    # Remove categories
+    text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
+    # Remove refs
+    text = re.sub(r'<ref[^>]*>[^<]*</ref>', '', text)
+    # Remove HTML tags
+    text = re.sub(r'<[^>]*>', '', text)
+    # Clean whitespace
+    text = re.sub(r'\n\n+', '\n', text)
+    text = re.sub(r' +', ' ', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_wikipedia.py <input_bz2> <output_dir> [limit]")
+        sys.exit(1)
+    
+    input_file = sys.argv[1]
+    output_dir = sys.argv[2]
+    limit = int(sys.argv[3]) if len(sys.argv) > 3 else None
+    
+    count = extract_text_from_wikipedia_dump(input_file, output_dir, limit)
+    print(f"Successfully processed {count} documents")