From 2a14350ee3e983ad0141b40dbb17a048aae2171d Mon Sep 17 00:00:00 2001 From: jze9 Date: Mon, 18 May 2026 01:14:40 +0500 Subject: [PATCH] test build --- .env.example | 34 ++ .github/workflows/deploy.yml | 52 +++ backend/Dockerfile | 23 ++ backend/app/__init__.py | 0 backend/app/api/__init__.py | 0 backend/app/api/admin.py | 115 +++++++ backend/app/api/auth.py | 115 +++++++ backend/app/celery_app.py | 18 + backend/app/config.py | 50 +++ backend/app/core/__init__.py | 0 backend/app/core/analyzer.py | 149 ++++++++ backend/app/core/embeddings.py | 76 +++++ backend/app/core/fingerprint.py | 82 +++++ backend/app/core/fulltext.py | 135 ++++++++ backend/app/main.py | 354 ++++++++++++++++++++ backend/app/models/__init__.py | 0 backend/app/models/db.py | 82 +++++ backend/app/schemas/__init__.py | 54 +++ backend/app/tasks/__init__.py | 95 ++++++ backend/app/tasks/indexing.py | 282 ++++++++++++++++ backend/scripts/index_documents.py | 119 +++++++ backend/scripts/normalize_text.py | 83 +++++ backend/scripts/parse_pdf.py | 90 +++++ backend/scripts/parse_wikipedia.py | 110 ++++++ docker-compose.yml | 141 ++++++++ frontend/Dockerfile | 13 + frontend/index.html | 12 + frontend/nginx.conf | 52 +++ frontend/package.json | 29 ++ frontend/postcss.config.js | 6 + frontend/src/App.tsx | 98 ++++++ frontend/src/api/client.ts | 160 +++++++++ frontend/src/components/DropZone.tsx | 56 ++++ frontend/src/components/HighlightedText.tsx | 109 ++++++ frontend/src/components/MatchCard.tsx | 76 +++++ frontend/src/components/SimilarityMeter.tsx | 33 ++ frontend/src/index.css | 3 + frontend/src/main.tsx | 10 + frontend/src/pages/AdminPage.tsx | 172 ++++++++++ frontend/src/pages/AuthPage.tsx | 85 +++++ frontend/src/pages/HistoryPage.tsx | 128 +++++++ frontend/src/pages/Home.tsx | 53 +++ frontend/src/pages/Report.tsx | 206 ++++++++++++ frontend/tailwind.config.js | 8 + frontend/vite.config.ts | 15 + requirements.txt | 37 ++ 46 files changed, 3620 insertions(+) create mode 100644 .env.example create mode 100644 .github/workflows/deploy.yml create mode 100644 backend/Dockerfile create mode 100644 backend/app/__init__.py create mode 100644 backend/app/api/__init__.py create mode 100644 backend/app/api/admin.py create mode 100644 backend/app/api/auth.py create mode 100644 backend/app/celery_app.py create mode 100644 backend/app/config.py create mode 100644 backend/app/core/__init__.py create mode 100644 backend/app/core/analyzer.py create mode 100644 backend/app/core/embeddings.py create mode 100644 backend/app/core/fingerprint.py create mode 100644 backend/app/core/fulltext.py create mode 100644 backend/app/main.py create mode 100644 backend/app/models/__init__.py create mode 100644 backend/app/models/db.py create mode 100644 backend/app/schemas/__init__.py create mode 100644 backend/app/tasks/__init__.py create mode 100644 backend/app/tasks/indexing.py create mode 100644 backend/scripts/index_documents.py create mode 100644 backend/scripts/normalize_text.py create mode 100644 backend/scripts/parse_pdf.py create mode 100644 backend/scripts/parse_wikipedia.py create mode 100644 docker-compose.yml create mode 100644 frontend/Dockerfile create mode 100644 frontend/index.html create mode 100644 frontend/nginx.conf create mode 100644 frontend/package.json create mode 100644 frontend/postcss.config.js create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/api/client.ts create mode 100644 frontend/src/components/DropZone.tsx create mode 100644 frontend/src/components/HighlightedText.tsx create mode 100644 frontend/src/components/MatchCard.tsx create mode 100644 frontend/src/components/SimilarityMeter.tsx create mode 100644 frontend/src/index.css create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/pages/AdminPage.tsx create mode 100644 frontend/src/pages/AuthPage.tsx create mode 100644 frontend/src/pages/HistoryPage.tsx create mode 100644 frontend/src/pages/Home.tsx create mode 100644 frontend/src/pages/Report.tsx create mode 100644 frontend/tailwind.config.js create mode 100644 frontend/vite.config.ts create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b91a4b0 --- /dev/null +++ b/.env.example @@ -0,0 +1,34 @@ +# PostgreSQL +DATABASE_URL=postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator + +# Elasticsearch +ELASTICSEARCH_HOST=localhost +ELASTICSEARCH_PORT=9200 + +# Redis / Celery +REDIS_URL=redis://localhost:6379/0 +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/0 + +# MinIO +MINIO_ENDPOINT=localhost:9000 +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +MINIO_BUCKET=antiplagiator +MINIO_SECURE=false + +# JWT — change in production! +SECRET_KEY=change-me-in-production +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=30 + +# Analysis thresholds +FINGERPRINT_THRESHOLD=0.5 +SEMANTIC_THRESHOLD=0.6 +TOP_K_RESULTS=10 + +# Data directories +DATA_DIR=./data +RAW_DATA_DIR=./data/raw +PROCESSED_DATA_DIR=./data/processed +INDEX_DIR=./data/index diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..11738cf --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,52 @@ +name: Deploy + +on: + push: + branches: [main] + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push backend + uses: docker/build-push-action@v5 + with: + context: ./backend + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-backend:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build & push frontend + uses: docker/build-push-action@v5 + with: + context: ./frontend + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/antiplagiator-frontend:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Deploy to server via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + script: | + cd /opt/antiplagiator + docker compose pull + docker compose up -d --remove-orphans + docker image prune -f diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..57294c4 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application +COPY . . + +# Create data directories +RUN mkdir -p /app/data/{raw,processed,index} + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py new file mode 100644 index 0000000..702c456 --- /dev/null +++ b/backend/app/api/admin.py @@ -0,0 +1,115 @@ +from fastapi import APIRouter, HTTPException, status, WebSocket, WebSocketDisconnect +from pydantic import BaseModel +import asyncio +import json as _json + +router = APIRouter(prefix="/api/admin", tags=["admin"]) + +SOURCES = { + "wikipedia_ru": "Wikipedia RU (~8 GB)", + "wikipedia_en": "Wikipedia EN (~22 GB)", +} + + +class JobResponse(BaseModel): + status: str + source: str | None = None + label: str | None = None + progress: int = 0 + message: str = "" + started_at: str | None = None + finished_at: str | None = None + docs_done: int = 0 + + +@router.get("/index/sources") +def list_sources(): + """Список доступных источников для индексации.""" + return [{"key": k, "label": v} for k, v in SOURCES.items()] + + +@router.post("/index/download/{source_key}", response_model=JobResponse) +def start_download(source_key: str): + """Скачать дамп и проиндексировать с нуля.""" + if source_key not in SOURCES: + raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}") + + from app.tasks.indexing import get_job_status, download_and_index + current = get_job_status() + if current.get("status") == "running": + raise HTTPException(status_code=409, detail="Задание уже выполняется") + + download_and_index.delay(source_key) + return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск скачивания…") + + +@router.post("/index/update/{source_key}", response_model=JobResponse) +def start_update(source_key: str): + """Обновить индекс — только новые документы (дамп не скачивается заново если уже есть).""" + if source_key not in SOURCES: + raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}") + + from app.tasks.indexing import get_job_status, update_index + current = get_job_status() + if current.get("status") == "running": + raise HTTPException(status_code=409, detail="Задание уже выполняется") + + update_index.delay(source_key) + return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск обновления…") + + +@router.get("/index/status", response_model=JobResponse) +def job_status(): + """Текущий статус задания индексации.""" + from app.tasks.indexing import get_job_status + data = get_job_status() + return JobResponse( + status=data.get("status", "idle"), + source=data.get("source"), + label=data.get("label"), + progress=data.get("progress", 0), + message=data.get("message", ""), + started_at=data.get("started_at"), + finished_at=data.get("finished_at"), + docs_done=data.get("docs_done", 0), + ) + + +@router.websocket("/ws/index-progress") +async def ws_index_progress(websocket: WebSocket): + """Stream indexing job progress via Redis pub/sub.""" + await websocket.accept() + + import redis.asyncio as aioredis + from app.config import settings + + r = aioredis.from_url(settings.REDIS_URL) + pubsub = r.pubsub() + await pubsub.subscribe("index_job_progress") + + # Send current state immediately + from app.tasks.indexing import get_job_status + await websocket.send_json(get_job_status()) + + try: + async for message in pubsub.listen(): + if message["type"] != "message": + continue + try: + data = _json.loads(message["data"]) + except Exception: + continue + await websocket.send_json(data) + if data.get("status") in ("done", "error"): + break + except (WebSocketDisconnect, asyncio.CancelledError): + pass + finally: + try: + await pubsub.unsubscribe("index_job_progress") + except Exception: + pass + try: + await r.aclose() + except Exception: + pass diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py new file mode 100644 index 0000000..700b89e --- /dev/null +++ b/backend/app/api/auth.py @@ -0,0 +1,115 @@ +from datetime import datetime, timedelta +from typing import Optional +import uuid + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm +from jose import JWTError, jwt +from passlib.context import CryptContext +from pydantic import BaseModel, EmailStr +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.db import get_db, User + +router = APIRouter(prefix="/api/auth", tags=["auth"]) + +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False) + + +# ── Schemas ──────────────────────────────────────────────────────────────── + +class RegisterRequest(BaseModel): + email: EmailStr + password: str + + +class TokenResponse(BaseModel): + access_token: str + token_type: str = "bearer" + + +class UserOut(BaseModel): + id: str + email: str + created_at: datetime + + class Config: + from_attributes = True + + +# ── Helpers ──────────────────────────────────────────────────────────────── + +def hash_password(password: str) -> str: + return pwd_context.hash(password) + + +def verify_password(plain: str, hashed: str) -> bool: + return pwd_context.verify(plain, hashed) + + +def create_token(user_id) -> str: + expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES) + return jwt.encode( + {"sub": str(user_id), "exp": expire}, + settings.SECRET_KEY, + algorithm=settings.ALGORITHM, + ) + + +def get_current_user( + token: Optional[str] = Depends(oauth2_scheme), + db: Session = Depends(get_db), +) -> Optional[User]: + if not token: + return None + try: + payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM]) + user_id_str: str = payload.get("sub") + if not user_id_str: + return None + import uuid as _uuid + user_id = _uuid.UUID(user_id_str) + except (JWTError, ValueError): + return None + return db.query(User).filter(User.id == user_id).first() + + +def require_user(current_user: Optional[User] = Depends(get_current_user)) -> User: + if not current_user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": "Bearer"}, + ) + return current_user + + +# ── Endpoints ────────────────────────────────────────────────────────────── + +@router.post("/register", response_model=UserOut, status_code=status.HTTP_201_CREATED) +def register(body: RegisterRequest, db: Session = Depends(get_db)): + if db.query(User).filter(User.email == body.email).first(): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered") + user = User( + email=body.email, + hashed_password=hash_password(body.password), + ) + db.add(user) + db.commit() + db.refresh(user) + return user + + +@router.post("/login", response_model=TokenResponse) +def login(form: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)): + user = db.query(User).filter(User.email == form.username).first() + if not user or not verify_password(form.password, user.hashed_password): + raise HTTPException(status_code=400, detail="Incorrect email or password") + return {"access_token": create_token(user.id)} + + +@router.get("/me", response_model=UserOut) +def me(current_user: User = Depends(require_user)): + return current_user diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py new file mode 100644 index 0000000..beb69d9 --- /dev/null +++ b/backend/app/celery_app.py @@ -0,0 +1,18 @@ +from celery import Celery +from app.config import settings + +celery_app = Celery( + "antiplagiator", + broker=settings.CELERY_BROKER_URL, + backend=settings.CELERY_RESULT_BACKEND, + include=["app.tasks", "app.tasks.indexing"], +) + +celery_app.conf.update( + task_serializer="json", + result_serializer="json", + accept_content=["json"], + timezone="UTC", + enable_utc=True, + task_track_started=True, +) diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..8b03f2a --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,50 @@ +import os +from pydantic_settings import BaseSettings + +class Settings(BaseSettings): + # Database + DATABASE_URL: str = "postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator" + + # Elasticsearch + ELASTICSEARCH_HOST: str = "localhost" + ELASTICSEARCH_PORT: int = 9200 + + # Redis + REDIS_URL: str = "redis://localhost:6379/0" + + # MinIO + MINIO_ENDPOINT: str = "localhost:9000" + MINIO_ACCESS_KEY: str = "minioadmin" + MINIO_SECRET_KEY: str = "minioadmin" + MINIO_BUCKET: str = "antiplagiator" + MINIO_SECURE: bool = False + + # JWT + SECRET_KEY: str = "your-secret-key-change-in-prod" + ALGORITHM: str = "HS256" + ACCESS_TOKEN_EXPIRE_MINUTES: int = 30 + + # Celery + CELERY_BROKER_URL: str = "redis://localhost:6379/0" + CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0" + + # API + API_TITLE: str = "Antiplagiator API" + API_VERSION: str = "0.1.0" + API_DESCRIPTION: str = "Text plagiarism detection system" + + # Paths + DATA_DIR: str = "./data" + RAW_DATA_DIR: str = "./data/raw" + PROCESSED_DATA_DIR: str = "./data/processed" + INDEX_DIR: str = "./data/index" + + # Analysis + FINGERPRINT_THRESHOLD: float = 0.5 + SEMANTIC_THRESHOLD: float = 0.6 + TOP_K_RESULTS: int = 10 + + class Config: + env_file = ".env" + +settings = Settings() diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/analyzer.py b/backend/app/core/analyzer.py new file mode 100644 index 0000000..bbfb189 --- /dev/null +++ b/backend/app/core/analyzer.py @@ -0,0 +1,149 @@ +import re +from typing import Dict, List, Any, Tuple +from app.core.fingerprint import WinNowing, MinHashLSHIndex +from app.core.embeddings import EmbeddingIndex +from app.core.fulltext import FulltextSearch +from app.config import settings + +FRAGMENT_WORDS = 40 + + +def _split_sentences(text: str) -> List[str]: + parts = re.split(r'(?<=[.!?])\s+', text.strip()) + return [p for p in parts if p] + + +def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]: + """Return (fragment_text, char_start, char_end) around the given sentence index.""" + sentences = _split_sentences(text) + start_i = max(0, sentence_idx - context_sentences) + end_i = min(len(sentences), sentence_idx + context_sentences + 1) + + fragment = " ".join(sentences[start_i:end_i]) + char_start = text.find(sentences[start_i]) if sentences else 0 + char_end = char_start + len(fragment) + return fragment, char_start, char_end + + +def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str: + words = text.split() + if len(words) <= max_words: + return text + return " ".join(words[:max_words]) + "…" + + +class DocumentAnalyzer: + def __init__(self): + self.winnowing = WinNowing() + self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD) + self.embeddings = EmbeddingIndex() + self.fulltext = FulltextSearch() + + def analyze(self, text: str, task_id: str) -> Dict[str, Any]: + matches: List[Dict[str, Any]] = [] + matches.extend(self._level_1_exact_match(text)) + matches.extend(self._level_2_fuzzy_match(text)) + matches.extend(self._level_3_semantic_match(text)) + + return { + "task_id": task_id, + "status": "completed", + "overall_similarity": self._calculate_overall_similarity(matches), + "matches": matches[:settings.TOP_K_RESULTS], + } + + def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]: + sentences = _split_sentences(text) + if not sentences: + return [] + + results = [] + for idx, sentence in enumerate(sentences[:20]): + if len(sentence.split()) < 6: + continue + try: + es_results = self.fulltext.search_phrase(sentence, top_k=2) + for r in es_results: + r["_sentence_idx"] = idx + results.extend(es_results) + except Exception: + continue + + matches = [] + seen_docs: set = set() + for result in results: + if result["doc_id"] in seen_docs: + continue + idx = result.get("_sentence_idx", 0) + fragment, char_start, char_end = _extract_fragment(text, idx) + matches.append({ + "method": "exact", + "similarity": min(result["score"] * 20, 100), + "source_id": str(result["doc_id"]), # передаём doc_id напрямую + "source_title": result["title"], + "source_db": result["source"], + "url": result["url"], + "fragment_text": _truncate(fragment), + "fragment_start": char_start, + "fragment_end": char_end, + }) + seen_docs.add(result["doc_id"]) + + return matches[:top_k] + + def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]: + try: + fuzzy_results = self.minhash_lsh.query(text, top_k=top_k) + except Exception: + return [] + + sentences = _split_sentences(text) + fragment = _truncate(sentences[0]) if sentences else "" + matches = [] + for doc_id, similarity in fuzzy_results: + if similarity >= settings.FINGERPRINT_THRESHOLD: + matches.append({ + "method": "fuzzy", + "similarity": similarity * 100, + "source_id": doc_id, + "fragment_text": fragment, + "fragment_start": 0, + "fragment_end": len(fragment), + }) + return matches + + def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]: + try: + embedding_results = self.embeddings.search(text, top_k=top_k) + except Exception: + return [] + + sentences = _split_sentences(text) + fragment = _truncate(sentences[0]) if sentences else "" + matches = [] + for doc_id, similarity in embedding_results: + if similarity >= settings.SEMANTIC_THRESHOLD: + matches.append({ + "method": "semantic", + "similarity": similarity * 100, + "source_id": doc_id, + "fragment_text": fragment, + "fragment_start": 0, + "fragment_end": len(fragment), + }) + return matches + + def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float: + if not matches: + return 0.0 + + weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3} + total_weight = 0.0 + total_similarity = 0.0 + + for match in matches: + w = weights.get(match["method"], 0.1) + total_similarity += match["similarity"] * w + total_weight += w + + return round(total_similarity / total_weight if total_weight > 0 else 0, 1) diff --git a/backend/app/core/embeddings.py b/backend/app/core/embeddings.py new file mode 100644 index 0000000..bc08676 --- /dev/null +++ b/backend/app/core/embeddings.py @@ -0,0 +1,76 @@ +import numpy as np +import faiss +from sentence_transformers import SentenceTransformer +from typing import List, Tuple +import os +from app.config import settings + +class EmbeddingIndex: + def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"): + self.model = SentenceTransformer(model_name) + self.embedding_dim = self.model.get_sentence_embedding_dimension() + self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32) + self.doc_ids = [] + self.embeddings = None + + # Try to load existing index + self._load_index() + + def add_document(self, doc_id: str, text: str): + embedding = self.model.encode([text], convert_to_numpy=True)[0] + self.index.add(np.array([embedding], dtype=np.float32)) + self.doc_ids.append(doc_id) + + if self.embeddings is None: + self.embeddings = np.array([embedding]) + else: + self.embeddings = np.vstack([self.embeddings, embedding]) + + def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32): + doc_ids = [doc[0] for doc in docs] + texts = [doc[1] for doc in docs] + + for i in range(0, len(texts), batch_size): + batch_texts = texts[i:i+batch_size] + embeddings = self.model.encode(batch_texts, convert_to_numpy=True) + + self.index.add(embeddings.astype(np.float32)) + self.doc_ids.extend(doc_ids[i:i+batch_size]) + + if self.embeddings is None: + self.embeddings = embeddings + else: + self.embeddings = np.vstack([self.embeddings, embeddings]) + + def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]: + query_embedding = self.model.encode([text], convert_to_numpy=True)[0] + query_embedding = np.array([query_embedding], dtype=np.float32) + + distances, indices = self.index.search(query_embedding, top_k) + + results = [] + for idx, distance in zip(indices[0], distances[0]): + if idx < len(self.doc_ids): + similarity = 1.0 / (1.0 + distance) + results.append((self.doc_ids[idx], similarity)) + + return results + + def _load_index(self): + index_path = os.path.join(settings.INDEX_DIR, "faiss_index") + ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy") + + if os.path.exists(index_path) and os.path.exists(ids_path): + try: + self.index = faiss.read_index(index_path) + self.doc_ids = list(np.load(ids_path, allow_pickle=True)) + except Exception as e: + print(f"Failed to load index: {e}") + + def save_index(self): + os.makedirs(settings.INDEX_DIR, exist_ok=True) + faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index")) + np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object)) + + def get_index_size(self) -> int: + return self.index.ntotal diff --git a/backend/app/core/fingerprint.py b/backend/app/core/fingerprint.py new file mode 100644 index 0000000..7240183 --- /dev/null +++ b/backend/app/core/fingerprint.py @@ -0,0 +1,82 @@ +import xxhash +from typing import Set, Tuple, List +from datasketch import MinHash, MinHashLSH +import numpy as np + +class WinNowing: + def __init__(self, k: int = 5, window_size: int = 4): + self.k = k + self.window_size = window_size + + def get_kgrams(self, text: str) -> List[str]: + words = text.split() + kgrams = [] + for i in range(len(words) - self.k + 1): + kgram = ' '.join(words[i:i+self.k]) + kgrams.append(kgram) + return kgrams + + def fingerprint(self, text: str) -> Set[int]: + kgrams = self.get_kgrams(text) + hashes = [] + for kgram in kgrams: + h = int(xxhash.xxh64(kgram).hexdigest(), 16) + hashes.append(h) + + if not hashes: + return set() + + fingerprints = set() + for i in range(len(hashes) - self.window_size + 1): + window = hashes[i:i+self.window_size] + min_hash = min(window) + fingerprints.add(min_hash) + + return fingerprints + + def compare(self, fp1: Set[int], fp2: Set[int]) -> float: + if not fp1 or not fp2: + return 0.0 + intersection = len(fp1 & fp2) + union = len(fp1 | fp2) + return intersection / union if union > 0 else 0.0 + +class MinHashLSHIndex: + def __init__(self, num_perm: int = 128, threshold: float = 0.5): + self.num_perm = num_perm + self.threshold = threshold + self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm) + self.documents = {} + + def add_document(self, doc_id: str, text: str): + kgrams = self._get_kgrams(text) + m = MinHash(num_perm=self.num_perm) + for kgram in kgrams: + m.update(kgram.encode()) + + self.lsh.insert(doc_id, m) + self.documents[doc_id] = m + + def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]: + kgrams = self._get_kgrams(text) + query_m = MinHash(num_perm=self.num_perm) + for kgram in kgrams: + query_m.update(kgram.encode()) + + candidates = self.lsh.query(query_m) + + results = [] + for doc_id in candidates: + similarity = query_m.jaccard(self.documents[doc_id]) + results.append((doc_id, similarity)) + + results.sort(key=lambda x: x[1], reverse=True) + return results[:top_k] + + def _get_kgrams(self, text: str, k: int = 5) -> List[str]: + words = text.split() + kgrams = [] + for i in range(max(1, len(words) - k + 1)): + kgram = ' '.join(words[i:i+k]) + kgrams.append(kgram) + return kgrams diff --git a/backend/app/core/fulltext.py b/backend/app/core/fulltext.py new file mode 100644 index 0000000..1c2f3b2 --- /dev/null +++ b/backend/app/core/fulltext.py @@ -0,0 +1,135 @@ +from elasticsearch import Elasticsearch +from typing import List, Tuple, Dict, Any +from app.config import settings + +class FulltextSearch: + def __init__(self): + self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"]) + self.index_name = "documents" + self._create_index() + + def _create_index(self): + if not self.es.indices.exists(index=self.index_name): + self.es.indices.create( + index=self.index_name, + body={ + "settings": { + "analysis": { + "analyzer": { + "ru_en": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "ru_stop", "ru_stem"] + } + }, + "filter": { + "ru_stop": { + "type": "stop", + "stopwords": "_russian_" + }, + "ru_stem": { + "type": "stemmer", + "language": "russian" + } + } + } + }, + "mappings": { + "properties": { + "doc_id": {"type": "keyword"}, + "text": {"type": "text", "analyzer": "ru_en"}, + "title": {"type": "text", "analyzer": "ru_en"}, + "source": {"type": "keyword"}, + "lang": {"type": "keyword"}, + "url": {"type": "keyword"} + } + } + } + ) + + def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""): + self.es.index( + index=self.index_name, + id=doc_id, + body={ + "doc_id": doc_id, + "title": title, + "text": text, + "source": source, + "lang": lang, + "url": url + } + ) + + def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]: + must = { + "multi_match": { + "query": query, + "fields": ["text^2", "title^3"], + "type": "best_fields", + "analyzer": "ru_en", + } + } + + if lang: + search_body = { + "size": top_k, + "query": { + "bool": { + "must": must, + "filter": {"term": {"lang": lang}}, + } + }, + } + else: + search_body = {"size": top_k, "query": must} + + results = self.es.search(index=self.index_name, body=search_body) + + formatted_results = [] + for hit in results['hits']['hits']: + source = hit['_source'] + formatted_results.append({ + "doc_id": source['doc_id'], + "title": source['title'], + "source": source['source'], + "url": source.get('url', ''), + "score": hit['_score'] + }) + + return formatted_results + + def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]: + results = self.es.search( + index=self.index_name, + body={ + "size": top_k, + "query": { + "match_phrase": { + "text": phrase + } + } + } + ) + + formatted_results = [] + for hit in results['hits']['hits']: + source = hit['_source'] + formatted_results.append({ + "doc_id": source['doc_id'], + "title": source['title'], + "source": source['source'], + "url": source.get('url', ''), + "score": hit['_score'] + }) + + return formatted_results + + def bulk_index(self, documents: List[Dict[str, Any]]): + actions = [] + for doc in documents: + actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}}) + actions.append(doc) + + from elasticsearch.helpers import bulk + bulk(self.es, actions) diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..d1b2bb8 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,354 @@ +from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse +import uuid +import io +import asyncio +import json as _json +from sqlalchemy.orm import Session +from app.config import settings +from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User +from app.schemas import ( + CheckResponse, ReportResponse, ReportSummary, + ReportsListResponse, StatsResponse, HealthResponse, MatchResult, +) +from app.core.embeddings import EmbeddingIndex +from app.api.auth import router as auth_router, get_current_user +from app.api.admin import router as admin_router +import pdfplumber +import docx as python_docx +from minio import Minio +from minio.error import S3Error +from typing import Optional + +Base.metadata.create_all(bind=engine) + +app = FastAPI( + title=settings.API_TITLE, + version=settings.API_VERSION, + description=settings.API_DESCRIPTION, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(auth_router) +app.include_router(admin_router) + +_minio: Minio | None = None +_embedding_index: EmbeddingIndex | None = None + + +def get_minio() -> Minio: + global _minio + if _minio is None: + _minio = Minio( + settings.MINIO_ENDPOINT, + access_key=settings.MINIO_ACCESS_KEY, + secret_key=settings.MINIO_SECRET_KEY, + secure=settings.MINIO_SECURE, + ) + try: + if not _minio.bucket_exists(settings.MINIO_BUCKET): + _minio.make_bucket(settings.MINIO_BUCKET) + except S3Error: + pass + return _minio + + +def get_embedding_index() -> EmbeddingIndex: + global _embedding_index + if _embedding_index is None: + _embedding_index = EmbeddingIndex() + return _embedding_index + + +def extract_text(content: bytes, filename: str) -> str: + if filename.endswith(".txt"): + return content.decode("utf-8", errors="replace") + + if filename.endswith(".pdf"): + with pdfplumber.open(io.BytesIO(content)) as pdf: + return "\n".join(page.extract_text() or "" for page in pdf.pages) + + if filename.endswith(".docx"): + doc = python_docx.Document(io.BytesIO(content)) + return "\n".join(p.text for p in doc.paragraphs) + + return content.decode("utf-8", errors="replace") + + +@app.get("/api/health", response_model=HealthResponse) +async def health_check(): + return {"status": "ok", "version": settings.API_VERSION} + + +@app.post("/api/check", response_model=CheckResponse) +async def check_document( + file: UploadFile = File(...), + db: Session = Depends(get_db), + current_user: Optional[User] = Depends(get_current_user), +): + if not file.filename: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename") + + allowed = {".txt", ".pdf", ".docx"} + ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else "" + if ext not in allowed: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unsupported file type. Allowed: {', '.join(allowed)}", + ) + + task_id = uuid.uuid4() + content = await file.read() + + minio_key = f"uploads/{task_id}/{file.filename}" + try: + minio = get_minio() + minio.put_object( + settings.MINIO_BUCKET, + minio_key, + io.BytesIO(content), + length=len(content), + content_type=file.content_type or "application/octet-stream", + ) + except Exception: + minio_key = None + + try: + text = extract_text(content, file.filename) + except Exception as e: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}") + + if not text.strip(): + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable") + + task = AnalysisTask( + id=task_id, + status="processing", + source_text=text, + user_id=current_user.id if current_user else None, + ) + db.add(task) + db.commit() + + from app.tasks import analyze_document + analyze_document.delay(text, str(task_id)) + + return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"} + + +@app.get("/api/report/{task_id}", response_model=ReportResponse) +async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)): + task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first() + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + + matches = [] + if task.status == "completed": + db_matches = db.query(Match).filter(Match.task_id == task_id).all() + for m in db_matches: + doc = db.query(Document).filter(Document.id == m.doc_id).first() + matches.append( + MatchResult( + method=m.method, + similarity=float(m.similarity), + source_title=doc.title if doc else None, + source_db=doc.source if doc else None, + url=doc.url if doc else None, + fragment_text=m.fragment_text, + fragment_start=m.fragment_start, + fragment_end=m.fragment_end, + ) + ) + + return ReportResponse( + task_id=str(task.id), + status=task.status, + overall_similarity=task.overall_similarity, + matches=matches or None, + source_text=task.source_text, + created_at=task.created_at, + completed_at=task.completed_at, + ) + + +@app.get("/api/report/{task_id}/pdf") +async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)): + """Generate and return a PDF report.""" + task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first() + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + if task.status != "completed": + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet") + + db_matches = db.query(Match).filter(Match.task_id == task_id).all() + matches_data = [] + for m in db_matches: + doc = db.query(Document).filter(Document.id == m.doc_id).first() + matches_data.append({ + "method": m.method, + "similarity": m.similarity, + "source_title": doc.title if doc else "—", + "source_db": doc.source if doc else "—", + "url": doc.url if doc else "", + "fragment_text": m.fragment_text or "", + }) + + pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data) + + return StreamingResponse( + io.BytesIO(pdf_bytes), + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'}, + ) + + +def _build_pdf(task_id: str, overall: float, matches: list) -> bytes: + from reportlab.lib.pagesizes import A4 + from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle + from reportlab.lib.units import cm + from reportlab.lib import colors + from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable + from reportlab.pdfbase import pdfmetrics + from reportlab.pdfbase.ttfonts import TTFont + import os + + buf = io.BytesIO() + doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm) + styles = getSampleStyleSheet() + + METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"} + METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")} + + title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6) + h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4) + body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14) + small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey) + + story = [ + Paragraph("Отчёт о проверке на плагиат", title_style), + Paragraph(f"ID задачи: {task_id}", small_style), + Spacer(1, 0.3*cm), + HRFlowable(width="100%", thickness=1, color=colors.lightgrey), + Spacer(1, 0.3*cm), + Paragraph(f"Общий процент заимствований: {overall:.1f}%", body_style), + Spacer(1, 0.5*cm), + ] + + if not matches: + story.append(Paragraph("Совпадений не обнаружено.", body_style)) + else: + story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style)) + for i, m in enumerate(matches, 1): + bg = METHOD_COLOR.get(m["method"], colors.whitesmoke) + data = [ + [Paragraph(f"#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%", body_style), ""], + ["Источник:", Paragraph(m["source_title"], body_style)], + ["База:", m["source_db"]], + ] + if m["url"]: + data.append(["URL:", Paragraph(f'{m["url"]}', small_style)]) + if m["fragment_text"]: + data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)]) + + tbl = Table(data, colWidths=[3.5*cm, None]) + tbl.setStyle(TableStyle([ + ("BACKGROUND", (0, 0), (-1, 0), bg), + ("SPAN", (0, 0), (-1, 0)), + ("FONTSIZE", (0, 0), (-1, -1), 9), + ("VALIGN", (0, 0), (-1, -1), "TOP"), + ("TOPPADDING", (0, 0), (-1, -1), 3), + ("BOTTOMPADDING", (0, 0), (-1, -1), 3), + ("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey), + ])) + story.append(tbl) + story.append(Spacer(1, 0.3*cm)) + + doc.build(story) + return buf.getvalue() + + +@app.get("/api/reports", response_model=ReportsListResponse) +async def list_reports( + user_id: str | None = Query(default=None), + db: Session = Depends(get_db), + current_user: Optional[User] = Depends(get_current_user), +): + query = db.query(AnalysisTask) + # Авторизованный пользователь видит только свои задачи + if current_user: + query = query.filter(AnalysisTask.user_id == current_user.id) + elif user_id: + query = query.filter(AnalysisTask.user_id == user_id) + + tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all() + + return ReportsListResponse( + tasks=[ + ReportSummary( + task_id=str(t.id), + status=t.status, + overall_similarity=t.overall_similarity, + created_at=t.created_at, + ) + for t in tasks + ] + ) + + +@app.get("/api/stats", response_model=StatsResponse) +async def get_stats(db: Session = Depends(get_db)): + total_documents = db.query(Document).count() + index = get_embedding_index() + + return StatsResponse( + total_documents_indexed=total_documents, + faiss_index_size=index.get_index_size(), + elasticsearch_available=True, + api_version=settings.API_VERSION, + ) + + +@app.websocket("/ws/progress/{task_id}") +async def ws_progress(websocket: WebSocket, task_id: str): + await websocket.accept() + import redis.asyncio as aioredis + + r = aioredis.from_url(settings.REDIS_URL) + pubsub = r.pubsub() + await pubsub.subscribe(f"progress:{task_id}") + + try: + async for message in pubsub.listen(): + if message["type"] != "message": + continue + try: + data = _json.loads(message["data"]) + except Exception: + continue + await websocket.send_json(data) + if data.get("stage") in ("done", "failed"): + break + except (WebSocketDisconnect, asyncio.CancelledError): + pass + finally: + try: + await pubsub.unsubscribe(f"progress:{task_id}") + except Exception: + pass + try: + await r.aclose() + except Exception: + pass + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/models/db.py b/backend/app/models/db.py new file mode 100644 index 0000000..82a110e --- /dev/null +++ b/backend/app/models/db.py @@ -0,0 +1,82 @@ +import uuid +from sqlalchemy import Column, String, Text, DateTime, Integer, Float, ForeignKey, Index, create_engine, Boolean +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from datetime import datetime +from app.config import settings + +Base = declarative_base() + + +class User(Base): + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + hashed_password = Column(String(255), nullable=False) + is_active = Column(Boolean, default=True) + created_at = Column(DateTime, default=datetime.utcnow) + + +class Document(Base): + __tablename__ = "documents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + source = Column(String(50), nullable=False) + ext_id = Column(String(200), nullable=True) + title = Column(Text, nullable=False) + url = Column(Text, nullable=True) + lang = Column(String(2), nullable=True) + minio_key = Column(Text, nullable=True) + indexed_at = Column(DateTime, default=datetime.utcnow) + + +class Fingerprint(Base): + __tablename__ = "fingerprints" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False) + hash = Column(String(64), nullable=False) # xxhash hex string + position = Column(Integer, nullable=True) + + +Index("ix_fingerprints_hash", Fingerprint.hash) +Index("ix_fingerprints_doc_id", Fingerprint.doc_id) + + +class AnalysisTask(Base): + __tablename__ = "analysis_tasks" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + status = Column(String(20), default="pending") + overall_similarity = Column(Float, nullable=True) + source_text = Column(Text, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + completed_at = Column(DateTime, nullable=True) + + +class Match(Base): + __tablename__ = "matches" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + task_id = Column(UUID(as_uuid=True), ForeignKey("analysis_tasks.id"), nullable=False) + doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=True) + similarity = Column(Float, nullable=False) + method = Column(String(20), nullable=False) + fragment_start = Column(Integer, nullable=True) + fragment_end = Column(Integer, nullable=True) + fragment_text = Column(Text, nullable=True) + + +engine = create_engine(settings.DATABASE_URL, echo=False) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py new file mode 100644 index 0000000..06d6f56 --- /dev/null +++ b/backend/app/schemas/__init__.py @@ -0,0 +1,54 @@ +from pydantic import BaseModel, Field +from typing import Optional, List +from datetime import datetime + + +class HealthResponse(BaseModel): + status: str + version: str + + +class CheckResponse(BaseModel): + task_id: str + status: str + message: str + + +class MatchResult(BaseModel): + method: str = Field(..., description="exact | fuzzy | semantic") + similarity: float = Field(..., ge=0, le=100) + source_title: Optional[str] = None + source_db: Optional[str] = None + url: Optional[str] = None + source_id: Optional[str] = None + fragment_text: Optional[str] = None + fragment_start: Optional[int] = None + fragment_end: Optional[int] = None + + +class ReportResponse(BaseModel): + task_id: str + status: str + overall_similarity: Optional[float] = None + matches: Optional[List[MatchResult]] = None + source_text: Optional[str] = None + created_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + +class ReportSummary(BaseModel): + task_id: str + status: str + overall_similarity: Optional[float] = None + created_at: Optional[datetime] = None + + +class ReportsListResponse(BaseModel): + tasks: List[ReportSummary] + + +class StatsResponse(BaseModel): + total_documents_indexed: int + faiss_index_size: int + elasticsearch_available: bool + api_version: str diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py new file mode 100644 index 0000000..a1d5cba --- /dev/null +++ b/backend/app/tasks/__init__.py @@ -0,0 +1,95 @@ +import json +import uuid +import redis +from app.config import settings +from app.models.db import SessionLocal, AnalysisTask, Match +from app.core.analyzer import DocumentAnalyzer +from app.celery_app import celery_app +from datetime import datetime + +_analyzer: DocumentAnalyzer | None = None +_redis: redis.Redis | None = None + + +def get_analyzer() -> DocumentAnalyzer: + global _analyzer + if _analyzer is None: + _analyzer = DocumentAnalyzer() + return _analyzer + + +def get_redis() -> redis.Redis: + global _redis + if _redis is None: + _redis = redis.from_url(settings.REDIS_URL) + return _redis + + +def _publish(task_id: str, stage: str, progress: int): + try: + get_redis().publish( + f"progress:{task_id}", + json.dumps({"stage": stage, "progress": progress}), + ) + except Exception: + pass + + +@celery_app.task(bind=True, name="tasks.analyze_document") +def analyze_document(self, text: str, task_id: str): + db = SessionLocal() + try: + analyzer = get_analyzer() + + _publish(task_id, "fingerprint", 10) + exact = analyzer._level_1_exact_match(text) + + _publish(task_id, "fuzzy", 40) + fuzzy = analyzer._level_2_fuzzy_match(text) + + _publish(task_id, "semantic", 70) + semantic = analyzer._level_3_semantic_match(text) + + matches = exact + fuzzy + semantic + overall_similarity = analyzer._calculate_overall_similarity(matches) + + _publish(task_id, "saving", 90) + + task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first() + if not task: + return + + task.status = "completed" + task.overall_similarity = overall_similarity + task.completed_at = datetime.utcnow() + + for match in matches[:settings.TOP_K_RESULTS]: + raw_id = match.get("source_id") + try: + doc_id = uuid.UUID(str(raw_id)) if raw_id is not None else None + except (ValueError, TypeError, AttributeError): + doc_id = None + + db.add(Match( + task_id=task_id, + doc_id=doc_id, + similarity=float(match["similarity"]), + method=match["method"], + fragment_start=match.get("fragment_start"), + fragment_end=match.get("fragment_end"), + fragment_text=match.get("fragment_text"), + )) + + db.commit() + _publish(task_id, "done", 100) + + except Exception as exc: + task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first() + if task: + task.status = "failed" + db.commit() + _publish(task_id, "failed", 0) + raise self.retry(exc=exc, countdown=5, max_retries=2) + + finally: + db.close() diff --git a/backend/app/tasks/indexing.py b/backend/app/tasks/indexing.py new file mode 100644 index 0000000..ea7ddb0 --- /dev/null +++ b/backend/app/tasks/indexing.py @@ -0,0 +1,282 @@ +""" +Celery tasks for downloading and indexing document sources. + +Redis key "index_job" stores current job state as JSON: + { status, source, progress, message, started_at, finished_at, docs_total, docs_done } +""" +import bz2 +import json +import os +import time +import requests +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +import redis +from app.celery_app import celery_app + +from app.config import settings +from app.models.db import SessionLocal, Document, Fingerprint + +JOB_KEY = "index_job" + +WIKIPEDIA_SOURCES = { + "wikipedia_ru": { + "url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-pages-articles.xml.bz2", + "meta_url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-md5sums.txt", + "lang": "ru", + "label": "Wikipedia RU", + }, + "wikipedia_en": { + "url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2", + "meta_url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-md5sums.txt", + "lang": "en", + "label": "Wikipedia EN", + }, +} + + +# ── Redis helpers ────────────────────────────────────────────────────────── + +def _r() -> redis.Redis: + return redis.from_url(settings.REDIS_URL) + + +def _set_status(data: dict): + _r().set(JOB_KEY, json.dumps(data, default=str)) + + +def _get_status() -> dict: + raw = _r().get(JOB_KEY) + if not raw: + return {"status": "idle"} + return json.loads(raw) + + +def _pub(message: str, progress: int = 0, **extra): + state = _get_status() + state.update({"message": message, "progress": progress, **extra}) + _set_status(state) + _r().publish("index_job_progress", json.dumps(state, default=str)) + + +# ── Text helpers ─────────────────────────────────────────────────────────── + +import re + + +def _clean_wiki(text: str) -> str: + text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text) + text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text) + text = re.sub(r'\{\{[^}]*\}\}', '', text) + text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text) + text = re.sub(r']*>.*?', '', text, flags=re.DOTALL) + text = re.sub(r'<[^>]+>', '', text) + text = re.sub(r'\n{3,}', '\n\n', text) + text = re.sub(r' +', ' ', text) + return text.strip() + + +def _normalize(text: str) -> str: + text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text) + text = re.sub(r'[—–‒―]', '-', text) + text = re.sub(r'[ \t]+', ' ', text) + text = re.sub(r'\n{3,}', '\n\n', text) + return text.strip() + + +# ── Download helper ──────────────────────────────────────────────────────── + +def _download_file(url: str, dest: Path, label: str) -> Path: + """Stream-download url → dest, publishing progress.""" + dest.parent.mkdir(parents=True, exist_ok=True) + r = requests.get(url, stream=True, timeout=30) + r.raise_for_status() + total = int(r.headers.get("Content-Length", 0)) + done = 0 + last_pct = -1 + + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=1 << 20): # 1 MB + f.write(chunk) + done += len(chunk) + pct = int(done / total * 40) if total else 0 # 0-40% of overall + if pct != last_pct: + _pub(f"Скачивание {label}: {done >> 20} MB", progress=pct) + last_pct = pct + + return dest + + +# ── Indexing helper ──────────────────────────────────────────────────────── + +def _index_wikipedia_bz2(bz2_path: Path, source: str, lang: str, update_only: bool) -> int: + """ + Parse Wikipedia bz2 dump and index documents. + update_only=True → skip ext_ids already in DB. + Returns count of newly indexed docs. + """ + from app.core.fingerprint import WinNowing, MinHashLSHIndex + from app.core.embeddings import EmbeddingIndex + from app.core.fulltext import FulltextSearch + + db = SessionLocal() + winnowing = WinNowing() + minhash = MinHashLSHIndex() + emb = EmbeddingIndex() + fulltext = FulltextSearch() + + # Pre-load known ext_ids for fast skip + known: set[str] = set() + if update_only: + for (ext_id,) in db.query(Document.ext_id).filter(Document.source == source).yield_per(10_000): + if ext_id: + known.add(ext_id) + + ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'} + + embed_batch: list[tuple[str, str]] = [] + BATCH = 32 + count = 0 + parsed = 0 + + with bz2.BZ2File(str(bz2_path), 'r') as f: + context = ET.iterparse(f, events=('end',)) + for event, elem in context: + if not elem.tag.endswith('page'): + continue + + title_el = elem.find('wiki:title', ns) or elem.find('.//title') + id_el = elem.find('wiki:id', ns) or elem.find('.//id') + rev = elem.find('.//revision') + text_el = rev.find('wiki:text', ns) if rev is not None else None + if rev is None: + text_el = elem.find('.//text') + + if title_el is None or text_el is None: + elem.clear(); continue + + title = (title_el.text or '').strip() + page_id = (id_el.text or '').strip() if id_el is not None else '' + raw_text = text_el.text or '' + ext_id = f"wiki_{page_id}" + + if raw_text.startswith('#REDIRECT') or title.startswith('Wikipedia:'): + elem.clear(); continue + + text = _normalize(_clean_wiki(raw_text)) + if len(text.split()) < 50: + elem.clear(); continue + + parsed += 1 + + if update_only and ext_id in known: + elem.clear(); continue + + url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}" + + # PostgreSQL + doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang) + db.add(doc) + db.flush() + + # Fingerprints + fp = winnowing.fingerprint(text) + for pos, h in enumerate(list(fp)[:500]): + db.add(Fingerprint(doc_id=doc.id, hash=format(h, '016x'), position=pos)) + + # MinHash + try: + minhash.add_document(str(doc.id), text) + except ValueError: + pass + + # Elasticsearch + fulltext.index_document(str(doc.id), title, text, source, lang, url) + + embed_batch.append((str(doc.id), text)) + if len(embed_batch) >= BATCH: + emb.batch_add(embed_batch, batch_size=BATCH) + embed_batch.clear() + + count += 1 + if count % 500 == 0: + db.commit() + pct = 40 + min(int(count / 100000 * 55), 55) # 40-95% + _pub(f"Проиндексировано: {count} документов", progress=pct, docs_done=count) + + elem.clear() + + if embed_batch: + emb.batch_add(embed_batch, batch_size=BATCH) + + db.commit() + db.close() + emb.save_index() + + return count + + +# ── Celery tasks ─────────────────────────────────────────────────────────── + +def _run_indexing(source_key: str, update_only: bool): + src = WIKIPEDIA_SOURCES[source_key] + label = src["label"] + lang = src["lang"] + + _set_status({ + "status": "running", + "source": source_key, + "label": label, + "progress": 0, + "message": "Запуск…", + "started_at": datetime.now(timezone.utc).isoformat(), + "docs_done": 0, + }) + + data_dir = Path(settings.RAW_DATA_DIR) / source_key + bz2_path = data_dir / "dump.xml.bz2" + + # --- Download if needed --- + if not bz2_path.exists() or not update_only: + _pub(f"Скачивание дампа {label}…", progress=1) + try: + _download_file(src["url"], bz2_path, label) + except Exception as e: + _set_status({"status": "error", "message": f"Ошибка скачивания: {e}"}) + return + + _pub("Индексация…", progress=41) + + try: + count = _index_wikipedia_bz2(bz2_path, source_key, lang, update_only) + except Exception as e: + _set_status({"status": "error", "message": f"Ошибка индексации: {e}"}) + return + + action = "обновлено" if update_only else "проиндексировано" + _set_status({ + "status": "done", + "source": source_key, + "label": label, + "progress": 100, + "message": f"Готово — {action} {count} документов", + "finished_at": datetime.now(timezone.utc).isoformat(), + "docs_done": count, + }) + _r().publish("index_job_progress", _r().get(JOB_KEY)) + + +@celery_app.task(name="tasks.download_and_index", bind=True) +def download_and_index(self, source_key: str): + _run_indexing(source_key, update_only=False) + + +@celery_app.task(name="tasks.update_index", bind=True) +def update_index(self, source_key: str): + _run_indexing(source_key, update_only=True) + + +def get_job_status() -> dict: + return _get_status() diff --git a/backend/scripts/index_documents.py b/backend/scripts/index_documents.py new file mode 100644 index 0000000..f47e254 --- /dev/null +++ b/backend/scripts/index_documents.py @@ -0,0 +1,119 @@ +""" +Index pre-parsed documents (JSON files from parse_pdf.py / parse_wikipedia.py) +into PostgreSQL, Elasticsearch, FAISS, and MinHash LSH. + +Usage: + python scripts/index_documents.py [--limit N] +""" +import sys +import os +import json +import argparse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from app.config import settings +from app.models.db import SessionLocal, Document, Fingerprint, engine, Base +from app.core.fingerprint import WinNowing, MinHashLSHIndex +from app.core.embeddings import EmbeddingIndex +from app.core.fulltext import FulltextSearch + + +Base.metadata.create_all(bind=engine) + + +def load_json_docs(directory: str, limit: int | None = None): + docs = [] + for path in sorted(Path(directory).glob("*.json")): + try: + with open(path, encoding="utf-8") as f: + doc = json.load(f) + if doc.get("text") and doc.get("title"): + docs.append(doc) + except Exception as e: + print(f"Skip {path.name}: {e}") + if limit and len(docs) >= limit: + break + return docs + + +def index_batch(docs: list, batch_size: int = 32): + db = SessionLocal() + winnowing = WinNowing() + minhash_lsh = MinHashLSHIndex() + embeddings = EmbeddingIndex() + fulltext = FulltextSearch() + + print(f"Indexing {len(docs)} documents …") + + embed_batch: list[tuple[str, str]] = [] + + for i, doc in enumerate(docs): + source = doc.get("source", "unknown") + ext_id = doc.get("id", "") + title = doc["title"] + text = doc["text"] + url = doc.get("url", "") + lang = doc.get("lang", "ru") + + # 1. PostgreSQL + db_doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang) + db.add(db_doc) + db.flush() + + # 2. Fingerprints → PostgreSQL + fp = winnowing.fingerprint(text) + for pos, h in enumerate(list(fp)[:500]): + db.add(Fingerprint(doc_id=db_doc.id, hash=format(h, '016x'), position=pos)) + + # 3. MinHash LSH (in-memory; call save separately if needed) + try: + minhash_lsh.add_document(str(db_doc.id), text) + except ValueError: + pass + + # 4. Elasticsearch + fulltext.index_document(str(db_doc.id), title, text, source, lang, url) + + # 5. Accumulate for batch embedding + embed_batch.append((str(db_doc.id), text)) + + if len(embed_batch) >= batch_size: + embeddings.batch_add(embed_batch, batch_size=batch_size) + embed_batch.clear() + + if (i + 1) % 100 == 0: + db.commit() + print(f" {i + 1}/{len(docs)}") + + # Flush remaining embeddings + if embed_batch: + embeddings.batch_add(embed_batch, batch_size=batch_size) + + db.commit() + db.close() + + embeddings.save_index() + print("Done. FAISS index saved.") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory", help="Directory with parsed JSON files") + parser.add_argument("--limit", type=int, default=None, help="Max documents to index") + parser.add_argument("--batch-size", type=int, default=32) + args = parser.parse_args() + + docs = load_json_docs(args.directory, args.limit) + print(f"Loaded {len(docs)} documents from {args.directory}") + + if not docs: + print("No documents found, exiting.") + sys.exit(0) + + index_batch(docs, batch_size=args.batch_size) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/normalize_text.py b/backend/scripts/normalize_text.py new file mode 100644 index 0000000..5cbe759 --- /dev/null +++ b/backend/scripts/normalize_text.py @@ -0,0 +1,83 @@ +""" +Normalize extracted text before indexing. + +Usage: + python scripts/normalize_text.py + +Reads JSON files produced by parse_wikipedia.py / parse_pdf.py, +cleans the text field, writes cleaned files to output_dir. +""" +import sys +import re +import json +import argparse +from pathlib import Path + + +def normalize(text: str) -> str: + # Collapse unicode whitespace and control characters + text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text) + + # Normalize various dash/hyphen variants to a regular hyphen + text = re.sub(r'[—–‒―]', '-', text) + + # Remove repeated punctuation (e.g. "!!!" → "!") + text = re.sub(r'([!?.]){2,}', r'\1', text) + + # Collapse multiple spaces / tabs to a single space + text = re.sub(r'[ \t]+', ' ', text) + + # Collapse more than two consecutive newlines + text = re.sub(r'\n{3,}', '\n\n', text) + + # Strip leading/trailing whitespace per line + lines = [line.strip() for line in text.splitlines()] + text = '\n'.join(lines) + + return text.strip() + + +def process_directory(input_dir: str, output_dir: str) -> int: + out_path = Path(output_dir) + out_path.mkdir(parents=True, exist_ok=True) + + count = 0 + for src in sorted(Path(input_dir).glob('*.json')): + try: + with open(src, encoding='utf-8') as f: + doc = json.load(f) + except Exception as e: + print(f'Skip {src.name}: {e}') + continue + + if 'text' not in doc: + continue + + doc['text'] = normalize(doc['text']) + + if len(doc['text'].split()) < 30: + continue + + dest = out_path / src.name + with open(dest, 'w', encoding='utf-8') as f: + json.dump(doc, f, ensure_ascii=False, indent=2) + + count += 1 + if count % 1000 == 0: + print(f'Normalized {count} documents') + + return count + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir') + parser.add_argument('output_dir') + args = parser.parse_args() + + count = process_directory(args.input_dir, args.output_dir) + print(f'Done: {count} documents written to {args.output_dir}') + + +if __name__ == '__main__': + main() diff --git a/backend/scripts/parse_pdf.py b/backend/scripts/parse_pdf.py new file mode 100644 index 0000000..05cca1c --- /dev/null +++ b/backend/scripts/parse_pdf.py @@ -0,0 +1,90 @@ +import os +import json +import pymupdf +from pathlib import Path +from typing import Dict, Any, List + +def extract_text_from_pdf(pdf_path: str) -> Dict[str, Any]: + """Extract text and metadata from PDF""" + try: + doc = pymupdf.open(pdf_path) + + text = "" + for page_num in range(len(doc)): + page = doc[page_num] + text += page.get_text() + + metadata = doc.metadata + doc.close() + + # Basic cleaning + text = clean_text(text) + + return { + "title": metadata.get('title', Path(pdf_path).stem), + "text": text, + "author": metadata.get('author', ''), + "subject": metadata.get('subject', ''), + "num_pages": len(doc) + } + except Exception as e: + print(f"Error processing {pdf_path}: {e}") + return None + +def process_pdf_directory(input_dir: str, output_dir: str) -> int: + """Process all PDFs in a directory""" + os.makedirs(output_dir, exist_ok=True) + + count = 0 + for pdf_file in Path(input_dir).glob("*.pdf"): + result = extract_text_from_pdf(str(pdf_file)) + + if result and result.get('text'): + output_file = os.path.join(output_dir, f"{pdf_file.stem}.json") + + doc = { + "id": f"pdf_{pdf_file.stem}", + "title": result['title'], + "text": result['text'], + "source": "pdf", + "metadata": { + "author": result.get('author'), + "subject": result.get('subject'), + "pages": result.get('num_pages') + } + } + + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(doc, f, ensure_ascii=False, indent=2) + + count += 1 + if count % 100 == 0: + print(f"Processed {count} PDFs") + + return count + +def clean_text(text: str) -> str: + """Clean extracted PDF text""" + import re + + # Remove multiple spaces + text = re.sub(r' +', ' ', text) + # Remove multiple newlines + text = re.sub(r'\n\n+', '\n', text) + # Remove special characters + text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text) + + return text.strip() + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 3: + print("Usage: python parse_pdf.py ") + sys.exit(1) + + input_dir = sys.argv[1] + output_dir = sys.argv[2] + + count = process_pdf_directory(input_dir, output_dir) + print(f"Successfully processed {count} PDFs") diff --git a/backend/scripts/parse_wikipedia.py b/backend/scripts/parse_wikipedia.py new file mode 100644 index 0000000..cf31187 --- /dev/null +++ b/backend/scripts/parse_wikipedia.py @@ -0,0 +1,110 @@ +import bz2 +import xml.etree.ElementTree as ET +import json +import os +from pathlib import Path +from typing import Generator, Dict, Any + +def extract_text_from_wikipedia_dump(bz2_path: str, output_dir: str, limit: int = None) -> int: + """Extract text from Wikipedia XML dump""" + os.makedirs(output_dir, exist_ok=True) + + count = 0 + with bz2.BZ2File(bz2_path, 'r') as f: + context = ET.iterparse(f, events=('start', 'end')) + context = iter(context) + + event, root = next(context) + + for event, elem in context: + if event == 'end' and elem.tag.endswith('page'): + doc = parse_page(elem) + if doc and doc.get('text'): + filename = f"{output_dir}/wiki_{doc['id']}.json" + with open(filename, 'w', encoding='utf-8') as out: + json.dump(doc, out, ensure_ascii=False) + + count += 1 + if count % 1000 == 0: + print(f"Processed {count} documents") + + if limit and count >= limit: + break + + root.clear() + + return count + +def parse_page(page_elem) -> Dict[str, Any]: + """Parse a single Wikipedia page""" + ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'} + + title_elem = page_elem.find('wiki:title', ns) or page_elem.find('.//title') + page_id_elem = page_elem.find('wiki:id', ns) or page_elem.find('.//id') + revision_elem = page_elem.find('.//revision') + + if revision_elem is None: + return None + + text_elem = revision_elem.find('wiki:text', ns) or revision_elem.find('.//text') + + if title_elem is None or text_elem is None: + return None + + title = title_elem.text or "" + page_id = page_id_elem.text if page_id_elem is not None else "" + text = text_elem.text or "" + + # Skip redirects and special pages + if text.startswith('#REDIRECT') or title.startswith('Wikipedia:'): + return None + + # Basic text cleaning + text = clean_wikipedia_markup(text) + + if len(text.split()) < 50: + return None + + return { + "id": f"wiki_{page_id}", + "title": title, + "text": text, + "source": "wikipedia", + "lang": "ru" # Change for English + } + +def clean_wikipedia_markup(text: str) -> str: + """Remove Wikipedia markup""" + import re + + # Remove wiki links + text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text) + # Remove external links + text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text) + # Remove templates + text = re.sub(r'\{\{[^}]*\}\}', '', text) + # Remove categories + text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text) + # Remove refs + text = re.sub(r']*>[^<]*', '', text) + # Remove HTML tags + text = re.sub(r'<[^>]*>', '', text) + # Clean whitespace + text = re.sub(r'\n\n+', '\n', text) + text = re.sub(r' +', ' ', text) + + return text.strip() + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 3: + print("Usage: python parse_wikipedia.py [limit]") + sys.exit(1) + + input_file = sys.argv[1] + output_dir = sys.argv[2] + limit = int(sys.argv[3]) if len(sys.argv) > 3 else None + + count = extract_text_from_wikipedia_dump(input_file, output_dir, limit) + print(f"Successfully processed {count} documents") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..263884f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,141 @@ +version: '3.8' + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: antiplagiator + POSTGRES_USER: antiplagiator + POSTGRES_PASSWORD: antiplagiator_dev + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U antiplagiator"] + interval: 10s + timeout: 5s + retries: 5 + + elasticsearch: + image: elasticsearch:8.13.0 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ports: + - "9200:9200" + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9200 | grep -q cluster_name"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + minio: + image: minio/minio + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 + + backend: + build: + context: ./backend + dockerfile: Dockerfile + ports: + - "8000:8000" + environment: + DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator + ELASTICSEARCH_HOST: elasticsearch + ELASTICSEARCH_PORT: 9200 + REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + MINIO_ENDPOINT: minio:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: antiplagiator + MINIO_SECURE: "false" + volumes: + - backend_data:/app/data + depends_on: + postgres: + condition: service_healthy + elasticsearch: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + restart: unless-stopped + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + ports: + - "80:80" + depends_on: + - backend + restart: unless-stopped + + celery-worker: + build: + context: ./backend + dockerfile: Dockerfile + command: celery -A app.celery_app worker --loglevel=info --concurrency=2 + environment: + DATABASE_URL: postgresql://antiplagiator:antiplagiator_dev@postgres:5432/antiplagiator + ELASTICSEARCH_HOST: elasticsearch + ELASTICSEARCH_PORT: 9200 + REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + MINIO_ENDPOINT: minio:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: antiplagiator + MINIO_SECURE: "false" + volumes: + - backend_data:/app/data + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + elasticsearch_data: + redis_data: + minio_data: + backend_data: + +networks: + default: + name: antiplagiator_network diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..f58cf6e --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,13 @@ +FROM node:20-alpine AS builder + +WORKDIR /app +COPY package.json ./ +RUN npm install + +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=builder /app/dist /usr/share/nginx/html +COPY nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..4b6c0ac --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + Антиплагиат + + +
+ + + diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 0000000..85cb24c --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,52 @@ +server { + listen 80; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + # Gzip + gzip on; + gzip_types text/plain text/css application/javascript application/json image/svg+xml; + gzip_min_length 1024; + gzip_vary on; + + # Cache static assets (JS/CSS с хешами в имени — кешируем надолго) + location ~* \.(js|css|woff2?|ttf|svg|png|jpg|ico)$ { + expires 1y; + add_header Cache-Control "public, immutable"; + try_files $uri =404; + } + + # React SPA — все неизвестные пути → index.html (не кешируем) + location / { + add_header Cache-Control "no-store"; + try_files $uri $uri/ /index.html; + } + + # Proxy REST API → FastAPI + location /api/ { + proxy_pass http://backend:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_read_timeout 120s; + proxy_send_timeout 120s; + } + + # Proxy WebSocket → FastAPI (/ws/ и /api/admin/ws/) + location ~ ^/(ws/|api/admin/ws/) { + proxy_pass http://backend:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_read_timeout 3600s; + } + + # MinIO console + location /minio/ { + proxy_pass http://minio:9001/; + proxy_set_header Host $host; + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..b24492a --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,29 @@ +{ + "name": "antiplagiator-frontend", + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "lint": "eslint . --ext .ts,.tsx" + }, + "dependencies": { + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-dropzone": "^14.2.3", + "zustand": "^4.4.0", + "axios": "^1.6.0", + "react-query": "^3.39.3", + "tailwindcss": "^3.3.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", + "@vitejs/plugin-react": "^4.2.0", + "vite": "^5.0.0", + "typescript": "^5.3.0", + "autoprefixer": "^10.4.16", + "postcss": "^8.4.31" + } +} diff --git a/frontend/postcss.config.js b/frontend/postcss.config.js new file mode 100644 index 0000000..2e7af2b --- /dev/null +++ b/frontend/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..3fcd3bb --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,98 @@ +import { useEffect, useState } from 'react' +import AuthPage from './pages/AuthPage' +import Home from './pages/Home' +import Report from './pages/Report' +import HistoryPage from './pages/HistoryPage' +import AdminPage from './pages/AdminPage' +import { getMe } from './api/client' + +type Screen = 'home' | 'report' | 'history' | 'admin' + +export default function App() { + const [authed, setAuthed] = useState(null) // null = checking + const [screen, setScreen] = useState('home') + const [taskId, setTaskId] = useState(null) + + // Check stored token on mount + useEffect(() => { + const token = localStorage.getItem('token') + if (!token) { setAuthed(false); return } + getMe() + .then(() => setAuthed(true)) + .catch(() => { localStorage.removeItem('token'); setAuthed(false) }) + }, []) + + const handleAuth = (token: string) => { + if (token) localStorage.setItem('token', token) + setAuthed(true) + } + + const handleLogout = () => { + localStorage.removeItem('token') + setAuthed(false) + setScreen('home') + setTaskId(null) + } + + const openReport = (id: string) => { + setTaskId(id) + setScreen('report') + } + + // Loading state while checking token + if (authed === null) { + return ( +
+ Загрузка… +
+ ) + } + + if (!authed) { + return + } + + return ( +
+ {/* Top nav — only on home/history */} + {screen !== 'report' && ( + + )} + + {screen === 'home' && ( + + )} + + {screen === 'history' && ( + setScreen('home')} + /> + )} + + {screen === 'admin' && } + + {screen === 'report' && taskId && ( + setScreen('home')} + /> + )} +
+ ) +} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts new file mode 100644 index 0000000..5c5e576 --- /dev/null +++ b/frontend/src/api/client.ts @@ -0,0 +1,160 @@ +import axios from 'axios' + +const api = axios.create({ baseURL: '/api' }) + +// Attach token from localStorage on every request +api.interceptors.request.use((config) => { + const token = localStorage.getItem('token') + if (token) config.headers.Authorization = `Bearer ${token}` + return config +}) + +export interface CheckResponse { + task_id: string + status: string + message: string +} + +export interface MatchResult { + method: string + similarity: number + source_title?: string + source_db?: string + url?: string + source_id?: string +} + +export interface ReportResponse { + task_id: string + status: string + overall_similarity?: number + matches?: MatchResult[] + source_text?: string + created_at?: string + completed_at?: string +} + +export interface ProgressEvent { + stage: 'fingerprint' | 'fuzzy' | 'semantic' | 'saving' | 'done' | 'failed' + progress: number +} + +export function openProgressSocket(taskId: string, onEvent: (e: ProgressEvent) => void): () => void { + const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws' + const ws = new WebSocket(`${protocol}://${window.location.host}/ws/progress/${taskId}`) + ws.onmessage = (e) => { + try { onEvent(JSON.parse(e.data)) } catch {} + } + return () => ws.close() +} + +export interface ReportSummary { + task_id: string + status: string + overall_similarity?: number + created_at?: string +} + +export const uploadDocument = async (file: File): Promise => { + const form = new FormData() + form.append('file', file) + const res = await api.post('/check', form) + return res.data +} + +export const getReport = async (taskId: string): Promise => { + const res = await api.get(`/report/${taskId}`) + return res.data +} + +export const listReports = async (): Promise => { + const res = await api.get<{ tasks: ReportSummary[] }>('/reports') + return res.data.tasks +} + +export interface UserInfo { + id: string + email: string + created_at: string +} + +export const register = async (email: string, password: string): Promise => { + await api.post('/auth/register', { email, password }) +} + +export const login = async (email: string, password: string): Promise => { + const form = new URLSearchParams({ username: email, password }) + const res = await api.post<{ access_token: string }>('/auth/login', form, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + }) + return res.data.access_token +} + +export const getMe = async (): Promise => { + const res = await api.get('/auth/me') + return res.data +} + +// ── Admin / Index ────────────────────────────────────────────────────────── + +export interface IndexSource { + key: string + label: string +} + +export interface JobStatus { + status: string + source?: string + label?: string + progress: number + message: string + started_at?: string + finished_at?: string + docs_done: number +} + +export const getIndexSources = async (): Promise => { + const res = await api.get('/admin/index/sources') + return res.data +} + +export const startDownload = async (sourceKey: string): Promise => { + const res = await api.post(`/admin/index/download/${sourceKey}`) + return res.data +} + +export const startUpdate = async (sourceKey: string): Promise => { + const res = await api.post(`/admin/index/update/${sourceKey}`) + return res.data +} + +export const getJobStatus = async (): Promise => { + const res = await api.get('/admin/index/status') + return res.data +} + +export function openIndexSocket(onEvent: (e: JobStatus) => void): () => void { + const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws' + const ws = new WebSocket(`${protocol}://${window.location.host}/api/admin/ws/index-progress`) + ws.onmessage = (e) => { + try { onEvent(JSON.parse(e.data)) } catch {} + } + return () => ws.close() +} + +export const downloadReportPdf = async (taskId: string): Promise => { + const token = localStorage.getItem('token') + const res = await fetch(`/api/report/${taskId}/pdf`, { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }) + if (!res.ok) throw new Error('PDF download failed') + const blob = await res.blob() + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `report-${taskId.slice(0, 8)}.pdf` + document.body.appendChild(a) + a.click() + document.body.removeChild(a) + URL.revokeObjectURL(url) +} diff --git a/frontend/src/components/DropZone.tsx b/frontend/src/components/DropZone.tsx new file mode 100644 index 0000000..7739fb9 --- /dev/null +++ b/frontend/src/components/DropZone.tsx @@ -0,0 +1,56 @@ +import { useCallback, useState } from 'react' +import { useDropzone } from 'react-dropzone' + +interface Props { + onFile: (file: File) => void + disabled?: boolean +} + +const ACCEPTED = { 'text/plain': ['.txt'], 'application/pdf': ['.pdf'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'] } + +export default function DropZone({ onFile, disabled }: Props) { + const [error, setError] = useState(null) + + const onDrop = useCallback((accepted: File[], rejected: { errors: { message: string }[] }[]) => { + setError(null) + if (rejected.length) { + setError('Поддерживаются только .txt, .pdf, .docx') + return + } + if (accepted[0]) onFile(accepted[0]) + }, [onFile]) + + const { getRootProps, getInputProps, isDragActive, acceptedFiles } = useDropzone({ + onDrop, + accept: ACCEPTED, + maxFiles: 1, + disabled, + }) + + const file = acceptedFiles[0] + + return ( +
+
+ +

+ {isDragActive + ? 'Отпустите файл здесь…' + : 'Перетащите файл или нажмите для выбора'} +

+

.txt, .pdf, .docx

+ {file && ( +

{file.name}

+ )} +
+ {error &&

{error}

} +
+ ) +} diff --git a/frontend/src/components/HighlightedText.tsx b/frontend/src/components/HighlightedText.tsx new file mode 100644 index 0000000..f2e45e0 --- /dev/null +++ b/frontend/src/components/HighlightedText.tsx @@ -0,0 +1,109 @@ +import { useEffect, useRef } from 'react' +import { MatchResult } from '../api/client' +import { HIGHLIGHT_COLOR } from './MatchCard' + +interface Span { + start: number + end: number + method: string + matchIndex: number +} + +interface Props { + text: string + matches: MatchResult[] + activeIndex: number | null + onMatchClick: (index: number) => void +} + +function buildSegments(text: string, spans: Span[]) { + // Sort by start, then merge/flatten overlapping spans keeping the one with highest priority + const priority: Record = { exact: 3, fuzzy: 2, semantic: 1 } + + // Build a char-level annotation array (sparse) + const events: Array<{ pos: number; type: 'open' | 'close'; span: Span }> = [] + for (const s of spans) { + events.push({ pos: s.start, type: 'open', span: s }) + events.push({ pos: s.end, type: 'close', span: s }) + } + events.sort((a, b) => a.pos - b.pos || (a.type === 'close' ? -1 : 1)) + + const segments: Array<{ text: string; span: Span | null }> = [] + let cursor = 0 + const active = new Map() + + const dominant = () => { + let best: Span | null = null + for (const s of active.keys()) { + if (!best || (priority[s.method] ?? 0) > (priority[best.method] ?? 0)) best = s + } + return best + } + + for (const ev of events) { + if (ev.pos > cursor) { + segments.push({ text: text.slice(cursor, ev.pos), span: dominant() }) + cursor = ev.pos + } + if (ev.type === 'open') active.set(ev.span, true) + else active.delete(ev.span) + } + + if (cursor < text.length) { + segments.push({ text: text.slice(cursor), span: null }) + } + + return segments +} + +export default function HighlightedText({ text, matches, activeIndex, onMatchClick }: Props) { + const spanRefs = useRef>(new Map()) + + // Scroll to active match + useEffect(() => { + if (activeIndex === null) return + const el = spanRefs.current.get(activeIndex) + el?.scrollIntoView({ behavior: 'smooth', block: 'center' }) + }, [activeIndex]) + + const spans: Span[] = matches + .map((m, i) => ({ + start: m.fragment_start ?? 0, + end: m.fragment_end ?? 0, + method: m.method, + matchIndex: i, + })) + .filter((s) => s.end > s.start) + + const segments = buildSegments(text, spans) + + return ( +
+ {segments.map((seg, i) => { + if (!seg.span) { + return {seg.text} + } + + const mi = seg.span.matchIndex + const isActive = mi === activeIndex + const color = HIGHLIGHT_COLOR[seg.span.method] ?? 'bg-gray-200' + + return ( + { if (el) spanRefs.current.set(mi, el) }} + onClick={() => onMatchClick(mi)} + className={[ + color, + 'cursor-pointer rounded px-0.5 transition-all', + isActive ? 'ring-2 ring-blue-500 brightness-90' : 'hover:brightness-90', + ].join(' ')} + title={`${matches[mi]?.source_title ?? ''} — ${matches[mi]?.similarity.toFixed(1)}%`} + > + {seg.text} + + ) + })} +
+ ) +} diff --git a/frontend/src/components/MatchCard.tsx b/frontend/src/components/MatchCard.tsx new file mode 100644 index 0000000..0219b8a --- /dev/null +++ b/frontend/src/components/MatchCard.tsx @@ -0,0 +1,76 @@ +import { MatchResult } from '../api/client' + +const METHOD_LABEL: Record = { + exact: 'Точное', + fuzzy: 'Нечёткое', + semantic: 'Семантика', +} + +const METHOD_COLOR: Record = { + exact: 'bg-red-100 text-red-700 border-red-200', + fuzzy: 'bg-orange-100 text-orange-700 border-orange-200', + semantic: 'bg-yellow-100 text-yellow-700 border-yellow-200', +} + +const HIGHLIGHT_COLOR: Record = { + exact: 'bg-red-200', + fuzzy: 'bg-orange-200', + semantic: 'bg-yellow-200', +} + +interface Props { + match: MatchResult + index: number + active: boolean + onClick: () => void +} + +export { HIGHLIGHT_COLOR } + +export default function MatchCard({ match, index, active, onClick }: Props) { + const colorClass = METHOD_COLOR[match.method] ?? 'bg-gray-100 text-gray-700 border-gray-200' + + return ( + + ) +} diff --git a/frontend/src/components/SimilarityMeter.tsx b/frontend/src/components/SimilarityMeter.tsx new file mode 100644 index 0000000..32c0671 --- /dev/null +++ b/frontend/src/components/SimilarityMeter.tsx @@ -0,0 +1,33 @@ +interface Props { + value: number +} + +function color(v: number) { + if (v < 20) return 'bg-green-500' + if (v < 50) return 'bg-yellow-400' + return 'bg-red-500' +} + +function label(v: number) { + if (v < 20) return 'Оригинально' + if (v < 50) return 'Умеренное сходство' + return 'Высокое заимствование' +} + +export default function SimilarityMeter({ value }: Props) { + const pct = Math.min(Math.max(value, 0), 100) + return ( +
+
+ {label(pct)} + {pct.toFixed(1)}% +
+
+
+
+
+ ) +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 0000000..b5c61c9 --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000..964aeb4 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,10 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import App from './App' +import './index.css' + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx new file mode 100644 index 0000000..0456d07 --- /dev/null +++ b/frontend/src/pages/AdminPage.tsx @@ -0,0 +1,172 @@ +import { useEffect, useState } from 'react' +import { + getIndexSources, startDownload, startUpdate, + getJobStatus, openIndexSocket, + IndexSource, JobStatus, +} from '../api/client' + +const STATUS_LABEL: Record = { + idle: 'Ожидание', + running: 'Выполняется', + done: 'Готово', + error: 'Ошибка', + started: 'Запуск…', +} + +const STATUS_COLOR: Record = { + idle: 'text-gray-400', + running: 'text-blue-600', + done: 'text-green-600', + error: 'text-red-600', + started: 'text-blue-500', +} + +const SOURCE_SIZE: Record = { + wikipedia_ru: '~8 GB', + wikipedia_en: '~22 GB', +} + +const SOURCE_DESC: Record = { + wikipedia_ru: 'Русскоязычные статьи Википедии', + wikipedia_en: 'Англоязычные статьи Википедии', +} + +export default function AdminPage() { + const [sources, setSources] = useState([]) + const [job, setJob] = useState({ status: 'idle', progress: 0, message: '', docs_done: 0 }) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + + // Load sources + current status + useEffect(() => { + getIndexSources().then(setSources).catch(() => {}) + getJobStatus().then(setJob).catch(() => {}) + }, []) + + // Subscribe to WebSocket progress + useEffect(() => { + const close = openIndexSocket((ev) => { + setJob(ev) + if (ev.status !== 'running') setBusy(false) + }) + return close + }, []) + + const isRunning = job.status === 'running' || busy + + const handleDownload = async (key: string) => { + setError(null) + setBusy(true) + try { + const j = await startDownload(key) + setJob(j) + } catch (e: unknown) { + const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail + setError(detail ?? 'Ошибка запуска') + setBusy(false) + } + } + + const handleUpdate = async (key: string) => { + setError(null) + setBusy(true) + try { + const j = await startUpdate(key) + setJob(j) + } catch (e: unknown) { + const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail + setError(detail ?? 'Ошибка запуска') + setBusy(false) + } + } + + return ( +
+
+

Управление базой документов

+ + {/* Current job status */} +
+
+ Статус задания + + {STATUS_LABEL[job.status] ?? job.status} + {job.label ? ` — ${job.label}` : ''} + +
+ + {job.message && ( +

{job.message}

+ )} + + {isRunning && ( +
+
+ {job.docs_done > 0 ? `Проиндексировано: ${job.docs_done.toLocaleString()} документов` : ''} + {job.progress}% +
+
+
+
+
+ )} + + {job.status === 'done' && job.finished_at && ( +

+ Завершено: {new Date(job.finished_at).toLocaleString('ru-RU')} +

+ )} +
+ + {error && ( +
+ {error} +
+ )} + + {/* Source cards */} +
+ {sources.map((src) => ( +
+
+

{src.label}

+

{SOURCE_DESC[src.key] ?? ''}

+

{SOURCE_SIZE[src.key] ?? ''}

+
+ +
+ + +
+
+ ))} +
+ +
+

Скачать и индексировать — скачивает свежий дамп (~часы) и строит индекс с нуля.

+

Обновить базу — использует уже скачанный дамп, добавляет только документы, которых ещё нет в базе. Быстро после первой загрузки.

+
+
+
+ ) +} diff --git a/frontend/src/pages/AuthPage.tsx b/frontend/src/pages/AuthPage.tsx new file mode 100644 index 0000000..e38b93e --- /dev/null +++ b/frontend/src/pages/AuthPage.tsx @@ -0,0 +1,85 @@ +import { useState } from 'react' +import { login, register } from '../api/client' + +interface Props { + onAuth: (token: string) => void +} + +export default function AuthPage({ onAuth }: Props) { + const [mode, setMode] = useState<'login' | 'register'>('login') + const [email, setEmail] = useState('') + const [password, setPassword] = useState('') + const [error, setError] = useState(null) + const [loading, setLoading] = useState(false) + + const handle = async () => { + setError(null) + setLoading(true) + try { + if (mode === 'register') { + await register(email, password) + } + const token = await login(email, password) + localStorage.setItem('token', token) + onAuth(token) + } catch (e: unknown) { + const detail = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail + setError(detail ?? 'Ошибка') + } finally { + setLoading(false) + } + } + + return ( +
+
+

Антиплагиат

+ +
+ {(['login', 'register'] as const).map((m) => ( + + ))} +
+ + setEmail(e.target.value)} + className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400" + /> + setPassword(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && handle()} + className="border rounded-xl px-4 py-2.5 text-sm outline-none focus:ring-2 focus:ring-blue-400" + /> + + {error &&

{error}

} + + + + +
+
+ ) +} diff --git a/frontend/src/pages/HistoryPage.tsx b/frontend/src/pages/HistoryPage.tsx new file mode 100644 index 0000000..c771949 --- /dev/null +++ b/frontend/src/pages/HistoryPage.tsx @@ -0,0 +1,128 @@ +import { useEffect, useState } from 'react' +import { listReports, downloadReportPdf, ReportSummary } from '../api/client' + +interface Props { + onOpen: (taskId: string) => void + onNewCheck: () => void +} + +function statusBadge(status: string) { + const map: Record = { + completed: 'bg-green-100 text-green-700', + processing: 'bg-blue-100 text-blue-700', + failed: 'bg-red-100 text-red-700', + pending: 'bg-gray-100 text-gray-500', + } + const label: Record = { + completed: 'Готово', + processing: 'Анализ…', + failed: 'Ошибка', + pending: 'Ожидание', + } + return ( + + {label[status] ?? status} + + ) +} + +function similarityColor(v?: number | null) { + if (v == null) return 'text-gray-400' + if (v < 20) return 'text-green-600' + if (v < 50) return 'text-yellow-600' + return 'text-red-600' +} + +export default function HistoryPage({ onOpen, onNewCheck }: Props) { + const [tasks, setTasks] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + + useEffect(() => { + listReports() + .then(setTasks) + .catch(() => setError('Не удалось загрузить историю')) + .finally(() => setLoading(false)) + }, []) + + return ( +
+
+

История проверок

+ +
+ +
+ {loading && ( +
Загрузка…
+ )} + + {error && ( +
+ {error} +
+ )} + + {!loading && !error && tasks.length === 0 && ( +
+

Проверок пока нет

+ +
+ )} + + {tasks.length > 0 && ( +
    + {tasks.map((t) => ( +
  • + {/* Similarity circle */} +
    + {t.overall_similarity != null ? `${t.overall_similarity}%` : '—'} +
    + + {/* Info */} +
    +
    + {statusBadge(t.status)} + + {t.created_at ? new Date(t.created_at).toLocaleString('ru-RU') : ''} + +
    +

    {t.task_id}

    +
    + + {/* Actions */} +
    + {t.status === 'completed' && ( + + )} + +
    +
    • + ))} +
+ )} +
+
+ ) +} diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx new file mode 100644 index 0000000..83a852a --- /dev/null +++ b/frontend/src/pages/Home.tsx @@ -0,0 +1,53 @@ +import { useState } from 'react' +import DropZone from '../components/DropZone' +import { uploadDocument } from '../api/client' + +interface Props { + onTaskCreated: (taskId: string) => void +} + +export default function Home({ onTaskCreated }: Props) { + const [file, setFile] = useState(null) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + const handleSubmit = async () => { + if (!file) return + setLoading(true) + setError(null) + try { + const res = await uploadDocument(file) + onTaskCreated(res.task_id) + } catch (e: unknown) { + const msg = (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail ?? 'Ошибка загрузки' + setError(msg) + } finally { + setLoading(false) + } + } + + return ( +
+
+

Антиплагиат

+

+ Загрузите документ для проверки на заимствования +

+ + + + {error && ( +

{error}

+ )} + + +
+
+ ) +} diff --git a/frontend/src/pages/Report.tsx b/frontend/src/pages/Report.tsx new file mode 100644 index 0000000..3d81765 --- /dev/null +++ b/frontend/src/pages/Report.tsx @@ -0,0 +1,206 @@ +import { useEffect, useState, useCallback, useRef } from 'react' +import { getReport, openProgressSocket, downloadReportPdf, ReportResponse, MatchResult, ProgressEvent } from '../api/client' +import SimilarityMeter from '../components/SimilarityMeter' +import MatchCard from '../components/MatchCard' +import HighlightedText from '../components/HighlightedText' + +interface Props { + taskId: string + onBack: () => void +} + +const STAGE_LABEL: Record = { + fingerprint: 'Поиск точных совпадений…', + fuzzy: 'Нечёткое сравнение…', + semantic: 'Семантический анализ…', + saving: 'Сохранение результатов…', + done: 'Готово', + failed: 'Ошибка', +} + +const POLL_INTERVAL = 4000 + +export default function Report({ taskId, onBack }: Props) { + const [report, setReport] = useState(null) + const [error, setError] = useState(null) + const [progress, setProgress] = useState(null) + const [activeMatch, setActiveMatch] = useState(null) + const pollRef = useRef | null>(null) + + const fetchReport = useCallback(async () => { + try { + const data = await getReport(taskId) + setReport(data) + return data.status + } catch { + setError('Не удалось загрузить отчёт') + return 'failed' + } + }, [taskId]) + + // WebSocket progress + fallback polling + useEffect(() => { + const closeWs = openProgressSocket(taskId, (ev) => { + setProgress(ev) + if (ev.stage === 'done' || ev.stage === 'failed') { + fetchReport() + } + }) + + // Polling fallback if WS doesn't deliver done + const poll = async () => { + const s = await fetchReport() + if (s === 'processing') { + pollRef.current = setTimeout(poll, POLL_INTERVAL) + } + } + poll() + + return () => { + closeWs() + if (pollRef.current) clearTimeout(pollRef.current) + } + }, [taskId, fetchReport]) + + const isProcessing = !report || report.status === 'processing' + const matches: MatchResult[] = report?.matches ?? [] + + return ( +
+ {/* Header */} +
+ +

Отчёт о проверке

+
+ {report?.status === 'completed' && ( + + )} + {taskId.slice(0, 8)}… +
+
+ + {/* Progress bar */} + {isProcessing && ( +
+
+ {progress ? STAGE_LABEL[progress.stage] : 'Запуск анализа…'} + {progress?.progress ?? 0}% +
+
+
+
+
+ {['fingerprint', 'fuzzy', 'semantic', 'saving', 'done'].map((stage) => ( + + {STAGE_LABEL[stage].replace('…', '')} + + ))} +
+
+ )} + + {error && ( +
+ {error} +
+ )} + + {/* Main content */} + {report && report.status === 'completed' && ( + <> + {/* Summary bar */} +
+
+ +
+
+ + {/* Legend */} +
+ Точное совпадение + Нечёткое + Семантика +
+ + {/* Two-column layout */} +
+ {/* Left: highlighted document text */} +
+
+ {report.source_text ? ( + + ) : ( +

Текст документа недоступен

+ )} +
+
+ + {/* Right: match list */} + +
+ + )} + + {/* Still processing with no result yet */} + {isProcessing && !error && ( +
+
+
⚙
+

Анализируем документ…

+
+
+ )} + + {report?.status === 'failed' && ( +
+
+

Анализ завершился с ошибкой

+ +
+
+ )} +
+ ) +} diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js new file mode 100644 index 0000000..efc1901 --- /dev/null +++ b/frontend/tailwind.config.js @@ -0,0 +1,8 @@ +/** @type {import('tailwindcss').Config} */ +export default { + content: ['./index.html', './src/**/*.{ts,tsx}'], + theme: { + extend: {}, + }, + plugins: [], +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..2b065ea --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' + +export default defineConfig({ + plugins: [react()], + server: { + port: 5173, + proxy: { + '/api': { + target: 'http://localhost:8000', + changeOrigin: true, + } + } + } +}) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e99f2f8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,37 @@ +fastapi==0.104.1 +uvicorn==0.24.0 +sqlalchemy==2.0.23 +psycopg2-binary==2.9.9 +pydantic==2.5.0 +pydantic-settings==2.1.0 +python-multipart==0.0.6 +python-jose==3.3.0 +passlib==1.7.4 +bcrypt==4.1.1 + +elasticsearch==8.11.0 +faiss-cpu==1.7.4 +sentence-transformers==2.2.2 +datasketch==1.0.8 +xxhash==3.4.1 + +celery==5.3.4 +redis[asyncio]==5.0.1 +minio==7.2.0 + +numpy==1.24.3 +scipy==1.11.4 +scikit-learn==1.3.2 + +wikitextparser==0.31.9 +pymupdf==1.23.8 +pdfplumber==0.10.3 +python-docx==0.8.11 + +requests==2.31.0 +aiofiles==23.2.1 +websockets==12.0 +pydantic-extra-types==2.3.0 + +python-dotenv==1.0.0 +reportlab==4.1.0