This commit is contained in:
jze9
0
backend/app/__init__.py
Normal file
0
backend/app/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
0
backend/app/api/__init__.py
Normal file
115
backend/app/api/admin.py
Normal file
115
backend/app/api/admin.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from fastapi import APIRouter, HTTPException, status, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
import asyncio
|
||||
import json as _json
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
SOURCES = {
|
||||
"wikipedia_ru": "Wikipedia RU (~8 GB)",
|
||||
"wikipedia_en": "Wikipedia EN (~22 GB)",
|
||||
}
|
||||
|
||||
|
||||
class JobResponse(BaseModel):
|
||||
status: str
|
||||
source: str | None = None
|
||||
label: str | None = None
|
||||
progress: int = 0
|
||||
message: str = ""
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
docs_done: int = 0
|
||||
|
||||
|
||||
@router.get("/index/sources")
|
||||
def list_sources():
|
||||
"""Список доступных источников для индексации."""
|
||||
return [{"key": k, "label": v} for k, v in SOURCES.items()]
|
||||
|
||||
|
||||
@router.post("/index/download/{source_key}", response_model=JobResponse)
|
||||
def start_download(source_key: str):
|
||||
"""Скачать дамп и проиндексировать с нуля."""
|
||||
if source_key not in SOURCES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
|
||||
|
||||
from app.tasks.indexing import get_job_status, download_and_index
|
||||
current = get_job_status()
|
||||
if current.get("status") == "running":
|
||||
raise HTTPException(status_code=409, detail="Задание уже выполняется")
|
||||
|
||||
download_and_index.delay(source_key)
|
||||
return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск скачивания…")
|
||||
|
||||
|
||||
@router.post("/index/update/{source_key}", response_model=JobResponse)
|
||||
def start_update(source_key: str):
|
||||
"""Обновить индекс — только новые документы (дамп не скачивается заново если уже есть)."""
|
||||
if source_key not in SOURCES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown source: {source_key}")
|
||||
|
||||
from app.tasks.indexing import get_job_status, update_index
|
||||
current = get_job_status()
|
||||
if current.get("status") == "running":
|
||||
raise HTTPException(status_code=409, detail="Задание уже выполняется")
|
||||
|
||||
update_index.delay(source_key)
|
||||
return JobResponse(status="started", source=source_key, label=SOURCES[source_key], message="Запуск обновления…")
|
||||
|
||||
|
||||
@router.get("/index/status", response_model=JobResponse)
|
||||
def job_status():
|
||||
"""Текущий статус задания индексации."""
|
||||
from app.tasks.indexing import get_job_status
|
||||
data = get_job_status()
|
||||
return JobResponse(
|
||||
status=data.get("status", "idle"),
|
||||
source=data.get("source"),
|
||||
label=data.get("label"),
|
||||
progress=data.get("progress", 0),
|
||||
message=data.get("message", ""),
|
||||
started_at=data.get("started_at"),
|
||||
finished_at=data.get("finished_at"),
|
||||
docs_done=data.get("docs_done", 0),
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/ws/index-progress")
|
||||
async def ws_index_progress(websocket: WebSocket):
|
||||
"""Stream indexing job progress via Redis pub/sub."""
|
||||
await websocket.accept()
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from app.config import settings
|
||||
|
||||
r = aioredis.from_url(settings.REDIS_URL)
|
||||
pubsub = r.pubsub()
|
||||
await pubsub.subscribe("index_job_progress")
|
||||
|
||||
# Send current state immediately
|
||||
from app.tasks.indexing import get_job_status
|
||||
await websocket.send_json(get_job_status())
|
||||
|
||||
try:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
try:
|
||||
data = _json.loads(message["data"])
|
||||
except Exception:
|
||||
continue
|
||||
await websocket.send_json(data)
|
||||
if data.get("status") in ("done", "error"):
|
||||
break
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
await pubsub.unsubscribe("index_job_progress")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await r.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
115
backend/app/api/auth.py
Normal file
115
backend/app/api/auth.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
from pydantic import BaseModel, EmailStr
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models.db import get_db, User
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
|
||||
|
||||
|
||||
# ── Schemas ────────────────────────────────────────────────────────────────
|
||||
|
||||
class RegisterRequest(BaseModel):
|
||||
email: EmailStr
|
||||
password: str
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
id: str
|
||||
email: str
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
def create_token(user_id) -> str:
|
||||
expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
return jwt.encode(
|
||||
{"sub": str(user_id), "exp": expire},
|
||||
settings.SECRET_KEY,
|
||||
algorithm=settings.ALGORITHM,
|
||||
)
|
||||
|
||||
|
||||
def get_current_user(
|
||||
token: Optional[str] = Depends(oauth2_scheme),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Optional[User]:
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
|
||||
user_id_str: str = payload.get("sub")
|
||||
if not user_id_str:
|
||||
return None
|
||||
import uuid as _uuid
|
||||
user_id = _uuid.UUID(user_id_str)
|
||||
except (JWTError, ValueError):
|
||||
return None
|
||||
return db.query(User).filter(User.id == user_id).first()
|
||||
|
||||
|
||||
def require_user(current_user: Optional[User] = Depends(get_current_user)) -> User:
|
||||
if not current_user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
# ── Endpoints ──────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/register", response_model=UserOut, status_code=status.HTTP_201_CREATED)
|
||||
def register(body: RegisterRequest, db: Session = Depends(get_db)):
|
||||
if db.query(User).filter(User.email == body.email).first():
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Email already registered")
|
||||
user = User(
|
||||
email=body.email,
|
||||
hashed_password=hash_password(body.password),
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
def login(form: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):
|
||||
user = db.query(User).filter(User.email == form.username).first()
|
||||
if not user or not verify_password(form.password, user.hashed_password):
|
||||
raise HTTPException(status_code=400, detail="Incorrect email or password")
|
||||
return {"access_token": create_token(user.id)}
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(current_user: User = Depends(require_user)):
|
||||
return current_user
|
||||
18
backend/app/celery_app.py
Normal file
18
backend/app/celery_app.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from celery import Celery
|
||||
from app.config import settings
|
||||
|
||||
celery_app = Celery(
|
||||
"antiplagiator",
|
||||
broker=settings.CELERY_BROKER_URL,
|
||||
backend=settings.CELERY_RESULT_BACKEND,
|
||||
include=["app.tasks", "app.tasks.indexing"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
)
|
||||
50
backend/app/config.py
Normal file
50
backend/app/config.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# Database
|
||||
DATABASE_URL: str = "postgresql://antiplagiator:antiplagiator_dev@localhost:5432/antiplagiator"
|
||||
|
||||
# Elasticsearch
|
||||
ELASTICSEARCH_HOST: str = "localhost"
|
||||
ELASTICSEARCH_PORT: int = 9200
|
||||
|
||||
# Redis
|
||||
REDIS_URL: str = "redis://localhost:6379/0"
|
||||
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: str = "localhost:9000"
|
||||
MINIO_ACCESS_KEY: str = "minioadmin"
|
||||
MINIO_SECRET_KEY: str = "minioadmin"
|
||||
MINIO_BUCKET: str = "antiplagiator"
|
||||
MINIO_SECURE: bool = False
|
||||
|
||||
# JWT
|
||||
SECRET_KEY: str = "your-secret-key-change-in-prod"
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
|
||||
|
||||
# Celery
|
||||
CELERY_BROKER_URL: str = "redis://localhost:6379/0"
|
||||
CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
|
||||
|
||||
# API
|
||||
API_TITLE: str = "Antiplagiator API"
|
||||
API_VERSION: str = "0.1.0"
|
||||
API_DESCRIPTION: str = "Text plagiarism detection system"
|
||||
|
||||
# Paths
|
||||
DATA_DIR: str = "./data"
|
||||
RAW_DATA_DIR: str = "./data/raw"
|
||||
PROCESSED_DATA_DIR: str = "./data/processed"
|
||||
INDEX_DIR: str = "./data/index"
|
||||
|
||||
# Analysis
|
||||
FINGERPRINT_THRESHOLD: float = 0.5
|
||||
SEMANTIC_THRESHOLD: float = 0.6
|
||||
TOP_K_RESULTS: int = 10
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
settings = Settings()
|
||||
0
backend/app/core/__init__.py
Normal file
0
backend/app/core/__init__.py
Normal file
149
backend/app/core/analyzer.py
Normal file
149
backend/app/core/analyzer.py
Normal file
@@ -0,0 +1,149 @@
|
||||
import re
|
||||
from typing import Dict, List, Any, Tuple
|
||||
from app.core.fingerprint import WinNowing, MinHashLSHIndex
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.core.fulltext import FulltextSearch
|
||||
from app.config import settings
|
||||
|
||||
FRAGMENT_WORDS = 40
|
||||
|
||||
|
||||
def _split_sentences(text: str) -> List[str]:
|
||||
parts = re.split(r'(?<=[.!?])\s+', text.strip())
|
||||
return [p for p in parts if p]
|
||||
|
||||
|
||||
def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]:
|
||||
"""Return (fragment_text, char_start, char_end) around the given sentence index."""
|
||||
sentences = _split_sentences(text)
|
||||
start_i = max(0, sentence_idx - context_sentences)
|
||||
end_i = min(len(sentences), sentence_idx + context_sentences + 1)
|
||||
|
||||
fragment = " ".join(sentences[start_i:end_i])
|
||||
char_start = text.find(sentences[start_i]) if sentences else 0
|
||||
char_end = char_start + len(fragment)
|
||||
return fragment, char_start, char_end
|
||||
|
||||
|
||||
def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str:
|
||||
words = text.split()
|
||||
if len(words) <= max_words:
|
||||
return text
|
||||
return " ".join(words[:max_words]) + "…"
|
||||
|
||||
|
||||
class DocumentAnalyzer:
|
||||
def __init__(self):
|
||||
self.winnowing = WinNowing()
|
||||
self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD)
|
||||
self.embeddings = EmbeddingIndex()
|
||||
self.fulltext = FulltextSearch()
|
||||
|
||||
def analyze(self, text: str, task_id: str) -> Dict[str, Any]:
|
||||
matches: List[Dict[str, Any]] = []
|
||||
matches.extend(self._level_1_exact_match(text))
|
||||
matches.extend(self._level_2_fuzzy_match(text))
|
||||
matches.extend(self._level_3_semantic_match(text))
|
||||
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"status": "completed",
|
||||
"overall_similarity": self._calculate_overall_similarity(matches),
|
||||
"matches": matches[:settings.TOP_K_RESULTS],
|
||||
}
|
||||
|
||||
def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
sentences = _split_sentences(text)
|
||||
if not sentences:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for idx, sentence in enumerate(sentences[:20]):
|
||||
if len(sentence.split()) < 6:
|
||||
continue
|
||||
try:
|
||||
es_results = self.fulltext.search_phrase(sentence, top_k=2)
|
||||
for r in es_results:
|
||||
r["_sentence_idx"] = idx
|
||||
results.extend(es_results)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
matches = []
|
||||
seen_docs: set = set()
|
||||
for result in results:
|
||||
if result["doc_id"] in seen_docs:
|
||||
continue
|
||||
idx = result.get("_sentence_idx", 0)
|
||||
fragment, char_start, char_end = _extract_fragment(text, idx)
|
||||
matches.append({
|
||||
"method": "exact",
|
||||
"similarity": min(result["score"] * 20, 100),
|
||||
"source_id": str(result["doc_id"]), # передаём doc_id напрямую
|
||||
"source_title": result["title"],
|
||||
"source_db": result["source"],
|
||||
"url": result["url"],
|
||||
"fragment_text": _truncate(fragment),
|
||||
"fragment_start": char_start,
|
||||
"fragment_end": char_end,
|
||||
})
|
||||
seen_docs.add(result["doc_id"])
|
||||
|
||||
return matches[:top_k]
|
||||
|
||||
def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
fuzzy_results = self.minhash_lsh.query(text, top_k=top_k)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
sentences = _split_sentences(text)
|
||||
fragment = _truncate(sentences[0]) if sentences else ""
|
||||
matches = []
|
||||
for doc_id, similarity in fuzzy_results:
|
||||
if similarity >= settings.FINGERPRINT_THRESHOLD:
|
||||
matches.append({
|
||||
"method": "fuzzy",
|
||||
"similarity": similarity * 100,
|
||||
"source_id": doc_id,
|
||||
"fragment_text": fragment,
|
||||
"fragment_start": 0,
|
||||
"fragment_end": len(fragment),
|
||||
})
|
||||
return matches
|
||||
|
||||
def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
embedding_results = self.embeddings.search(text, top_k=top_k)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
sentences = _split_sentences(text)
|
||||
fragment = _truncate(sentences[0]) if sentences else ""
|
||||
matches = []
|
||||
for doc_id, similarity in embedding_results:
|
||||
if similarity >= settings.SEMANTIC_THRESHOLD:
|
||||
matches.append({
|
||||
"method": "semantic",
|
||||
"similarity": similarity * 100,
|
||||
"source_id": doc_id,
|
||||
"fragment_text": fragment,
|
||||
"fragment_start": 0,
|
||||
"fragment_end": len(fragment),
|
||||
})
|
||||
return matches
|
||||
|
||||
def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float:
|
||||
if not matches:
|
||||
return 0.0
|
||||
|
||||
weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3}
|
||||
total_weight = 0.0
|
||||
total_similarity = 0.0
|
||||
|
||||
for match in matches:
|
||||
w = weights.get(match["method"], 0.1)
|
||||
total_similarity += match["similarity"] * w
|
||||
total_weight += w
|
||||
|
||||
return round(total_similarity / total_weight if total_weight > 0 else 0, 1)
|
||||
76
backend/app/core/embeddings.py
Normal file
76
backend/app/core/embeddings.py
Normal file
@@ -0,0 +1,76 @@
|
||||
import numpy as np
|
||||
import faiss
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from typing import List, Tuple
|
||||
import os
|
||||
from app.config import settings
|
||||
|
||||
class EmbeddingIndex:
|
||||
def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"):
|
||||
self.model = SentenceTransformer(model_name)
|
||||
self.embedding_dim = self.model.get_sentence_embedding_dimension()
|
||||
self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32)
|
||||
self.doc_ids = []
|
||||
self.embeddings = None
|
||||
|
||||
# Try to load existing index
|
||||
self._load_index()
|
||||
|
||||
def add_document(self, doc_id: str, text: str):
|
||||
embedding = self.model.encode([text], convert_to_numpy=True)[0]
|
||||
self.index.add(np.array([embedding], dtype=np.float32))
|
||||
self.doc_ids.append(doc_id)
|
||||
|
||||
if self.embeddings is None:
|
||||
self.embeddings = np.array([embedding])
|
||||
else:
|
||||
self.embeddings = np.vstack([self.embeddings, embedding])
|
||||
|
||||
def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32):
|
||||
doc_ids = [doc[0] for doc in docs]
|
||||
texts = [doc[1] for doc in docs]
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch_texts = texts[i:i+batch_size]
|
||||
embeddings = self.model.encode(batch_texts, convert_to_numpy=True)
|
||||
|
||||
self.index.add(embeddings.astype(np.float32))
|
||||
self.doc_ids.extend(doc_ids[i:i+batch_size])
|
||||
|
||||
if self.embeddings is None:
|
||||
self.embeddings = embeddings
|
||||
else:
|
||||
self.embeddings = np.vstack([self.embeddings, embeddings])
|
||||
|
||||
def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
|
||||
query_embedding = self.model.encode([text], convert_to_numpy=True)[0]
|
||||
query_embedding = np.array([query_embedding], dtype=np.float32)
|
||||
|
||||
distances, indices = self.index.search(query_embedding, top_k)
|
||||
|
||||
results = []
|
||||
for idx, distance in zip(indices[0], distances[0]):
|
||||
if idx < len(self.doc_ids):
|
||||
similarity = 1.0 / (1.0 + distance)
|
||||
results.append((self.doc_ids[idx], similarity))
|
||||
|
||||
return results
|
||||
|
||||
def _load_index(self):
|
||||
index_path = os.path.join(settings.INDEX_DIR, "faiss_index")
|
||||
ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy")
|
||||
|
||||
if os.path.exists(index_path) and os.path.exists(ids_path):
|
||||
try:
|
||||
self.index = faiss.read_index(index_path)
|
||||
self.doc_ids = list(np.load(ids_path, allow_pickle=True))
|
||||
except Exception as e:
|
||||
print(f"Failed to load index: {e}")
|
||||
|
||||
def save_index(self):
|
||||
os.makedirs(settings.INDEX_DIR, exist_ok=True)
|
||||
faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index"))
|
||||
np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object))
|
||||
|
||||
def get_index_size(self) -> int:
|
||||
return self.index.ntotal
|
||||
82
backend/app/core/fingerprint.py
Normal file
82
backend/app/core/fingerprint.py
Normal file
@@ -0,0 +1,82 @@
|
||||
import xxhash
|
||||
from typing import Set, Tuple, List
|
||||
from datasketch import MinHash, MinHashLSH
|
||||
import numpy as np
|
||||
|
||||
class WinNowing:
|
||||
def __init__(self, k: int = 5, window_size: int = 4):
|
||||
self.k = k
|
||||
self.window_size = window_size
|
||||
|
||||
def get_kgrams(self, text: str) -> List[str]:
|
||||
words = text.split()
|
||||
kgrams = []
|
||||
for i in range(len(words) - self.k + 1):
|
||||
kgram = ' '.join(words[i:i+self.k])
|
||||
kgrams.append(kgram)
|
||||
return kgrams
|
||||
|
||||
def fingerprint(self, text: str) -> Set[int]:
|
||||
kgrams = self.get_kgrams(text)
|
||||
hashes = []
|
||||
for kgram in kgrams:
|
||||
h = int(xxhash.xxh64(kgram).hexdigest(), 16)
|
||||
hashes.append(h)
|
||||
|
||||
if not hashes:
|
||||
return set()
|
||||
|
||||
fingerprints = set()
|
||||
for i in range(len(hashes) - self.window_size + 1):
|
||||
window = hashes[i:i+self.window_size]
|
||||
min_hash = min(window)
|
||||
fingerprints.add(min_hash)
|
||||
|
||||
return fingerprints
|
||||
|
||||
def compare(self, fp1: Set[int], fp2: Set[int]) -> float:
|
||||
if not fp1 or not fp2:
|
||||
return 0.0
|
||||
intersection = len(fp1 & fp2)
|
||||
union = len(fp1 | fp2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
class MinHashLSHIndex:
|
||||
def __init__(self, num_perm: int = 128, threshold: float = 0.5):
|
||||
self.num_perm = num_perm
|
||||
self.threshold = threshold
|
||||
self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
|
||||
self.documents = {}
|
||||
|
||||
def add_document(self, doc_id: str, text: str):
|
||||
kgrams = self._get_kgrams(text)
|
||||
m = MinHash(num_perm=self.num_perm)
|
||||
for kgram in kgrams:
|
||||
m.update(kgram.encode())
|
||||
|
||||
self.lsh.insert(doc_id, m)
|
||||
self.documents[doc_id] = m
|
||||
|
||||
def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
|
||||
kgrams = self._get_kgrams(text)
|
||||
query_m = MinHash(num_perm=self.num_perm)
|
||||
for kgram in kgrams:
|
||||
query_m.update(kgram.encode())
|
||||
|
||||
candidates = self.lsh.query(query_m)
|
||||
|
||||
results = []
|
||||
for doc_id in candidates:
|
||||
similarity = query_m.jaccard(self.documents[doc_id])
|
||||
results.append((doc_id, similarity))
|
||||
|
||||
results.sort(key=lambda x: x[1], reverse=True)
|
||||
return results[:top_k]
|
||||
|
||||
def _get_kgrams(self, text: str, k: int = 5) -> List[str]:
|
||||
words = text.split()
|
||||
kgrams = []
|
||||
for i in range(max(1, len(words) - k + 1)):
|
||||
kgram = ' '.join(words[i:i+k])
|
||||
kgrams.append(kgram)
|
||||
return kgrams
|
||||
135
backend/app/core/fulltext.py
Normal file
135
backend/app/core/fulltext.py
Normal file
@@ -0,0 +1,135 @@
|
||||
from elasticsearch import Elasticsearch
|
||||
from typing import List, Tuple, Dict, Any
|
||||
from app.config import settings
|
||||
|
||||
class FulltextSearch:
|
||||
def __init__(self):
|
||||
self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"])
|
||||
self.index_name = "documents"
|
||||
self._create_index()
|
||||
|
||||
def _create_index(self):
|
||||
if not self.es.indices.exists(index=self.index_name):
|
||||
self.es.indices.create(
|
||||
index=self.index_name,
|
||||
body={
|
||||
"settings": {
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"ru_en": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "ru_stop", "ru_stem"]
|
||||
}
|
||||
},
|
||||
"filter": {
|
||||
"ru_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_russian_"
|
||||
},
|
||||
"ru_stem": {
|
||||
"type": "stemmer",
|
||||
"language": "russian"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"doc_id": {"type": "keyword"},
|
||||
"text": {"type": "text", "analyzer": "ru_en"},
|
||||
"title": {"type": "text", "analyzer": "ru_en"},
|
||||
"source": {"type": "keyword"},
|
||||
"lang": {"type": "keyword"},
|
||||
"url": {"type": "keyword"}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""):
|
||||
self.es.index(
|
||||
index=self.index_name,
|
||||
id=doc_id,
|
||||
body={
|
||||
"doc_id": doc_id,
|
||||
"title": title,
|
||||
"text": text,
|
||||
"source": source,
|
||||
"lang": lang,
|
||||
"url": url
|
||||
}
|
||||
)
|
||||
|
||||
def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]:
|
||||
must = {
|
||||
"multi_match": {
|
||||
"query": query,
|
||||
"fields": ["text^2", "title^3"],
|
||||
"type": "best_fields",
|
||||
"analyzer": "ru_en",
|
||||
}
|
||||
}
|
||||
|
||||
if lang:
|
||||
search_body = {
|
||||
"size": top_k,
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must,
|
||||
"filter": {"term": {"lang": lang}},
|
||||
}
|
||||
},
|
||||
}
|
||||
else:
|
||||
search_body = {"size": top_k, "query": must}
|
||||
|
||||
results = self.es.search(index=self.index_name, body=search_body)
|
||||
|
||||
formatted_results = []
|
||||
for hit in results['hits']['hits']:
|
||||
source = hit['_source']
|
||||
formatted_results.append({
|
||||
"doc_id": source['doc_id'],
|
||||
"title": source['title'],
|
||||
"source": source['source'],
|
||||
"url": source.get('url', ''),
|
||||
"score": hit['_score']
|
||||
})
|
||||
|
||||
return formatted_results
|
||||
|
||||
def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]:
|
||||
results = self.es.search(
|
||||
index=self.index_name,
|
||||
body={
|
||||
"size": top_k,
|
||||
"query": {
|
||||
"match_phrase": {
|
||||
"text": phrase
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
formatted_results = []
|
||||
for hit in results['hits']['hits']:
|
||||
source = hit['_source']
|
||||
formatted_results.append({
|
||||
"doc_id": source['doc_id'],
|
||||
"title": source['title'],
|
||||
"source": source['source'],
|
||||
"url": source.get('url', ''),
|
||||
"score": hit['_score']
|
||||
})
|
||||
|
||||
return formatted_results
|
||||
|
||||
def bulk_index(self, documents: List[Dict[str, Any]]):
|
||||
actions = []
|
||||
for doc in documents:
|
||||
actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}})
|
||||
actions.append(doc)
|
||||
|
||||
from elasticsearch.helpers import bulk
|
||||
bulk(self.es, actions)
|
||||
354
backend/app/main.py
Normal file
354
backend/app/main.py
Normal file
@@ -0,0 +1,354 @@
|
||||
from fastapi import FastAPI, UploadFile, File, Depends, HTTPException, status, Query, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import StreamingResponse
|
||||
import uuid
|
||||
import io
|
||||
import asyncio
|
||||
import json as _json
|
||||
from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.models.db import get_db, engine, Base, AnalysisTask, Document, Match, User
|
||||
from app.schemas import (
|
||||
CheckResponse, ReportResponse, ReportSummary,
|
||||
ReportsListResponse, StatsResponse, HealthResponse, MatchResult,
|
||||
)
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.api.auth import router as auth_router, get_current_user
|
||||
from app.api.admin import router as admin_router
|
||||
import pdfplumber
|
||||
import docx as python_docx
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
from typing import Optional
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.API_TITLE,
|
||||
version=settings.API_VERSION,
|
||||
description=settings.API_DESCRIPTION,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(auth_router)
|
||||
app.include_router(admin_router)
|
||||
|
||||
_minio: Minio | None = None
|
||||
_embedding_index: EmbeddingIndex | None = None
|
||||
|
||||
|
||||
def get_minio() -> Minio:
|
||||
global _minio
|
||||
if _minio is None:
|
||||
_minio = Minio(
|
||||
settings.MINIO_ENDPOINT,
|
||||
access_key=settings.MINIO_ACCESS_KEY,
|
||||
secret_key=settings.MINIO_SECRET_KEY,
|
||||
secure=settings.MINIO_SECURE,
|
||||
)
|
||||
try:
|
||||
if not _minio.bucket_exists(settings.MINIO_BUCKET):
|
||||
_minio.make_bucket(settings.MINIO_BUCKET)
|
||||
except S3Error:
|
||||
pass
|
||||
return _minio
|
||||
|
||||
|
||||
def get_embedding_index() -> EmbeddingIndex:
|
||||
global _embedding_index
|
||||
if _embedding_index is None:
|
||||
_embedding_index = EmbeddingIndex()
|
||||
return _embedding_index
|
||||
|
||||
|
||||
def extract_text(content: bytes, filename: str) -> str:
|
||||
if filename.endswith(".txt"):
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
if filename.endswith(".pdf"):
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
return "\n".join(page.extract_text() or "" for page in pdf.pages)
|
||||
|
||||
if filename.endswith(".docx"):
|
||||
doc = python_docx.Document(io.BytesIO(content))
|
||||
return "\n".join(p.text for p in doc.paragraphs)
|
||||
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
@app.get("/api/health", response_model=HealthResponse)
|
||||
async def health_check():
|
||||
return {"status": "ok", "version": settings.API_VERSION}
|
||||
|
||||
|
||||
@app.post("/api/check", response_model=CheckResponse)
|
||||
async def check_document(
|
||||
file: UploadFile = File(...),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: Optional[User] = Depends(get_current_user),
|
||||
):
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No filename")
|
||||
|
||||
allowed = {".txt", ".pdf", ".docx"}
|
||||
ext = "." + file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else ""
|
||||
if ext not in allowed:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unsupported file type. Allowed: {', '.join(allowed)}",
|
||||
)
|
||||
|
||||
task_id = uuid.uuid4()
|
||||
content = await file.read()
|
||||
|
||||
minio_key = f"uploads/{task_id}/{file.filename}"
|
||||
try:
|
||||
minio = get_minio()
|
||||
minio.put_object(
|
||||
settings.MINIO_BUCKET,
|
||||
minio_key,
|
||||
io.BytesIO(content),
|
||||
length=len(content),
|
||||
content_type=file.content_type or "application/octet-stream",
|
||||
)
|
||||
except Exception:
|
||||
minio_key = None
|
||||
|
||||
try:
|
||||
text = extract_text(content, file.filename)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Cannot parse file: {e}")
|
||||
|
||||
if not text.strip():
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="File is empty or unreadable")
|
||||
|
||||
task = AnalysisTask(
|
||||
id=task_id,
|
||||
status="processing",
|
||||
source_text=text,
|
||||
user_id=current_user.id if current_user else None,
|
||||
)
|
||||
db.add(task)
|
||||
db.commit()
|
||||
|
||||
from app.tasks import analyze_document
|
||||
analyze_document.delay(text, str(task_id))
|
||||
|
||||
return {"task_id": str(task_id), "status": "processing", "message": "Document received and analysis started"}
|
||||
|
||||
|
||||
@app.get("/api/report/{task_id}", response_model=ReportResponse)
|
||||
async def get_report(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
|
||||
matches = []
|
||||
if task.status == "completed":
|
||||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||||
for m in db_matches:
|
||||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||||
matches.append(
|
||||
MatchResult(
|
||||
method=m.method,
|
||||
similarity=float(m.similarity),
|
||||
source_title=doc.title if doc else None,
|
||||
source_db=doc.source if doc else None,
|
||||
url=doc.url if doc else None,
|
||||
fragment_text=m.fragment_text,
|
||||
fragment_start=m.fragment_start,
|
||||
fragment_end=m.fragment_end,
|
||||
)
|
||||
)
|
||||
|
||||
return ReportResponse(
|
||||
task_id=str(task.id),
|
||||
status=task.status,
|
||||
overall_similarity=task.overall_similarity,
|
||||
matches=matches or None,
|
||||
source_text=task.source_text,
|
||||
created_at=task.created_at,
|
||||
completed_at=task.completed_at,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/report/{task_id}/pdf")
|
||||
async def download_report_pdf(task_id: uuid.UUID, db: Session = Depends(get_db)):
|
||||
"""Generate and return a PDF report."""
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
if task.status != "completed":
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Report not ready yet")
|
||||
|
||||
db_matches = db.query(Match).filter(Match.task_id == task_id).all()
|
||||
matches_data = []
|
||||
for m in db_matches:
|
||||
doc = db.query(Document).filter(Document.id == m.doc_id).first()
|
||||
matches_data.append({
|
||||
"method": m.method,
|
||||
"similarity": m.similarity,
|
||||
"source_title": doc.title if doc else "—",
|
||||
"source_db": doc.source if doc else "—",
|
||||
"url": doc.url if doc else "",
|
||||
"fragment_text": m.fragment_text or "",
|
||||
})
|
||||
|
||||
pdf_bytes = _build_pdf(task_id, task.overall_similarity or 0, matches_data)
|
||||
|
||||
return StreamingResponse(
|
||||
io.BytesIO(pdf_bytes),
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": f'attachment; filename="report-{task_id[:8]}.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
def _build_pdf(task_id: str, overall: float, matches: list) -> bytes:
|
||||
from reportlab.lib.pagesizes import A4
|
||||
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
||||
from reportlab.lib.units import cm
|
||||
from reportlab.lib import colors
|
||||
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, HRFlowable
|
||||
from reportlab.pdfbase import pdfmetrics
|
||||
from reportlab.pdfbase.ttfonts import TTFont
|
||||
import os
|
||||
|
||||
buf = io.BytesIO()
|
||||
doc = SimpleDocTemplate(buf, pagesize=A4, rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
|
||||
styles = getSampleStyleSheet()
|
||||
|
||||
METHOD_RU = {"exact": "Точное", "fuzzy": "Нечёткое", "semantic": "Семантика"}
|
||||
METHOD_COLOR = {"exact": colors.HexColor("#fee2e2"), "fuzzy": colors.HexColor("#ffedd5"), "semantic": colors.HexColor("#fef9c3")}
|
||||
|
||||
title_style = ParagraphStyle("title", parent=styles["Title"], fontSize=18, spaceAfter=6)
|
||||
h2_style = ParagraphStyle("h2", parent=styles["Heading2"], fontSize=13, spaceBefore=12, spaceAfter=4)
|
||||
body_style = ParagraphStyle("body", parent=styles["Normal"], fontSize=10, leading=14)
|
||||
small_style = ParagraphStyle("small", parent=styles["Normal"], fontSize=8, textColor=colors.grey)
|
||||
|
||||
story = [
|
||||
Paragraph("Отчёт о проверке на плагиат", title_style),
|
||||
Paragraph(f"ID задачи: {task_id}", small_style),
|
||||
Spacer(1, 0.3*cm),
|
||||
HRFlowable(width="100%", thickness=1, color=colors.lightgrey),
|
||||
Spacer(1, 0.3*cm),
|
||||
Paragraph(f"Общий процент заимствований: <b>{overall:.1f}%</b>", body_style),
|
||||
Spacer(1, 0.5*cm),
|
||||
]
|
||||
|
||||
if not matches:
|
||||
story.append(Paragraph("Совпадений не обнаружено.", body_style))
|
||||
else:
|
||||
story.append(Paragraph(f"Найдено совпадений: {len(matches)}", h2_style))
|
||||
for i, m in enumerate(matches, 1):
|
||||
bg = METHOD_COLOR.get(m["method"], colors.whitesmoke)
|
||||
data = [
|
||||
[Paragraph(f"<b>#{i} {METHOD_RU.get(m['method'], m['method'])} — {m['similarity']}%</b>", body_style), ""],
|
||||
["Источник:", Paragraph(m["source_title"], body_style)],
|
||||
["База:", m["source_db"]],
|
||||
]
|
||||
if m["url"]:
|
||||
data.append(["URL:", Paragraph(f'<link href="{m["url"]}">{m["url"]}</link>', small_style)])
|
||||
if m["fragment_text"]:
|
||||
data.append(["Фрагмент:", Paragraph(f'«{m["fragment_text"]}»', small_style)])
|
||||
|
||||
tbl = Table(data, colWidths=[3.5*cm, None])
|
||||
tbl.setStyle(TableStyle([
|
||||
("BACKGROUND", (0, 0), (-1, 0), bg),
|
||||
("SPAN", (0, 0), (-1, 0)),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 9),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 3),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 3),
|
||||
("GRID", (0, 0), (-1, -1), 0.25, colors.lightgrey),
|
||||
]))
|
||||
story.append(tbl)
|
||||
story.append(Spacer(1, 0.3*cm))
|
||||
|
||||
doc.build(story)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@app.get("/api/reports", response_model=ReportsListResponse)
|
||||
async def list_reports(
|
||||
user_id: str | None = Query(default=None),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: Optional[User] = Depends(get_current_user),
|
||||
):
|
||||
query = db.query(AnalysisTask)
|
||||
# Авторизованный пользователь видит только свои задачи
|
||||
if current_user:
|
||||
query = query.filter(AnalysisTask.user_id == current_user.id)
|
||||
elif user_id:
|
||||
query = query.filter(AnalysisTask.user_id == user_id)
|
||||
|
||||
tasks = query.order_by(AnalysisTask.created_at.desc()).limit(50).all()
|
||||
|
||||
return ReportsListResponse(
|
||||
tasks=[
|
||||
ReportSummary(
|
||||
task_id=str(t.id),
|
||||
status=t.status,
|
||||
overall_similarity=t.overall_similarity,
|
||||
created_at=t.created_at,
|
||||
)
|
||||
for t in tasks
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/stats", response_model=StatsResponse)
|
||||
async def get_stats(db: Session = Depends(get_db)):
|
||||
total_documents = db.query(Document).count()
|
||||
index = get_embedding_index()
|
||||
|
||||
return StatsResponse(
|
||||
total_documents_indexed=total_documents,
|
||||
faiss_index_size=index.get_index_size(),
|
||||
elasticsearch_available=True,
|
||||
api_version=settings.API_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/ws/progress/{task_id}")
|
||||
async def ws_progress(websocket: WebSocket, task_id: str):
|
||||
await websocket.accept()
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
r = aioredis.from_url(settings.REDIS_URL)
|
||||
pubsub = r.pubsub()
|
||||
await pubsub.subscribe(f"progress:{task_id}")
|
||||
|
||||
try:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] != "message":
|
||||
continue
|
||||
try:
|
||||
data = _json.loads(message["data"])
|
||||
except Exception:
|
||||
continue
|
||||
await websocket.send_json(data)
|
||||
if data.get("stage") in ("done", "failed"):
|
||||
break
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
await pubsub.unsubscribe(f"progress:{task_id}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await r.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
0
backend/app/models/__init__.py
Normal file
0
backend/app/models/__init__.py
Normal file
82
backend/app/models/db.py
Normal file
82
backend/app/models/db.py
Normal file
@@ -0,0 +1,82 @@
|
||||
import uuid
|
||||
from sqlalchemy import Column, String, Text, DateTime, Integer, Float, ForeignKey, Index, create_engine, Boolean
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from datetime import datetime
|
||||
from app.config import settings
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
email = Column(String(255), unique=True, nullable=False, index=True)
|
||||
hashed_password = Column(String(255), nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class Document(Base):
|
||||
__tablename__ = "documents"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source = Column(String(50), nullable=False)
|
||||
ext_id = Column(String(200), nullable=True)
|
||||
title = Column(Text, nullable=False)
|
||||
url = Column(Text, nullable=True)
|
||||
lang = Column(String(2), nullable=True)
|
||||
minio_key = Column(Text, nullable=True)
|
||||
indexed_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class Fingerprint(Base):
|
||||
__tablename__ = "fingerprints"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False)
|
||||
hash = Column(String(64), nullable=False) # xxhash hex string
|
||||
position = Column(Integer, nullable=True)
|
||||
|
||||
|
||||
Index("ix_fingerprints_hash", Fingerprint.hash)
|
||||
Index("ix_fingerprints_doc_id", Fingerprint.doc_id)
|
||||
|
||||
|
||||
class AnalysisTask(Base):
|
||||
__tablename__ = "analysis_tasks"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
|
||||
status = Column(String(20), default="pending")
|
||||
overall_similarity = Column(Float, nullable=True)
|
||||
source_text = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
|
||||
|
||||
class Match(Base):
|
||||
__tablename__ = "matches"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
task_id = Column(UUID(as_uuid=True), ForeignKey("analysis_tasks.id"), nullable=False)
|
||||
doc_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=True)
|
||||
similarity = Column(Float, nullable=False)
|
||||
method = Column(String(20), nullable=False)
|
||||
fragment_start = Column(Integer, nullable=True)
|
||||
fragment_end = Column(Integer, nullable=True)
|
||||
fragment_text = Column(Text, nullable=True)
|
||||
|
||||
|
||||
engine = create_engine(settings.DATABASE_URL, echo=False)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
54
backend/app/schemas/__init__.py
Normal file
54
backend/app/schemas/__init__.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: str
|
||||
version: str
|
||||
|
||||
|
||||
class CheckResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class MatchResult(BaseModel):
|
||||
method: str = Field(..., description="exact | fuzzy | semantic")
|
||||
similarity: float = Field(..., ge=0, le=100)
|
||||
source_title: Optional[str] = None
|
||||
source_db: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
source_id: Optional[str] = None
|
||||
fragment_text: Optional[str] = None
|
||||
fragment_start: Optional[int] = None
|
||||
fragment_end: Optional[int] = None
|
||||
|
||||
|
||||
class ReportResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
overall_similarity: Optional[float] = None
|
||||
matches: Optional[List[MatchResult]] = None
|
||||
source_text: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class ReportSummary(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
overall_similarity: Optional[float] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class ReportsListResponse(BaseModel):
|
||||
tasks: List[ReportSummary]
|
||||
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
total_documents_indexed: int
|
||||
faiss_index_size: int
|
||||
elasticsearch_available: bool
|
||||
api_version: str
|
||||
95
backend/app/tasks/__init__.py
Normal file
95
backend/app/tasks/__init__.py
Normal file
@@ -0,0 +1,95 @@
|
||||
import json
|
||||
import uuid
|
||||
import redis
|
||||
from app.config import settings
|
||||
from app.models.db import SessionLocal, AnalysisTask, Match
|
||||
from app.core.analyzer import DocumentAnalyzer
|
||||
from app.celery_app import celery_app
|
||||
from datetime import datetime
|
||||
|
||||
_analyzer: DocumentAnalyzer | None = None
|
||||
_redis: redis.Redis | None = None
|
||||
|
||||
|
||||
def get_analyzer() -> DocumentAnalyzer:
|
||||
global _analyzer
|
||||
if _analyzer is None:
|
||||
_analyzer = DocumentAnalyzer()
|
||||
return _analyzer
|
||||
|
||||
|
||||
def get_redis() -> redis.Redis:
|
||||
global _redis
|
||||
if _redis is None:
|
||||
_redis = redis.from_url(settings.REDIS_URL)
|
||||
return _redis
|
||||
|
||||
|
||||
def _publish(task_id: str, stage: str, progress: int):
|
||||
try:
|
||||
get_redis().publish(
|
||||
f"progress:{task_id}",
|
||||
json.dumps({"stage": stage, "progress": progress}),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="tasks.analyze_document")
|
||||
def analyze_document(self, text: str, task_id: str):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
analyzer = get_analyzer()
|
||||
|
||||
_publish(task_id, "fingerprint", 10)
|
||||
exact = analyzer._level_1_exact_match(text)
|
||||
|
||||
_publish(task_id, "fuzzy", 40)
|
||||
fuzzy = analyzer._level_2_fuzzy_match(text)
|
||||
|
||||
_publish(task_id, "semantic", 70)
|
||||
semantic = analyzer._level_3_semantic_match(text)
|
||||
|
||||
matches = exact + fuzzy + semantic
|
||||
overall_similarity = analyzer._calculate_overall_similarity(matches)
|
||||
|
||||
_publish(task_id, "saving", 90)
|
||||
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if not task:
|
||||
return
|
||||
|
||||
task.status = "completed"
|
||||
task.overall_similarity = overall_similarity
|
||||
task.completed_at = datetime.utcnow()
|
||||
|
||||
for match in matches[:settings.TOP_K_RESULTS]:
|
||||
raw_id = match.get("source_id")
|
||||
try:
|
||||
doc_id = uuid.UUID(str(raw_id)) if raw_id is not None else None
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
doc_id = None
|
||||
|
||||
db.add(Match(
|
||||
task_id=task_id,
|
||||
doc_id=doc_id,
|
||||
similarity=float(match["similarity"]),
|
||||
method=match["method"],
|
||||
fragment_start=match.get("fragment_start"),
|
||||
fragment_end=match.get("fragment_end"),
|
||||
fragment_text=match.get("fragment_text"),
|
||||
))
|
||||
|
||||
db.commit()
|
||||
_publish(task_id, "done", 100)
|
||||
|
||||
except Exception as exc:
|
||||
task = db.query(AnalysisTask).filter(AnalysisTask.id == task_id).first()
|
||||
if task:
|
||||
task.status = "failed"
|
||||
db.commit()
|
||||
_publish(task_id, "failed", 0)
|
||||
raise self.retry(exc=exc, countdown=5, max_retries=2)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
282
backend/app/tasks/indexing.py
Normal file
282
backend/app/tasks/indexing.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
Celery tasks for downloading and indexing document sources.
|
||||
|
||||
Redis key "index_job" stores current job state as JSON:
|
||||
{ status, source, progress, message, started_at, finished_at, docs_total, docs_done }
|
||||
"""
|
||||
import bz2
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
from app.celery_app import celery_app
|
||||
|
||||
from app.config import settings
|
||||
from app.models.db import SessionLocal, Document, Fingerprint
|
||||
|
||||
JOB_KEY = "index_job"
|
||||
|
||||
WIKIPEDIA_SOURCES = {
|
||||
"wikipedia_ru": {
|
||||
"url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-pages-articles.xml.bz2",
|
||||
"meta_url": "https://dumps.wikimedia.org/ruwiki/latest/ruwiki-latest-md5sums.txt",
|
||||
"lang": "ru",
|
||||
"label": "Wikipedia RU",
|
||||
},
|
||||
"wikipedia_en": {
|
||||
"url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2",
|
||||
"meta_url": "https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-md5sums.txt",
|
||||
"lang": "en",
|
||||
"label": "Wikipedia EN",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Redis helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
def _r() -> redis.Redis:
|
||||
return redis.from_url(settings.REDIS_URL)
|
||||
|
||||
|
||||
def _set_status(data: dict):
|
||||
_r().set(JOB_KEY, json.dumps(data, default=str))
|
||||
|
||||
|
||||
def _get_status() -> dict:
|
||||
raw = _r().get(JOB_KEY)
|
||||
if not raw:
|
||||
return {"status": "idle"}
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def _pub(message: str, progress: int = 0, **extra):
|
||||
state = _get_status()
|
||||
state.update({"message": message, "progress": progress, **extra})
|
||||
_set_status(state)
|
||||
_r().publish("index_job_progress", json.dumps(state, default=str))
|
||||
|
||||
|
||||
# ── Text helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def _clean_wiki(text: str) -> str:
|
||||
text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
|
||||
text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
|
||||
text = re.sub(r'\{\{[^}]*\}\}', '', text)
|
||||
text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
|
||||
text = re.sub(r'<ref[^>]*>.*?</ref>', '', text, flags=re.DOTALL)
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
text = re.sub(r' +', ' ', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _normalize(text: str) -> str:
|
||||
text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
|
||||
text = re.sub(r'[—–‒―]', '-', text)
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
# ── Download helper ────────────────────────────────────────────────────────
|
||||
|
||||
def _download_file(url: str, dest: Path, label: str) -> Path:
|
||||
"""Stream-download url → dest, publishing progress."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
r = requests.get(url, stream=True, timeout=30)
|
||||
r.raise_for_status()
|
||||
total = int(r.headers.get("Content-Length", 0))
|
||||
done = 0
|
||||
last_pct = -1
|
||||
|
||||
with open(dest, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=1 << 20): # 1 MB
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
pct = int(done / total * 40) if total else 0 # 0-40% of overall
|
||||
if pct != last_pct:
|
||||
_pub(f"Скачивание {label}: {done >> 20} MB", progress=pct)
|
||||
last_pct = pct
|
||||
|
||||
return dest
|
||||
|
||||
|
||||
# ── Indexing helper ────────────────────────────────────────────────────────
|
||||
|
||||
def _index_wikipedia_bz2(bz2_path: Path, source: str, lang: str, update_only: bool) -> int:
|
||||
"""
|
||||
Parse Wikipedia bz2 dump and index documents.
|
||||
update_only=True → skip ext_ids already in DB.
|
||||
Returns count of newly indexed docs.
|
||||
"""
|
||||
from app.core.fingerprint import WinNowing, MinHashLSHIndex
|
||||
from app.core.embeddings import EmbeddingIndex
|
||||
from app.core.fulltext import FulltextSearch
|
||||
|
||||
db = SessionLocal()
|
||||
winnowing = WinNowing()
|
||||
minhash = MinHashLSHIndex()
|
||||
emb = EmbeddingIndex()
|
||||
fulltext = FulltextSearch()
|
||||
|
||||
# Pre-load known ext_ids for fast skip
|
||||
known: set[str] = set()
|
||||
if update_only:
|
||||
for (ext_id,) in db.query(Document.ext_id).filter(Document.source == source).yield_per(10_000):
|
||||
if ext_id:
|
||||
known.add(ext_id)
|
||||
|
||||
ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
|
||||
|
||||
embed_batch: list[tuple[str, str]] = []
|
||||
BATCH = 32
|
||||
count = 0
|
||||
parsed = 0
|
||||
|
||||
with bz2.BZ2File(str(bz2_path), 'r') as f:
|
||||
context = ET.iterparse(f, events=('end',))
|
||||
for event, elem in context:
|
||||
if not elem.tag.endswith('page'):
|
||||
continue
|
||||
|
||||
title_el = elem.find('wiki:title', ns) or elem.find('.//title')
|
||||
id_el = elem.find('wiki:id', ns) or elem.find('.//id')
|
||||
rev = elem.find('.//revision')
|
||||
text_el = rev.find('wiki:text', ns) if rev is not None else None
|
||||
if rev is None:
|
||||
text_el = elem.find('.//text')
|
||||
|
||||
if title_el is None or text_el is None:
|
||||
elem.clear(); continue
|
||||
|
||||
title = (title_el.text or '').strip()
|
||||
page_id = (id_el.text or '').strip() if id_el is not None else ''
|
||||
raw_text = text_el.text or ''
|
||||
ext_id = f"wiki_{page_id}"
|
||||
|
||||
if raw_text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
|
||||
elem.clear(); continue
|
||||
|
||||
text = _normalize(_clean_wiki(raw_text))
|
||||
if len(text.split()) < 50:
|
||||
elem.clear(); continue
|
||||
|
||||
parsed += 1
|
||||
|
||||
if update_only and ext_id in known:
|
||||
elem.clear(); continue
|
||||
|
||||
url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}"
|
||||
|
||||
# PostgreSQL
|
||||
doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
|
||||
# Fingerprints
|
||||
fp = winnowing.fingerprint(text)
|
||||
for pos, h in enumerate(list(fp)[:500]):
|
||||
db.add(Fingerprint(doc_id=doc.id, hash=format(h, '016x'), position=pos))
|
||||
|
||||
# MinHash
|
||||
try:
|
||||
minhash.add_document(str(doc.id), text)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Elasticsearch
|
||||
fulltext.index_document(str(doc.id), title, text, source, lang, url)
|
||||
|
||||
embed_batch.append((str(doc.id), text))
|
||||
if len(embed_batch) >= BATCH:
|
||||
emb.batch_add(embed_batch, batch_size=BATCH)
|
||||
embed_batch.clear()
|
||||
|
||||
count += 1
|
||||
if count % 500 == 0:
|
||||
db.commit()
|
||||
pct = 40 + min(int(count / 100000 * 55), 55) # 40-95%
|
||||
_pub(f"Проиндексировано: {count} документов", progress=pct, docs_done=count)
|
||||
|
||||
elem.clear()
|
||||
|
||||
if embed_batch:
|
||||
emb.batch_add(embed_batch, batch_size=BATCH)
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
emb.save_index()
|
||||
|
||||
return count
|
||||
|
||||
|
||||
# ── Celery tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
def _run_indexing(source_key: str, update_only: bool):
|
||||
src = WIKIPEDIA_SOURCES[source_key]
|
||||
label = src["label"]
|
||||
lang = src["lang"]
|
||||
|
||||
_set_status({
|
||||
"status": "running",
|
||||
"source": source_key,
|
||||
"label": label,
|
||||
"progress": 0,
|
||||
"message": "Запуск…",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
"docs_done": 0,
|
||||
})
|
||||
|
||||
data_dir = Path(settings.RAW_DATA_DIR) / source_key
|
||||
bz2_path = data_dir / "dump.xml.bz2"
|
||||
|
||||
# --- Download if needed ---
|
||||
if not bz2_path.exists() or not update_only:
|
||||
_pub(f"Скачивание дампа {label}…", progress=1)
|
||||
try:
|
||||
_download_file(src["url"], bz2_path, label)
|
||||
except Exception as e:
|
||||
_set_status({"status": "error", "message": f"Ошибка скачивания: {e}"})
|
||||
return
|
||||
|
||||
_pub("Индексация…", progress=41)
|
||||
|
||||
try:
|
||||
count = _index_wikipedia_bz2(bz2_path, source_key, lang, update_only)
|
||||
except Exception as e:
|
||||
_set_status({"status": "error", "message": f"Ошибка индексации: {e}"})
|
||||
return
|
||||
|
||||
action = "обновлено" if update_only else "проиндексировано"
|
||||
_set_status({
|
||||
"status": "done",
|
||||
"source": source_key,
|
||||
"label": label,
|
||||
"progress": 100,
|
||||
"message": f"Готово — {action} {count} документов",
|
||||
"finished_at": datetime.now(timezone.utc).isoformat(),
|
||||
"docs_done": count,
|
||||
})
|
||||
_r().publish("index_job_progress", _r().get(JOB_KEY))
|
||||
|
||||
|
||||
@celery_app.task(name="tasks.download_and_index", bind=True)
|
||||
def download_and_index(self, source_key: str):
|
||||
_run_indexing(source_key, update_only=False)
|
||||
|
||||
|
||||
@celery_app.task(name="tasks.update_index", bind=True)
|
||||
def update_index(self, source_key: str):
|
||||
_run_indexing(source_key, update_only=True)
|
||||
|
||||
|
||||
def get_job_status() -> dict:
|
||||
return _get_status()
|
||||
Reference in New Issue
Block a user