test build

backend/app/core/analyzer.py

@@ -0,0 +1,149 @@
+import re
+from typing import Dict, List, Any, Tuple
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+from app.config import settings
+
+FRAGMENT_WORDS = 40
+
+
+def _split_sentences(text: str) -> List[str]:
+    parts = re.split(r'(?<=[.!?])\s+', text.strip())
+    return [p for p in parts if p]
+
+
+def _extract_fragment(text: str, sentence_idx: int, context_sentences: int = 2) -> Tuple[str, int, int]:
+    """Return (fragment_text, char_start, char_end) around the given sentence index."""
+    sentences = _split_sentences(text)
+    start_i = max(0, sentence_idx - context_sentences)
+    end_i = min(len(sentences), sentence_idx + context_sentences + 1)
+
+    fragment = " ".join(sentences[start_i:end_i])
+    char_start = text.find(sentences[start_i]) if sentences else 0
+    char_end = char_start + len(fragment)
+    return fragment, char_start, char_end
+
+
+def _truncate(text: str, max_words: int = FRAGMENT_WORDS) -> str:
+    words = text.split()
+    if len(words) <= max_words:
+        return text
+    return " ".join(words[:max_words]) + "…"
+
+
+class DocumentAnalyzer:
+    def __init__(self):
+        self.winnowing = WinNowing()
+        self.minhash_lsh = MinHashLSHIndex(threshold=settings.FINGERPRINT_THRESHOLD)
+        self.embeddings = EmbeddingIndex()
+        self.fulltext = FulltextSearch()
+
+    def analyze(self, text: str, task_id: str) -> Dict[str, Any]:
+        matches: List[Dict[str, Any]] = []
+        matches.extend(self._level_1_exact_match(text))
+        matches.extend(self._level_2_fuzzy_match(text))
+        matches.extend(self._level_3_semantic_match(text))
+
+        return {
+            "task_id": task_id,
+            "status": "completed",
+            "overall_similarity": self._calculate_overall_similarity(matches),
+            "matches": matches[:settings.TOP_K_RESULTS],
+        }
+
+    def _level_1_exact_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        sentences = _split_sentences(text)
+        if not sentences:
+            return []
+
+        results = []
+        for idx, sentence in enumerate(sentences[:20]):
+            if len(sentence.split()) < 6:
+                continue
+            try:
+                es_results = self.fulltext.search_phrase(sentence, top_k=2)
+                for r in es_results:
+                    r["_sentence_idx"] = idx
+                results.extend(es_results)
+            except Exception:
+                continue
+
+        matches = []
+        seen_docs: set = set()
+        for result in results:
+            if result["doc_id"] in seen_docs:
+                continue
+            idx = result.get("_sentence_idx", 0)
+            fragment, char_start, char_end = _extract_fragment(text, idx)
+            matches.append({
+                "method": "exact",
+                "similarity": min(result["score"] * 20, 100),
+                "source_id": str(result["doc_id"]),   # передаём doc_id напрямую
+                "source_title": result["title"],
+                "source_db": result["source"],
+                "url": result["url"],
+                "fragment_text": _truncate(fragment),
+                "fragment_start": char_start,
+                "fragment_end": char_end,
+            })
+            seen_docs.add(result["doc_id"])
+
+        return matches[:top_k]
+
+    def _level_2_fuzzy_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            fuzzy_results = self.minhash_lsh.query(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in fuzzy_results:
+            if similarity >= settings.FINGERPRINT_THRESHOLD:
+                matches.append({
+                    "method": "fuzzy",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _level_3_semantic_match(self, text: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        try:
+            embedding_results = self.embeddings.search(text, top_k=top_k)
+        except Exception:
+            return []
+
+        sentences = _split_sentences(text)
+        fragment = _truncate(sentences[0]) if sentences else ""
+        matches = []
+        for doc_id, similarity in embedding_results:
+            if similarity >= settings.SEMANTIC_THRESHOLD:
+                matches.append({
+                    "method": "semantic",
+                    "similarity": similarity * 100,
+                    "source_id": doc_id,
+                    "fragment_text": fragment,
+                    "fragment_start": 0,
+                    "fragment_end": len(fragment),
+                })
+        return matches
+
+    def _calculate_overall_similarity(self, matches: List[Dict[str, Any]]) -> float:
+        if not matches:
+            return 0.0
+
+        weights = {"exact": 0.4, "fuzzy": 0.3, "semantic": 0.3}
+        total_weight = 0.0
+        total_similarity = 0.0
+
+        for match in matches:
+            w = weights.get(match["method"], 0.1)
+            total_similarity += match["similarity"] * w
+            total_weight += w
+
+        return round(total_similarity / total_weight if total_weight > 0 else 0, 1)

backend/app/core/embeddings.py

@@ -0,0 +1,76 @@
+import numpy as np
+import faiss
+from sentence_transformers import SentenceTransformer
+from typing import List, Tuple
+import os
+from app.config import settings
+
+class EmbeddingIndex:
+    def __init__(self, model_name: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"):
+        self.model = SentenceTransformer(model_name)
+        self.embedding_dim = self.model.get_sentence_embedding_dimension()
+        self.index = faiss.IndexHNSWFlat(self.embedding_dim, 32)
+        self.doc_ids = []
+        self.embeddings = None
+        
+        # Try to load existing index
+        self._load_index()
+
+    def add_document(self, doc_id: str, text: str):
+        embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        self.index.add(np.array([embedding], dtype=np.float32))
+        self.doc_ids.append(doc_id)
+        
+        if self.embeddings is None:
+            self.embeddings = np.array([embedding])
+        else:
+            self.embeddings = np.vstack([self.embeddings, embedding])
+
+    def batch_add(self, docs: List[Tuple[str, str]], batch_size: int = 32):
+        doc_ids = [doc[0] for doc in docs]
+        texts = [doc[1] for doc in docs]
+        
+        for i in range(0, len(texts), batch_size):
+            batch_texts = texts[i:i+batch_size]
+            embeddings = self.model.encode(batch_texts, convert_to_numpy=True)
+            
+            self.index.add(embeddings.astype(np.float32))
+            self.doc_ids.extend(doc_ids[i:i+batch_size])
+            
+            if self.embeddings is None:
+                self.embeddings = embeddings
+            else:
+                self.embeddings = np.vstack([self.embeddings, embeddings])
+
+    def search(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        query_embedding = self.model.encode([text], convert_to_numpy=True)[0]
+        query_embedding = np.array([query_embedding], dtype=np.float32)
+        
+        distances, indices = self.index.search(query_embedding, top_k)
+        
+        results = []
+        for idx, distance in zip(indices[0], distances[0]):
+            if idx < len(self.doc_ids):
+                similarity = 1.0 / (1.0 + distance)
+                results.append((self.doc_ids[idx], similarity))
+        
+        return results
+
+    def _load_index(self):
+        index_path = os.path.join(settings.INDEX_DIR, "faiss_index")
+        ids_path = os.path.join(settings.INDEX_DIR, "doc_ids.npy")
+        
+        if os.path.exists(index_path) and os.path.exists(ids_path):
+            try:
+                self.index = faiss.read_index(index_path)
+                self.doc_ids = list(np.load(ids_path, allow_pickle=True))
+            except Exception as e:
+                print(f"Failed to load index: {e}")
+
+    def save_index(self):
+        os.makedirs(settings.INDEX_DIR, exist_ok=True)
+        faiss.write_index(self.index, os.path.join(settings.INDEX_DIR, "faiss_index"))
+        np.save(os.path.join(settings.INDEX_DIR, "doc_ids.npy"), np.array(self.doc_ids, dtype=object))
+
+    def get_index_size(self) -> int:
+        return self.index.ntotal

backend/app/core/fingerprint.py

@@ -0,0 +1,82 @@
+import xxhash
+from typing import Set, Tuple, List
+from datasketch import MinHash, MinHashLSH
+import numpy as np
+
+class WinNowing:
+    def __init__(self, k: int = 5, window_size: int = 4):
+        self.k = k
+        self.window_size = window_size
+
+    def get_kgrams(self, text: str) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(len(words) - self.k + 1):
+            kgram = ' '.join(words[i:i+self.k])
+            kgrams.append(kgram)
+        return kgrams
+
+    def fingerprint(self, text: str) -> Set[int]:
+        kgrams = self.get_kgrams(text)
+        hashes = []
+        for kgram in kgrams:
+            h = int(xxhash.xxh64(kgram).hexdigest(), 16)
+            hashes.append(h)
+        
+        if not hashes:
+            return set()
+        
+        fingerprints = set()
+        for i in range(len(hashes) - self.window_size + 1):
+            window = hashes[i:i+self.window_size]
+            min_hash = min(window)
+            fingerprints.add(min_hash)
+        
+        return fingerprints
+
+    def compare(self, fp1: Set[int], fp2: Set[int]) -> float:
+        if not fp1 or not fp2:
+            return 0.0
+        intersection = len(fp1 & fp2)
+        union = len(fp1 | fp2)
+        return intersection / union if union > 0 else 0.0
+
+class MinHashLSHIndex:
+    def __init__(self, num_perm: int = 128, threshold: float = 0.5):
+        self.num_perm = num_perm
+        self.threshold = threshold
+        self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
+        self.documents = {}
+
+    def add_document(self, doc_id: str, text: str):
+        kgrams = self._get_kgrams(text)
+        m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            m.update(kgram.encode())
+        
+        self.lsh.insert(doc_id, m)
+        self.documents[doc_id] = m
+
+    def query(self, text: str, top_k: int = 10) -> List[Tuple[str, float]]:
+        kgrams = self._get_kgrams(text)
+        query_m = MinHash(num_perm=self.num_perm)
+        for kgram in kgrams:
+            query_m.update(kgram.encode())
+        
+        candidates = self.lsh.query(query_m)
+        
+        results = []
+        for doc_id in candidates:
+            similarity = query_m.jaccard(self.documents[doc_id])
+            results.append((doc_id, similarity))
+        
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def _get_kgrams(self, text: str, k: int = 5) -> List[str]:
+        words = text.split()
+        kgrams = []
+        for i in range(max(1, len(words) - k + 1)):
+            kgram = ' '.join(words[i:i+k])
+            kgrams.append(kgram)
+        return kgrams

backend/app/core/fulltext.py

@@ -0,0 +1,135 @@
+from elasticsearch import Elasticsearch
+from typing import List, Tuple, Dict, Any
+from app.config import settings
+
+class FulltextSearch:
+    def __init__(self):
+        self.es = Elasticsearch([f"http://{settings.ELASTICSEARCH_HOST}:{settings.ELASTICSEARCH_PORT}"])
+        self.index_name = "documents"
+        self._create_index()
+
+    def _create_index(self):
+        if not self.es.indices.exists(index=self.index_name):
+            self.es.indices.create(
+                index=self.index_name,
+                body={
+                    "settings": {
+                        "analysis": {
+                            "analyzer": {
+                                "ru_en": {
+                                    "type": "custom",
+                                    "tokenizer": "standard",
+                                    "filter": ["lowercase", "ru_stop", "ru_stem"]
+                                }
+                            },
+                            "filter": {
+                                "ru_stop": {
+                                    "type": "stop",
+                                    "stopwords": "_russian_"
+                                },
+                                "ru_stem": {
+                                    "type": "stemmer",
+                                    "language": "russian"
+                                }
+                            }
+                        }
+                    },
+                    "mappings": {
+                        "properties": {
+                            "doc_id": {"type": "keyword"},
+                            "text": {"type": "text", "analyzer": "ru_en"},
+                            "title": {"type": "text", "analyzer": "ru_en"},
+                            "source": {"type": "keyword"},
+                            "lang": {"type": "keyword"},
+                            "url": {"type": "keyword"}
+                        }
+                    }
+                }
+            )
+
+    def index_document(self, doc_id: str, title: str, text: str, source: str, lang: str, url: str = ""):
+        self.es.index(
+            index=self.index_name,
+            id=doc_id,
+            body={
+                "doc_id": doc_id,
+                "title": title,
+                "text": text,
+                "source": source,
+                "lang": lang,
+                "url": url
+            }
+        )
+
+    def search(self, query: str, top_k: int = 10, lang: str = None) -> List[Dict[str, Any]]:
+        must = {
+            "multi_match": {
+                "query": query,
+                "fields": ["text^2", "title^3"],
+                "type": "best_fields",
+                "analyzer": "ru_en",
+            }
+        }
+
+        if lang:
+            search_body = {
+                "size": top_k,
+                "query": {
+                    "bool": {
+                        "must": must,
+                        "filter": {"term": {"lang": lang}},
+                    }
+                },
+            }
+        else:
+            search_body = {"size": top_k, "query": must}
+
+        results = self.es.search(index=self.index_name, body=search_body)
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def search_phrase(self, phrase: str, top_k: int = 10) -> List[Dict[str, Any]]:
+        results = self.es.search(
+            index=self.index_name,
+            body={
+                "size": top_k,
+                "query": {
+                    "match_phrase": {
+                        "text": phrase
+                    }
+                }
+            }
+        )
+        
+        formatted_results = []
+        for hit in results['hits']['hits']:
+            source = hit['_source']
+            formatted_results.append({
+                "doc_id": source['doc_id'],
+                "title": source['title'],
+                "source": source['source'],
+                "url": source.get('url', ''),
+                "score": hit['_score']
+            })
+        
+        return formatted_results
+
+    def bulk_index(self, documents: List[Dict[str, Any]]):
+        actions = []
+        for doc in documents:
+            actions.append({"index": {"_index": self.index_name, "_id": doc['doc_id']}})
+            actions.append(doc)
+        
+        from elasticsearch.helpers import bulk
+        bulk(self.es, actions)