test build

backend/scripts/index_documents.py

@@ -0,0 +1,119 @@
+"""
+Index pre-parsed documents (JSON files from parse_pdf.py / parse_wikipedia.py)
+into PostgreSQL, Elasticsearch, FAISS, and MinHash LSH.
+
+Usage:
+    python scripts/index_documents.py <dir_with_json_files> [--limit N]
+"""
+import sys
+import os
+import json
+import argparse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from app.config import settings
+from app.models.db import SessionLocal, Document, Fingerprint, engine, Base
+from app.core.fingerprint import WinNowing, MinHashLSHIndex
+from app.core.embeddings import EmbeddingIndex
+from app.core.fulltext import FulltextSearch
+
+
+Base.metadata.create_all(bind=engine)
+
+
+def load_json_docs(directory: str, limit: int | None = None):
+    docs = []
+    for path in sorted(Path(directory).glob("*.json")):
+        try:
+            with open(path, encoding="utf-8") as f:
+                doc = json.load(f)
+            if doc.get("text") and doc.get("title"):
+                docs.append(doc)
+        except Exception as e:
+            print(f"Skip {path.name}: {e}")
+        if limit and len(docs) >= limit:
+            break
+    return docs
+
+
+def index_batch(docs: list, batch_size: int = 32):
+    db = SessionLocal()
+    winnowing = WinNowing()
+    minhash_lsh = MinHashLSHIndex()
+    embeddings = EmbeddingIndex()
+    fulltext = FulltextSearch()
+
+    print(f"Indexing {len(docs)} documents …")
+
+    embed_batch: list[tuple[str, str]] = []
+
+    for i, doc in enumerate(docs):
+        source = doc.get("source", "unknown")
+        ext_id = doc.get("id", "")
+        title = doc["title"]
+        text = doc["text"]
+        url = doc.get("url", "")
+        lang = doc.get("lang", "ru")
+
+        # 1. PostgreSQL
+        db_doc = Document(source=source, ext_id=ext_id, title=title, url=url, lang=lang)
+        db.add(db_doc)
+        db.flush()
+
+        # 2. Fingerprints → PostgreSQL
+        fp = winnowing.fingerprint(text)
+        for pos, h in enumerate(list(fp)[:500]):
+            db.add(Fingerprint(doc_id=db_doc.id, hash=format(h, '016x'), position=pos))
+
+        # 3. MinHash LSH (in-memory; call save separately if needed)
+        try:
+            minhash_lsh.add_document(str(db_doc.id), text)
+        except ValueError:
+            pass
+
+        # 4. Elasticsearch
+        fulltext.index_document(str(db_doc.id), title, text, source, lang, url)
+
+        # 5. Accumulate for batch embedding
+        embed_batch.append((str(db_doc.id), text))
+
+        if len(embed_batch) >= batch_size:
+            embeddings.batch_add(embed_batch, batch_size=batch_size)
+            embed_batch.clear()
+
+        if (i + 1) % 100 == 0:
+            db.commit()
+            print(f"  {i + 1}/{len(docs)}")
+
+    # Flush remaining embeddings
+    if embed_batch:
+        embeddings.batch_add(embed_batch, batch_size=batch_size)
+
+    db.commit()
+    db.close()
+
+    embeddings.save_index()
+    print("Done. FAISS index saved.")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", help="Directory with parsed JSON files")
+    parser.add_argument("--limit", type=int, default=None, help="Max documents to index")
+    parser.add_argument("--batch-size", type=int, default=32)
+    args = parser.parse_args()
+
+    docs = load_json_docs(args.directory, args.limit)
+    print(f"Loaded {len(docs)} documents from {args.directory}")
+
+    if not docs:
+        print("No documents found, exiting.")
+        sys.exit(0)
+
+    index_batch(docs, batch_size=args.batch_size)
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/normalize_text.py

@@ -0,0 +1,83 @@
+"""
+Normalize extracted text before indexing.
+
+Usage:
+    python scripts/normalize_text.py <input_dir> <output_dir>
+
+Reads JSON files produced by parse_wikipedia.py / parse_pdf.py,
+cleans the text field, writes cleaned files to output_dir.
+"""
+import sys
+import re
+import json
+import argparse
+from pathlib import Path
+
+
+def normalize(text: str) -> str:
+    # Collapse unicode whitespace and control characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]', '', text)
+
+    # Normalize various dash/hyphen variants to a regular hyphen
+    text = re.sub(r'[—–‒―]', '-', text)
+
+    # Remove repeated punctuation (e.g. "!!!" → "!")
+    text = re.sub(r'([!?.]){2,}', r'\1', text)
+
+    # Collapse multiple spaces / tabs to a single space
+    text = re.sub(r'[ \t]+', ' ', text)
+
+    # Collapse more than two consecutive newlines
+    text = re.sub(r'\n{3,}', '\n\n', text)
+
+    # Strip leading/trailing whitespace per line
+    lines = [line.strip() for line in text.splitlines()]
+    text = '\n'.join(lines)
+
+    return text.strip()
+
+
+def process_directory(input_dir: str, output_dir: str) -> int:
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    count = 0
+    for src in sorted(Path(input_dir).glob('*.json')):
+        try:
+            with open(src, encoding='utf-8') as f:
+                doc = json.load(f)
+        except Exception as e:
+            print(f'Skip {src.name}: {e}')
+            continue
+
+        if 'text' not in doc:
+            continue
+
+        doc['text'] = normalize(doc['text'])
+
+        if len(doc['text'].split()) < 30:
+            continue
+
+        dest = out_path / src.name
+        with open(dest, 'w', encoding='utf-8') as f:
+            json.dump(doc, f, ensure_ascii=False, indent=2)
+
+        count += 1
+        if count % 1000 == 0:
+            print(f'Normalized {count} documents')
+
+    return count
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_dir')
+    parser.add_argument('output_dir')
+    args = parser.parse_args()
+
+    count = process_directory(args.input_dir, args.output_dir)
+    print(f'Done: {count} documents written to {args.output_dir}')
+
+
+if __name__ == '__main__':
+    main()

backend/scripts/parse_pdf.py

@@ -0,0 +1,90 @@
+import os
+import json
+import pymupdf
+from pathlib import Path
+from typing import Dict, Any, List
+
+def extract_text_from_pdf(pdf_path: str) -> Dict[str, Any]:
+    """Extract text and metadata from PDF"""
+    try:
+        doc = pymupdf.open(pdf_path)
+        
+        text = ""
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text += page.get_text()
+        
+        metadata = doc.metadata
+        doc.close()
+        
+        # Basic cleaning
+        text = clean_text(text)
+        
+        return {
+            "title": metadata.get('title', Path(pdf_path).stem),
+            "text": text,
+            "author": metadata.get('author', ''),
+            "subject": metadata.get('subject', ''),
+            "num_pages": len(doc)
+        }
+    except Exception as e:
+        print(f"Error processing {pdf_path}: {e}")
+        return None
+
+def process_pdf_directory(input_dir: str, output_dir: str) -> int:
+    """Process all PDFs in a directory"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    for pdf_file in Path(input_dir).glob("*.pdf"):
+        result = extract_text_from_pdf(str(pdf_file))
+        
+        if result and result.get('text'):
+            output_file = os.path.join(output_dir, f"{pdf_file.stem}.json")
+            
+            doc = {
+                "id": f"pdf_{pdf_file.stem}",
+                "title": result['title'],
+                "text": result['text'],
+                "source": "pdf",
+                "metadata": {
+                    "author": result.get('author'),
+                    "subject": result.get('subject'),
+                    "pages": result.get('num_pages')
+                }
+            }
+            
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(doc, f, ensure_ascii=False, indent=2)
+            
+            count += 1
+            if count % 100 == 0:
+                print(f"Processed {count} PDFs")
+    
+    return count
+
+def clean_text(text: str) -> str:
+    """Clean extracted PDF text"""
+    import re
+    
+    # Remove multiple spaces
+    text = re.sub(r'  +', ' ', text)
+    # Remove multiple newlines
+    text = re.sub(r'\n\n+', '\n', text)
+    # Remove special characters
+    text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_pdf.py <input_dir> <output_dir>")
+        sys.exit(1)
+    
+    input_dir = sys.argv[1]
+    output_dir = sys.argv[2]
+    
+    count = process_pdf_directory(input_dir, output_dir)
+    print(f"Successfully processed {count} PDFs")

backend/scripts/parse_wikipedia.py

@@ -0,0 +1,110 @@
+import bz2
+import xml.etree.ElementTree as ET
+import json
+import os
+from pathlib import Path
+from typing import Generator, Dict, Any
+
+def extract_text_from_wikipedia_dump(bz2_path: str, output_dir: str, limit: int = None) -> int:
+    """Extract text from Wikipedia XML dump"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    count = 0
+    with bz2.BZ2File(bz2_path, 'r') as f:
+        context = ET.iterparse(f, events=('start', 'end'))
+        context = iter(context)
+        
+        event, root = next(context)
+        
+        for event, elem in context:
+            if event == 'end' and elem.tag.endswith('page'):
+                doc = parse_page(elem)
+                if doc and doc.get('text'):
+                    filename = f"{output_dir}/wiki_{doc['id']}.json"
+                    with open(filename, 'w', encoding='utf-8') as out:
+                        json.dump(doc, out, ensure_ascii=False)
+                    
+                    count += 1
+                    if count % 1000 == 0:
+                        print(f"Processed {count} documents")
+                    
+                    if limit and count >= limit:
+                        break
+                
+                root.clear()
+    
+    return count
+
+def parse_page(page_elem) -> Dict[str, Any]:
+    """Parse a single Wikipedia page"""
+    ns = {'wiki': 'http://www.mediawiki.org/xml/export-0.10/'}
+    
+    title_elem = page_elem.find('wiki:title', ns) or page_elem.find('.//title')
+    page_id_elem = page_elem.find('wiki:id', ns) or page_elem.find('.//id')
+    revision_elem = page_elem.find('.//revision')
+    
+    if revision_elem is None:
+        return None
+    
+    text_elem = revision_elem.find('wiki:text', ns) or revision_elem.find('.//text')
+    
+    if title_elem is None or text_elem is None:
+        return None
+    
+    title = title_elem.text or ""
+    page_id = page_id_elem.text if page_id_elem is not None else ""
+    text = text_elem.text or ""
+    
+    # Skip redirects and special pages
+    if text.startswith('#REDIRECT') or title.startswith('Wikipedia:'):
+        return None
+    
+    # Basic text cleaning
+    text = clean_wikipedia_markup(text)
+    
+    if len(text.split()) < 50:
+        return None
+    
+    return {
+        "id": f"wiki_{page_id}",
+        "title": title,
+        "text": text,
+        "source": "wikipedia",
+        "lang": "ru"  # Change for English
+    }
+
+def clean_wikipedia_markup(text: str) -> str:
+    """Remove Wikipedia markup"""
+    import re
+    
+    # Remove wiki links
+    text = re.sub(r'\[\[([^\|]*\|)?([^\]]*)\]\]', r'\2', text)
+    # Remove external links
+    text = re.sub(r'\[http[^\s]+ ([^\]]*)\]', r'\1', text)
+    # Remove templates
+    text = re.sub(r'\{\{[^}]*\}\}', '', text)
+    # Remove categories
+    text = re.sub(r'\[\[Category:[^\]]*\]\]', '', text)
+    # Remove refs
+    text = re.sub(r'<ref[^>]*>[^<]*</ref>', '', text)
+    # Remove HTML tags
+    text = re.sub(r'<[^>]*>', '', text)
+    # Clean whitespace
+    text = re.sub(r'\n\n+', '\n', text)
+    text = re.sub(r' +', ' ', text)
+    
+    return text.strip()
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) < 3:
+        print("Usage: python parse_wikipedia.py <input_bz2> <output_dir> [limit]")
+        sys.exit(1)
+    
+    input_file = sys.argv[1]
+    output_dir = sys.argv[2]
+    limit = int(sys.argv[3]) if len(sys.argv) > 3 else None
+    
+    count = extract_text_from_wikipedia_dump(input_file, output_dir, limit)
+    print(f"Successfully processed {count} documents")