feat: initial microservices project structure
Services: - api: FastAPI gateway with JWT auth, async endpoints, WebSocket - worker-gpu: CUDA sentence-transformers, FAISS IVFFlat, Ollama LLM - worker-indexer: Winnowing+MinHash plagiarism detection, PDF/DOCX extraction - worker-notifier: SMTP email notifications - worker-gost: GOST 7.1-2003 and GOST R 7.0.5-2008 formatting Infrastructure: - docker-compose.yml (production) + docker-compose.dev.yml (hot reload) - Nginx reverse proxy + WebSocket support - PostgreSQL 16 with Alembic migrations - Elasticsearch 8 with Russian/English analyzers - MinIO, RabbitMQ, Redis, Ollama Frontend: - React 18 + Vite + TypeScript + TailwindCSS + Zustand + React Query v5 - 9 pages: Home, Search, Cabinet, Task, Check, Bibliography, Pricing, Login, Register Scripts: - Parser stubs: OpenAlex, КиберЛенинка, arXiv (Phase 0 - to be filled) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
80
services/worker-indexer/app/extractors/pdf.py
Normal file
80
services/worker-indexer/app/extractors/pdf.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Извлечение текста из PDF файлов через PyMuPDF."""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_text_from_pdf(data: bytes) -> str:
|
||||
"""
|
||||
Извлечь текст из PDF файла.
|
||||
|
||||
Обрабатывает многостраничные документы.
|
||||
Удаляет лишние переносы строк и пробелы.
|
||||
|
||||
Args:
|
||||
data: Байты PDF файла
|
||||
|
||||
Returns:
|
||||
Извлечённый текст
|
||||
|
||||
Raises:
|
||||
ValueError: Если не удалось открыть PDF
|
||||
"""
|
||||
import fitz # PyMuPDF
|
||||
|
||||
try:
|
||||
doc = fitz.open(stream=data, filetype="pdf")
|
||||
except Exception as e:
|
||||
raise ValueError(f"Не удалось открыть PDF: {e}") from e
|
||||
|
||||
texts: list[str] = []
|
||||
|
||||
for page_num, page in enumerate(doc):
|
||||
try:
|
||||
page_text = page.get_text("text")
|
||||
if page_text.strip():
|
||||
texts.append(page_text)
|
||||
except Exception as e:
|
||||
logger.warning(f"Ошибка извлечения текста со страницы {page_num}: {e}")
|
||||
continue
|
||||
|
||||
doc.close()
|
||||
|
||||
full_text = "\n".join(texts)
|
||||
|
||||
# Нормализация: убрать множественные переносы строк
|
||||
import re
|
||||
full_text = re.sub(r"\n{3,}", "\n\n", full_text)
|
||||
full_text = re.sub(r"[ \t]+", " ", full_text)
|
||||
|
||||
return full_text.strip()
|
||||
|
||||
|
||||
def extract_metadata_from_pdf(data: bytes) -> dict:
|
||||
"""
|
||||
Извлечь метаданные из PDF (title, author, subject и т.д.).
|
||||
|
||||
Args:
|
||||
data: Байты PDF файла
|
||||
|
||||
Returns:
|
||||
Словарь метаданных
|
||||
"""
|
||||
import fitz
|
||||
|
||||
try:
|
||||
doc = fitz.open(stream=data, filetype="pdf")
|
||||
metadata = doc.metadata or {}
|
||||
doc.close()
|
||||
return {
|
||||
"title": metadata.get("title", ""),
|
||||
"author": metadata.get("author", ""),
|
||||
"subject": metadata.get("subject", ""),
|
||||
"keywords": metadata.get("keywords", ""),
|
||||
"creator": metadata.get("creator", ""),
|
||||
"pages": doc.page_count if hasattr(doc, "page_count") else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Ошибка извлечения метаданных PDF: {e}")
|
||||
return {}
|
||||
Reference in New Issue
Block a user