feat: initial microservices project structure
Services: - api: FastAPI gateway with JWT auth, async endpoints, WebSocket - worker-gpu: CUDA sentence-transformers, FAISS IVFFlat, Ollama LLM - worker-indexer: Winnowing+MinHash plagiarism detection, PDF/DOCX extraction - worker-notifier: SMTP email notifications - worker-gost: GOST 7.1-2003 and GOST R 7.0.5-2008 formatting Infrastructure: - docker-compose.yml (production) + docker-compose.dev.yml (hot reload) - Nginx reverse proxy + WebSocket support - PostgreSQL 16 with Alembic migrations - Elasticsearch 8 with Russian/English analyzers - MinIO, RabbitMQ, Redis, Ollama Frontend: - React 18 + Vite + TypeScript + TailwindCSS + Zustand + React Query v5 - 9 pages: Home, Search, Cabinet, Task, Check, Bibliography, Pricing, Login, Register Scripts: - Parser stubs: OpenAlex, КиберЛенинка, arXiv (Phase 0 - to be filled) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
69
services/worker-indexer/app/extractors/docx.py
Normal file
69
services/worker-indexer/app/extractors/docx.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""Извлечение текста из DOCX файлов через python-docx."""
|
||||
|
||||
import io
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_text_from_docx(data: bytes) -> str:
|
||||
"""
|
||||
Извлечь текст из DOCX файла.
|
||||
|
||||
Обрабатывает параграфы, таблицы и заголовки.
|
||||
|
||||
Args:
|
||||
data: Байты DOCX файла
|
||||
|
||||
Returns:
|
||||
Извлечённый текст
|
||||
|
||||
Raises:
|
||||
ValueError: Если не удалось открыть DOCX
|
||||
"""
|
||||
from docx import Document as DocxDocument
|
||||
from docx.oxml.ns import qn
|
||||
|
||||
try:
|
||||
doc = DocxDocument(io.BytesIO(data))
|
||||
except Exception as e:
|
||||
raise ValueError(f"Не удалось открыть DOCX: {e}") from e
|
||||
|
||||
texts: list[str] = []
|
||||
|
||||
# Основной текст из параграфов
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
|
||||
# Текст из таблиц
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
cell_text = cell.text.strip()
|
||||
if cell_text:
|
||||
texts.append(cell_text)
|
||||
|
||||
return "\n".join(texts)
|
||||
|
||||
|
||||
def extract_text_from_txt(data: bytes) -> str:
|
||||
"""
|
||||
Извлечь текст из TXT файла с определением кодировки.
|
||||
|
||||
Args:
|
||||
data: Байты TXT файла
|
||||
|
||||
Returns:
|
||||
Текст файла
|
||||
"""
|
||||
# Пробуем UTF-8, затем cp1251 (Windows-1251 для русских текстов)
|
||||
for encoding in ("utf-8", "cp1251", "latin-1"):
|
||||
try:
|
||||
return data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
# Последний вариант — игнорировать ошибки
|
||||
return data.decode("utf-8", errors="ignore")
|
||||
Reference in New Issue
Block a user