feat: initial microservices project structure
Services: - api: FastAPI gateway with JWT auth, async endpoints, WebSocket - worker-gpu: CUDA sentence-transformers, FAISS IVFFlat, Ollama LLM - worker-indexer: Winnowing+MinHash plagiarism detection, PDF/DOCX extraction - worker-notifier: SMTP email notifications - worker-gost: GOST 7.1-2003 and GOST R 7.0.5-2008 formatting Infrastructure: - docker-compose.yml (production) + docker-compose.dev.yml (hot reload) - Nginx reverse proxy + WebSocket support - PostgreSQL 16 with Alembic migrations - Elasticsearch 8 with Russian/English analyzers - MinIO, RabbitMQ, Redis, Ollama Frontend: - React 18 + Vite + TypeScript + TailwindCSS + Zustand + React Query v5 - 9 pages: Home, Search, Cabinet, Task, Check, Bibliography, Pricing, Login, Register Scripts: - Parser stubs: OpenAlex, КиберЛенинка, arXiv (Phase 0 - to be filled) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jze9
245
scripts/parsers/openalex.py
Normal file
245
scripts/parsers/openalex.py
Normal file
@@ -0,0 +1,245 @@
|
||||
"""Парсер OpenAlex API.
|
||||
|
||||
OpenAlex — открытая академическая база данных с >250 млн работ.
|
||||
API: https://docs.openalex.org/
|
||||
|
||||
Особенности:
|
||||
- Cursor-based пагинация (не offset, чтобы не было дублей)
|
||||
- Rate limit: 100,000 запросов/день без ключа
|
||||
- Идемпотентность: проверка по ext_id перед добавлением
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Any, Generator
|
||||
|
||||
import httpx
|
||||
|
||||
from base import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENALEX_API = "https://api.openalex.org"
|
||||
DEFAULT_EMAIL = "noreply@jze9.ru" # Для вежливого агента
|
||||
|
||||
|
||||
class OpenAlexParser(BaseParser):
|
||||
"""Парсер OpenAlex API с cursor-based пагинацией."""
|
||||
|
||||
source_name = "openalex"
|
||||
|
||||
def __init__(self, email: str = DEFAULT_EMAIL) -> None:
|
||||
super().__init__()
|
||||
self.email = email
|
||||
self.client = httpx.Client(
|
||||
headers={"User-Agent": f"AcademicHelper/1.0 ({email})"},
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
query: str = "",
|
||||
limit: int = 1000,
|
||||
lang: str | None = None,
|
||||
year_from: int | None = None,
|
||||
year_to: int | None = None,
|
||||
type_filter: str = "journal-article",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Получить документы из OpenAlex.
|
||||
|
||||
Args:
|
||||
query: Поисковый запрос
|
||||
limit: Максимальное количество документов
|
||||
lang: Язык ('ru', 'en', None = все)
|
||||
year_from: Год публикации от
|
||||
year_to: Год публикации до
|
||||
type_filter: Тип документа (journal-article, book, и т.д.)
|
||||
|
||||
Returns:
|
||||
Список сырых словарей из OpenAlex API
|
||||
"""
|
||||
results = []
|
||||
|
||||
for page in self._paginate(
|
||||
query=query,
|
||||
limit=limit,
|
||||
lang=lang,
|
||||
year_from=year_from,
|
||||
year_to=year_to,
|
||||
type_filter=type_filter,
|
||||
):
|
||||
results.extend(page)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
def _paginate(
|
||||
self,
|
||||
query: str,
|
||||
limit: int,
|
||||
lang: str | None,
|
||||
year_from: int | None,
|
||||
year_to: int | None,
|
||||
type_filter: str,
|
||||
) -> Generator[list[dict], None, None]:
|
||||
"""Cursor-based пагинация OpenAlex."""
|
||||
cursor = "*"
|
||||
per_page = min(200, limit)
|
||||
total_fetched = 0
|
||||
|
||||
while total_fetched < limit:
|
||||
params: dict[str, Any] = {
|
||||
"per-page": per_page,
|
||||
"cursor": cursor,
|
||||
"mailto": self.email,
|
||||
}
|
||||
|
||||
# Фильтры
|
||||
filters = [f"type:{type_filter}", "is_oa:true"]
|
||||
|
||||
if query:
|
||||
params["search"] = query
|
||||
|
||||
if lang:
|
||||
filters.append(f"language:{lang}")
|
||||
|
||||
if year_from and year_to:
|
||||
filters.append(f"publication_year:{year_from}-{year_to}")
|
||||
elif year_from:
|
||||
filters.append(f"publication_year:>{year_from}")
|
||||
elif year_to:
|
||||
filters.append(f"publication_year:<{year_to}")
|
||||
|
||||
params["filter"] = ",".join(filters)
|
||||
|
||||
try:
|
||||
response = self.client.get(f"{OPENALEX_API}/works", params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
works = data.get("results", [])
|
||||
if not works:
|
||||
break
|
||||
|
||||
yield works
|
||||
total_fetched += len(works)
|
||||
|
||||
# Следующий курсор
|
||||
cursor = data.get("meta", {}).get("next_cursor")
|
||||
if not cursor:
|
||||
break
|
||||
|
||||
# Rate limiting: 10 запросов/сек без ключа
|
||||
time.sleep(0.1)
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"OpenAlex HTTP ошибка: {e.response.status_code}")
|
||||
if e.response.status_code == 429:
|
||||
logger.warning("Rate limit! Ожидаем 60 секунд...")
|
||||
time.sleep(60)
|
||||
continue
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Ошибка запроса OpenAlex: {e}")
|
||||
break
|
||||
|
||||
def transform(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Преобразовать документ OpenAlex в унифицированный формат.
|
||||
|
||||
OpenAlex структура:
|
||||
- id: строка вида "https://openalex.org/W..."
|
||||
- doi: DOI ссылка
|
||||
- title: название
|
||||
- authorships: список авторов
|
||||
- publication_year: год
|
||||
- primary_location.source.display_name: журнал
|
||||
- open_access.oa_url: ссылка на PDF
|
||||
- abstract_inverted_index: инвертированный индекс аннотации
|
||||
"""
|
||||
# Нормализовать ext_id (убрать URL-часть)
|
||||
raw_id = raw.get("id", "")
|
||||
ext_id = raw_id.replace("https://openalex.org/", "") if raw_id else ""
|
||||
|
||||
if not ext_id:
|
||||
return {}
|
||||
|
||||
# Авторы
|
||||
authorships = raw.get("authorships", [])
|
||||
authors_raw = []
|
||||
for a in authorships[:10]: # Максимум 10 авторов
|
||||
author = a.get("author", {})
|
||||
display_name = author.get("display_name", "")
|
||||
if display_name:
|
||||
# OpenAlex даёт "Иван Иванов" → разбиваем
|
||||
parts = display_name.strip().split()
|
||||
if len(parts) >= 2:
|
||||
authors_raw.append({
|
||||
"last_name": parts[-1],
|
||||
"first_name": " ".join(parts[:-1]),
|
||||
})
|
||||
elif parts:
|
||||
authors_raw.append({"last_name": parts[0], "first_name": ""})
|
||||
|
||||
authors = self.normalize_authors(authors_raw)
|
||||
|
||||
# Журнал
|
||||
primary_location = raw.get("primary_location") or {}
|
||||
source = primary_location.get("source") or {}
|
||||
journal = source.get("display_name")
|
||||
|
||||
# URL на полный текст
|
||||
oa = raw.get("open_access") or {}
|
||||
url = oa.get("oa_url") or primary_location.get("landing_page_url")
|
||||
|
||||
# Аннотация (восстановить из инвертированного индекса)
|
||||
abstract = None
|
||||
inv_index = raw.get("abstract_inverted_index")
|
||||
if inv_index:
|
||||
abstract = _reconstruct_abstract(inv_index)
|
||||
|
||||
# Бибинформация
|
||||
biblio = raw.get("biblio") or {}
|
||||
|
||||
# DOI
|
||||
doi = raw.get("doi", "")
|
||||
if doi and doi.startswith("https://doi.org/"):
|
||||
doi = doi.replace("https://doi.org/", "")
|
||||
|
||||
# Язык
|
||||
lang = raw.get("language")
|
||||
|
||||
return {
|
||||
"source": self.source_name,
|
||||
"ext_id": ext_id,
|
||||
"doi": doi or None,
|
||||
"title": (raw.get("title") or "").strip() or None,
|
||||
"authors": authors,
|
||||
"year": raw.get("publication_year"),
|
||||
"lang": lang,
|
||||
"journal": journal,
|
||||
"volume": biblio.get("volume"),
|
||||
"issue": biblio.get("issue"),
|
||||
"pages": f"{biblio.get('first_page', '')}-{biblio.get('last_page', '')}".strip("-") or None,
|
||||
"abstract": abstract,
|
||||
"url": url,
|
||||
"full_text": None, # Полный текст скачивается отдельно
|
||||
}
|
||||
|
||||
|
||||
def _reconstruct_abstract(inverted_index: dict) -> str:
|
||||
"""
|
||||
Восстановить аннотацию из инвертированного индекса OpenAlex.
|
||||
|
||||
Инвертированный индекс: {слово: [позиция1, позиция2, ...], ...}
|
||||
"""
|
||||
try:
|
||||
positions: dict[int, str] = {}
|
||||
for word, pos_list in inverted_index.items():
|
||||
for pos in pos_list:
|
||||
positions[pos] = word
|
||||
return " ".join(positions[k] for k in sorted(positions.keys()))
|
||||
except Exception:
|
||||
return ""
|
||||
Reference in New Issue
Block a user