feat: initial microservices project structure
Services: - api: FastAPI gateway with JWT auth, async endpoints, WebSocket - worker-gpu: CUDA sentence-transformers, FAISS IVFFlat, Ollama LLM - worker-indexer: Winnowing+MinHash plagiarism detection, PDF/DOCX extraction - worker-notifier: SMTP email notifications - worker-gost: GOST 7.1-2003 and GOST R 7.0.5-2008 formatting Infrastructure: - docker-compose.yml (production) + docker-compose.dev.yml (hot reload) - Nginx reverse proxy + WebSocket support - PostgreSQL 16 with Alembic migrations - Elasticsearch 8 with Russian/English analyzers - MinIO, RabbitMQ, Redis, Ollama Frontend: - React 18 + Vite + TypeScript + TailwindCSS + Zustand + React Query v5 - 9 pages: Home, Search, Cabinet, Task, Check, Bibliography, Pricing, Login, Register Scripts: - Parser stubs: OpenAlex, КиберЛенинка, arXiv (Phase 0 - to be filled) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
75
infra/elasticsearch/mappings.json
Normal file
75
infra/elasticsearch/mappings.json
Normal file
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"russian_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "russian_stop", "russian_stemmer"]
|
||||
},
|
||||
"english_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "english_stop", "english_stemmer"]
|
||||
},
|
||||
"multilingual_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "russian_stop", "russian_stemmer", "english_stop"]
|
||||
}
|
||||
},
|
||||
"filter": {
|
||||
"russian_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_russian_"
|
||||
},
|
||||
"russian_stemmer": {
|
||||
"type": "stemmer",
|
||||
"language": "russian"
|
||||
},
|
||||
"english_stop": {
|
||||
"type": "stop",
|
||||
"stopwords": "_english_"
|
||||
},
|
||||
"english_stemmer": {
|
||||
"type": "stemmer",
|
||||
"language": "english"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"doc_id": {"type": "long"},
|
||||
"source": {"type": "keyword"},
|
||||
"title": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer",
|
||||
"fields": {
|
||||
"keyword": {"type": "keyword", "ignore_above": 512},
|
||||
"ru": {"type": "text", "analyzer": "russian_analyzer"},
|
||||
"en": {"type": "text", "analyzer": "english_analyzer"}
|
||||
}
|
||||
},
|
||||
"abstract": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer",
|
||||
"fields": {
|
||||
"ru": {"type": "text", "analyzer": "russian_analyzer"},
|
||||
"en": {"type": "text", "analyzer": "english_analyzer"}
|
||||
}
|
||||
},
|
||||
"authors": {
|
||||
"type": "text",
|
||||
"analyzer": "multilingual_analyzer"
|
||||
},
|
||||
"year": {"type": "integer"},
|
||||
"lang": {"type": "keyword"},
|
||||
"journal": {"type": "keyword", "ignore_above": 512},
|
||||
"doi": {"type": "keyword"},
|
||||
"url": {"type": "keyword", "index": false}
|
||||
}
|
||||
}
|
||||
}
|
||||
36
infra/elasticsearch/setup.sh
Executable file
36
infra/elasticsearch/setup.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Создание ES индекса с маппингом для документов
|
||||
# Запускать после старта контейнера Elasticsearch
|
||||
|
||||
set -e
|
||||
|
||||
ES_URL="${ELASTICSEARCH_URL:-http://localhost:9200}"
|
||||
INDEX_NAME="documents"
|
||||
MAPPINGS_FILE="$(dirname "$0")/mappings.json"
|
||||
|
||||
echo "Ожидание готовности Elasticsearch..."
|
||||
until curl -s "$ES_URL/_cluster/health" | grep -qv '"status":"red"'; do
|
||||
sleep 2
|
||||
done
|
||||
echo "Elasticsearch готов."
|
||||
|
||||
# Проверить существование индекса
|
||||
if curl -s -o /dev/null -w "%{http_code}" "$ES_URL/$INDEX_NAME" | grep -q "200"; then
|
||||
echo "Индекс '$INDEX_NAME' уже существует."
|
||||
read -p "Пересоздать? (y/N): " confirm
|
||||
if [[ $confirm != "y" ]]; then
|
||||
echo "Пропускаем."
|
||||
exit 0
|
||||
fi
|
||||
echo "Удаление старого индекса..."
|
||||
curl -s -X DELETE "$ES_URL/$INDEX_NAME"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "Создание индекса '$INDEX_NAME'..."
|
||||
curl -s -X PUT "$ES_URL/$INDEX_NAME" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @"$MAPPINGS_FILE" | python3 -m json.tool
|
||||
|
||||
echo ""
|
||||
echo "Индекс '$INDEX_NAME' создан."
|
||||
Reference in New Issue
Block a user