eac61923f4
web-react/: SPA Vite+React+TS, nginx раздаёт статику и проксирует /api на api:8000 по локалке. docker-compose: web(Flet)->web-react на :80. transfer_crud.backup_restore: чистим содержимое data/radar вместо rmtree точки монтирования (EBUSY).
337 lines
13 KiB
Python
337 lines
13 KiB
Python
"""
|
||
transfer_crud.py — полный экспорт и импорт всех данных между инстанциями API.
|
||
|
||
Экспортирует ВСЁ: справочники, тесты, пользователей, ответы, результаты.
|
||
Импорт — upsert (INSERT ON CONFLICT DO NOTHING), UUID сохраняются,
|
||
FK-связи не рвутся. Безопасно запускать повторно — дубли пропускаются.
|
||
|
||
Порядок импорта (FK-зависимости):
|
||
Organization → Group → Poll → Question → Choice
|
||
→ ScaleDimension → ChoiceScore
|
||
→ User → Response → Answer
|
||
→ RadarResult → RadarResultItem + SVG-файлы
|
||
|
||
Оба эндпоинта защищены X-Admin-Key.
|
||
"""
|
||
import io
|
||
import json
|
||
import os
|
||
import shutil
|
||
import zipfile
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||
from fastapi.responses import StreamingResponse
|
||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||
from bd import get_session
|
||
from bd.tables.organization import Organization
|
||
from bd.tables.group import Group
|
||
from bd.tables.poll import Poll
|
||
from bd.tables.question import Question
|
||
from bd.tables.choice import Choice
|
||
from bd.tables.scale import ScaleDimension, ChoiceScore
|
||
from bd.tables.users import User
|
||
from bd.tables.response import Response, Answer
|
||
from bd.tables.radar_result import RadarResult, RadarResultItem
|
||
from route.auth_utils import require_admin_key
|
||
|
||
router = APIRouter(prefix="/transfer", tags=["transfer"])
|
||
|
||
EXPORT_FORMAT_VERSION = "2.0"
|
||
_APP_PATH = os.getenv("APP_PATH", "/app")
|
||
_RADAR_DIR = Path(_APP_PATH) / "data" / "radar"
|
||
|
||
# Порядок экспорта/импорта — строго по FK-зависимостям
|
||
_TABLES: list[tuple[str, object]] = [
|
||
("organizations", Organization),
|
||
("groups", Group),
|
||
("polls", Poll),
|
||
("questions", Question),
|
||
("choices", Choice),
|
||
("scale_dimensions", ScaleDimension),
|
||
("choice_scores", ChoiceScore),
|
||
("users", User),
|
||
("responses", Response),
|
||
("answers", Answer),
|
||
("radar_results", RadarResult),
|
||
("radar_result_items", RadarResultItem),
|
||
]
|
||
|
||
|
||
|
||
|
||
def _row_to_dict(row) -> dict:
|
||
result = {}
|
||
for col in row.__table__.columns:
|
||
val = getattr(row, col.name)
|
||
if val is None:
|
||
result[col.name] = None
|
||
elif hasattr(val, "hex"): # UUID
|
||
result[col.name] = str(val)
|
||
elif hasattr(val, "isoformat"): # datetime
|
||
result[col.name] = val.isoformat()
|
||
else:
|
||
result[col.name] = val
|
||
return result
|
||
|
||
|
||
# ── EXPORT ──────────────────────────────────────────────────────────────────
|
||
|
||
@router.get(
|
||
"/export",
|
||
summary="Экспортировать все данные в ZIP-архив",
|
||
response_class=StreamingResponse,
|
||
dependencies=[Depends(require_admin_key)],
|
||
)
|
||
def export_data():
|
||
Session = get_session()
|
||
buf = io.BytesIO()
|
||
counts = {}
|
||
|
||
with Session() as session:
|
||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||
for filename, model_class in _TABLES:
|
||
rows = [_row_to_dict(r) for r in session.query(model_class).all()]
|
||
counts[filename] = len(rows)
|
||
zf.writestr(f"{filename}.json", json.dumps(rows, ensure_ascii=False))
|
||
|
||
# SVG-файлы радара
|
||
svg_count = 0
|
||
if _RADAR_DIR.exists():
|
||
for svg_file in _RADAR_DIR.rglob("*.svg"):
|
||
arc_name = "radar_svgs/" + svg_file.relative_to(_RADAR_DIR).as_posix()
|
||
zf.write(svg_file, arc_name)
|
||
svg_count += 1
|
||
counts["svg_files"] = svg_count
|
||
|
||
manifest = {
|
||
"format_version": EXPORT_FORMAT_VERSION,
|
||
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||
"counts": counts,
|
||
}
|
||
zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
||
|
||
buf.seek(0)
|
||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||
return StreamingResponse(
|
||
buf,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="export_{ts}.zip"'},
|
||
)
|
||
|
||
|
||
# ── IMPORT ──────────────────────────────────────────────────────────────────
|
||
|
||
@router.post(
|
||
"/import",
|
||
summary="Импортировать данные из ZIP-архива (upsert по UUID)",
|
||
dependencies=[Depends(require_admin_key)],
|
||
)
|
||
def import_data(file: UploadFile = File(...)):
|
||
content = file.file.read()
|
||
try:
|
||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(status_code=400, detail="Файл не является валидным ZIP-архивом")
|
||
|
||
if "manifest.json" not in zf.namelist():
|
||
raise HTTPException(status_code=400, detail="Отсутствует manifest.json")
|
||
|
||
manifest = json.loads(zf.read("manifest.json"))
|
||
fmt = manifest.get("format_version", "")
|
||
if fmt not in (EXPORT_FORMAT_VERSION, "1.0"):
|
||
raise HTTPException(status_code=400, detail=f"Неподдерживаемая версия формата: {fmt}")
|
||
|
||
Session = get_session()
|
||
result = {"inserted": {}, "skipped": {}}
|
||
|
||
with Session() as session:
|
||
for filename, model_class in _TABLES:
|
||
arc_file = f"{filename}.json"
|
||
if arc_file not in zf.namelist():
|
||
result["inserted"][filename] = 0
|
||
result["skipped"][filename] = 0
|
||
continue
|
||
|
||
rows: list[dict] = json.loads(zf.read(arc_file))
|
||
if not rows:
|
||
result["inserted"][filename] = 0
|
||
result["skipped"][filename] = 0
|
||
continue
|
||
|
||
table = model_class.__table__
|
||
inserted = 0
|
||
for row in rows:
|
||
stmt = (
|
||
pg_insert(table)
|
||
.values(**row)
|
||
.on_conflict_do_nothing(index_elements=["id"])
|
||
)
|
||
res = session.execute(stmt)
|
||
inserted += res.rowcount
|
||
|
||
result["inserted"][filename] = inserted
|
||
result["skipped"][filename] = len(rows) - inserted
|
||
|
||
session.commit()
|
||
|
||
# SVG-файлы
|
||
svg_names = [n for n in zf.namelist() if n.startswith("radar_svgs/")]
|
||
svg_written = 0
|
||
for arc_name in svg_names:
|
||
rel_path = arc_name[len("radar_svgs/"):]
|
||
if not rel_path:
|
||
continue
|
||
dest = _RADAR_DIR / rel_path
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
dest.write_bytes(zf.read(arc_name))
|
||
svg_written += 1
|
||
|
||
result["inserted"]["svg_files"] = svg_written
|
||
|
||
return {
|
||
"status": "ok",
|
||
"source_exported_at": manifest.get("exported_at"),
|
||
"result": result,
|
||
}
|
||
|
||
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
# BACKUP — полное резервное копирование и полное восстановление
|
||
# /backup/export — скачать полный снимок БД + SVG
|
||
# /backup/restore — ПОЛНАЯ замена БД из архива (все старые данные удаляются)
|
||
# ════════════════════════════════════════════════════════════════════════════
|
||
|
||
backup_router = APIRouter(prefix="/backup", tags=["backup"])
|
||
|
||
|
||
@backup_router.get(
|
||
"/export",
|
||
summary="Создать резервную копию всей БД (ZIP)",
|
||
response_class=StreamingResponse,
|
||
dependencies=[Depends(require_admin_key)],
|
||
)
|
||
def backup_export():
|
||
"""Идентично /transfer/export, но в имени файла указан префикс backup_."""
|
||
Session = get_session()
|
||
buf = io.BytesIO()
|
||
counts = {}
|
||
|
||
with Session() as session:
|
||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||
for filename, model_class in _TABLES:
|
||
rows = [_row_to_dict(r) for r in session.query(model_class).all()]
|
||
counts[filename] = len(rows)
|
||
zf.writestr(f"{filename}.json", json.dumps(rows, ensure_ascii=False))
|
||
|
||
svg_count = 0
|
||
if _RADAR_DIR.exists():
|
||
for svg_file in _RADAR_DIR.rglob("*.svg"):
|
||
arc_name = "radar_svgs/" + svg_file.relative_to(_RADAR_DIR).as_posix()
|
||
zf.write(svg_file, arc_name)
|
||
svg_count += 1
|
||
counts["svg_files"] = svg_count
|
||
|
||
manifest = {
|
||
"format_version": EXPORT_FORMAT_VERSION,
|
||
"backup_type": "full",
|
||
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||
"counts": counts,
|
||
}
|
||
zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
||
|
||
buf.seek(0)
|
||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||
return StreamingResponse(
|
||
buf,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="backup_{ts}.zip"'},
|
||
)
|
||
|
||
|
||
@backup_router.post(
|
||
"/restore",
|
||
summary="Полное восстановление БД из резервной копии (УДАЛЯЕТ все текущие данные)",
|
||
dependencies=[Depends(require_admin_key)],
|
||
)
|
||
def backup_restore(file: UploadFile = File(...)):
|
||
"""
|
||
ВНИМАНИЕ: удаляет ВСЕ существующие данные в обратном FK-порядке,
|
||
затем вставляет данные из архива в прямом FK-порядке.
|
||
SVG-диаграммы тоже полностью заменяются.
|
||
"""
|
||
content = file.file.read()
|
||
try:
|
||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(status_code=400, detail="Файл не является валидным ZIP-архивом")
|
||
|
||
if "manifest.json" not in zf.namelist():
|
||
raise HTTPException(status_code=400, detail="Отсутствует manifest.json")
|
||
|
||
manifest = json.loads(zf.read("manifest.json"))
|
||
fmt = manifest.get("format_version", "")
|
||
if fmt not in (EXPORT_FORMAT_VERSION, "1.0"):
|
||
raise HTTPException(status_code=400, detail=f"Неподдерживаемая версия формата: {fmt}")
|
||
|
||
Session = get_session()
|
||
deleted_counts: dict[str, int] = {}
|
||
inserted_counts: dict[str, int] = {}
|
||
|
||
with Session() as session:
|
||
# 1. Удаляем в ОБРАТНОМ порядке FK-зависимостей
|
||
for filename, model_class in reversed(_TABLES):
|
||
table = model_class.__table__
|
||
res = session.execute(table.delete())
|
||
deleted_counts[filename] = res.rowcount
|
||
|
||
session.commit()
|
||
|
||
# 2. Вставляем в ПРЯМОМ порядке FK-зависимостей
|
||
for filename, model_class in _TABLES:
|
||
arc_file = f"{filename}.json"
|
||
if arc_file not in zf.namelist():
|
||
inserted_counts[filename] = 0
|
||
continue
|
||
|
||
rows: list[dict] = json.loads(zf.read(arc_file))
|
||
if not rows:
|
||
inserted_counts[filename] = 0
|
||
continue
|
||
|
||
table = model_class.__table__
|
||
session.execute(table.insert(), rows)
|
||
inserted_counts[filename] = len(rows)
|
||
|
||
session.commit()
|
||
|
||
# 3. SVG-диаграммы: полностью очищаем каталог и восстанавливаем
|
||
svg_names = [n for n in zf.namelist() if n.startswith("radar_svgs/") and n != "radar_svgs/"]
|
||
if svg_names and _RADAR_DIR.exists():
|
||
for child in _RADAR_DIR.iterdir():
|
||
if child.is_dir():
|
||
shutil.rmtree(child)
|
||
else:
|
||
child.unlink()
|
||
_RADAR_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
svg_written = 0
|
||
for arc_name in svg_names:
|
||
rel_path = arc_name[len("radar_svgs/"):]
|
||
if not rel_path:
|
||
continue
|
||
dest = _RADAR_DIR / rel_path
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
dest.write_bytes(zf.read(arc_name))
|
||
svg_written += 1
|
||
|
||
inserted_counts["svg_files"] = svg_written
|
||
|
||
return {
|
||
"status": "ok",
|
||
"source_exported_at": manifest.get("exported_at"),
|
||
"deleted": deleted_counts,
|
||
"inserted": inserted_counts,
|
||
}
|