big update new dashboard and import export
This commit is contained in:
336
route/transfer_crud.py
Normal file
336
route/transfer_crud.py
Normal file
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
transfer_crud.py — полный экспорт и импорт всех данных между инстанциями API.
|
||||
|
||||
Экспортирует ВСЁ: справочники, тесты, пользователей, ответы, результаты.
|
||||
Импорт — upsert (INSERT ON CONFLICT DO NOTHING), UUID сохраняются,
|
||||
FK-связи не рвутся. Безопасно запускать повторно — дубли пропускаются.
|
||||
|
||||
Порядок импорта (FK-зависимости):
|
||||
Organization → Group → Poll → Question → Choice
|
||||
→ ScaleDimension → ChoiceScore
|
||||
→ User → Response → Answer
|
||||
→ RadarResult → RadarResultItem + SVG-файлы
|
||||
|
||||
Оба эндпоинта защищены X-Admin-Key.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from bd import make_engine
|
||||
from bd.tables.organization import Organization
|
||||
from bd.tables.group import Group
|
||||
from bd.tables.poll import Poll
|
||||
from bd.tables.question import Question
|
||||
from bd.tables.choice import Choice
|
||||
from bd.tables.scale import ScaleDimension, ChoiceScore
|
||||
from bd.tables.users import User
|
||||
from bd.tables.response import Response, Answer
|
||||
from bd.tables.radar_result import RadarResult, RadarResultItem
|
||||
from route.auth_utils import require_admin_key
|
||||
|
||||
router = APIRouter(prefix="/transfer", tags=["transfer"])
|
||||
|
||||
EXPORT_FORMAT_VERSION = "2.0"
|
||||
_APP_PATH = os.getenv("APP_PATH", "/app")
|
||||
_RADAR_DIR = Path(_APP_PATH) / "data" / "radar"
|
||||
|
||||
# Порядок экспорта/импорта — строго по FK-зависимостям
|
||||
_TABLES: list[tuple[str, object]] = [
|
||||
("organizations", Organization),
|
||||
("groups", Group),
|
||||
("polls", Poll),
|
||||
("questions", Question),
|
||||
("choices", Choice),
|
||||
("scale_dimensions", ScaleDimension),
|
||||
("choice_scores", ChoiceScore),
|
||||
("users", User),
|
||||
("responses", Response),
|
||||
("answers", Answer),
|
||||
("radar_results", RadarResult),
|
||||
("radar_result_items", RadarResultItem),
|
||||
]
|
||||
|
||||
|
||||
def _get_session():
|
||||
return sessionmaker(bind=make_engine(), future=True)
|
||||
|
||||
|
||||
def _row_to_dict(row) -> dict:
|
||||
result = {}
|
||||
for col in row.__table__.columns:
|
||||
val = getattr(row, col.name)
|
||||
if val is None:
|
||||
result[col.name] = None
|
||||
elif hasattr(val, "hex"): # UUID
|
||||
result[col.name] = str(val)
|
||||
elif hasattr(val, "isoformat"): # datetime
|
||||
result[col.name] = val.isoformat()
|
||||
else:
|
||||
result[col.name] = val
|
||||
return result
|
||||
|
||||
|
||||
# ── EXPORT ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/export",
|
||||
summary="Экспортировать все данные в ZIP-архив",
|
||||
response_class=StreamingResponse,
|
||||
dependencies=[Depends(require_admin_key)],
|
||||
)
|
||||
def export_data():
|
||||
Session = _get_session()
|
||||
buf = io.BytesIO()
|
||||
counts = {}
|
||||
|
||||
with Session() as session:
|
||||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for filename, model_class in _TABLES:
|
||||
rows = [_row_to_dict(r) for r in session.query(model_class).all()]
|
||||
counts[filename] = len(rows)
|
||||
zf.writestr(f"{filename}.json", json.dumps(rows, ensure_ascii=False))
|
||||
|
||||
# SVG-файлы радара
|
||||
svg_count = 0
|
||||
if _RADAR_DIR.exists():
|
||||
for svg_file in _RADAR_DIR.rglob("*.svg"):
|
||||
arc_name = "radar_svgs/" + svg_file.relative_to(_RADAR_DIR).as_posix()
|
||||
zf.write(svg_file, arc_name)
|
||||
svg_count += 1
|
||||
counts["svg_files"] = svg_count
|
||||
|
||||
manifest = {
|
||||
"format_version": EXPORT_FORMAT_VERSION,
|
||||
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||||
"counts": counts,
|
||||
}
|
||||
zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
buf.seek(0)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
return StreamingResponse(
|
||||
buf,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="export_{ts}.zip"'},
|
||||
)
|
||||
|
||||
|
||||
# ── IMPORT ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post(
|
||||
"/import",
|
||||
summary="Импортировать данные из ZIP-архива (upsert по UUID)",
|
||||
dependencies=[Depends(require_admin_key)],
|
||||
)
|
||||
def import_data(file: UploadFile = File(...)):
|
||||
content = file.file.read()
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
except zipfile.BadZipFile:
|
||||
raise HTTPException(status_code=400, detail="Файл не является валидным ZIP-архивом")
|
||||
|
||||
if "manifest.json" not in zf.namelist():
|
||||
raise HTTPException(status_code=400, detail="Отсутствует manifest.json")
|
||||
|
||||
manifest = json.loads(zf.read("manifest.json"))
|
||||
fmt = manifest.get("format_version", "")
|
||||
if fmt not in (EXPORT_FORMAT_VERSION, "1.0"):
|
||||
raise HTTPException(status_code=400, detail=f"Неподдерживаемая версия формата: {fmt}")
|
||||
|
||||
Session = _get_session()
|
||||
result = {"inserted": {}, "skipped": {}}
|
||||
|
||||
with Session() as session:
|
||||
for filename, model_class in _TABLES:
|
||||
arc_file = f"{filename}.json"
|
||||
if arc_file not in zf.namelist():
|
||||
result["inserted"][filename] = 0
|
||||
result["skipped"][filename] = 0
|
||||
continue
|
||||
|
||||
rows: list[dict] = json.loads(zf.read(arc_file))
|
||||
if not rows:
|
||||
result["inserted"][filename] = 0
|
||||
result["skipped"][filename] = 0
|
||||
continue
|
||||
|
||||
table = model_class.__table__
|
||||
inserted = 0
|
||||
for row in rows:
|
||||
stmt = (
|
||||
pg_insert(table)
|
||||
.values(**row)
|
||||
.on_conflict_do_nothing(index_elements=["id"])
|
||||
)
|
||||
res = session.execute(stmt)
|
||||
inserted += res.rowcount
|
||||
|
||||
result["inserted"][filename] = inserted
|
||||
result["skipped"][filename] = len(rows) - inserted
|
||||
|
||||
session.commit()
|
||||
|
||||
# SVG-файлы
|
||||
svg_names = [n for n in zf.namelist() if n.startswith("radar_svgs/")]
|
||||
svg_written = 0
|
||||
for arc_name in svg_names:
|
||||
rel_path = arc_name[len("radar_svgs/"):]
|
||||
if not rel_path:
|
||||
continue
|
||||
dest = _RADAR_DIR / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(zf.read(arc_name))
|
||||
svg_written += 1
|
||||
|
||||
result["inserted"]["svg_files"] = svg_written
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"source_exported_at": manifest.get("exported_at"),
|
||||
"result": result,
|
||||
}
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# BACKUP — полное резервное копирование и полное восстановление
|
||||
# /backup/export — скачать полный снимок БД + SVG
|
||||
# /backup/restore — ПОЛНАЯ замена БД из архива (все старые данные удаляются)
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
backup_router = APIRouter(prefix="/backup", tags=["backup"])
|
||||
|
||||
|
||||
@backup_router.get(
|
||||
"/export",
|
||||
summary="Создать резервную копию всей БД (ZIP)",
|
||||
response_class=StreamingResponse,
|
||||
dependencies=[Depends(require_admin_key)],
|
||||
)
|
||||
def backup_export():
|
||||
"""Идентично /transfer/export, но в имени файла указан префикс backup_."""
|
||||
Session = _get_session()
|
||||
buf = io.BytesIO()
|
||||
counts = {}
|
||||
|
||||
with Session() as session:
|
||||
with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for filename, model_class in _TABLES:
|
||||
rows = [_row_to_dict(r) for r in session.query(model_class).all()]
|
||||
counts[filename] = len(rows)
|
||||
zf.writestr(f"{filename}.json", json.dumps(rows, ensure_ascii=False))
|
||||
|
||||
svg_count = 0
|
||||
if _RADAR_DIR.exists():
|
||||
for svg_file in _RADAR_DIR.rglob("*.svg"):
|
||||
arc_name = "radar_svgs/" + svg_file.relative_to(_RADAR_DIR).as_posix()
|
||||
zf.write(svg_file, arc_name)
|
||||
svg_count += 1
|
||||
counts["svg_files"] = svg_count
|
||||
|
||||
manifest = {
|
||||
"format_version": EXPORT_FORMAT_VERSION,
|
||||
"backup_type": "full",
|
||||
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||||
"counts": counts,
|
||||
}
|
||||
zf.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
buf.seek(0)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
return StreamingResponse(
|
||||
buf,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="backup_{ts}.zip"'},
|
||||
)
|
||||
|
||||
|
||||
@backup_router.post(
|
||||
"/restore",
|
||||
summary="Полное восстановление БД из резервной копии (УДАЛЯЕТ все текущие данные)",
|
||||
dependencies=[Depends(require_admin_key)],
|
||||
)
|
||||
def backup_restore(file: UploadFile = File(...)):
|
||||
"""
|
||||
ВНИМАНИЕ: удаляет ВСЕ существующие данные в обратном FK-порядке,
|
||||
затем вставляет данные из архива в прямом FK-порядке.
|
||||
SVG-диаграммы тоже полностью заменяются.
|
||||
"""
|
||||
content = file.file.read()
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
except zipfile.BadZipFile:
|
||||
raise HTTPException(status_code=400, detail="Файл не является валидным ZIP-архивом")
|
||||
|
||||
if "manifest.json" not in zf.namelist():
|
||||
raise HTTPException(status_code=400, detail="Отсутствует manifest.json")
|
||||
|
||||
manifest = json.loads(zf.read("manifest.json"))
|
||||
fmt = manifest.get("format_version", "")
|
||||
if fmt not in (EXPORT_FORMAT_VERSION, "1.0"):
|
||||
raise HTTPException(status_code=400, detail=f"Неподдерживаемая версия формата: {fmt}")
|
||||
|
||||
Session = _get_session()
|
||||
deleted_counts: dict[str, int] = {}
|
||||
inserted_counts: dict[str, int] = {}
|
||||
|
||||
with Session() as session:
|
||||
# 1. Удаляем в ОБРАТНОМ порядке FK-зависимостей
|
||||
for filename, model_class in reversed(_TABLES):
|
||||
table = model_class.__table__
|
||||
res = session.execute(table.delete())
|
||||
deleted_counts[filename] = res.rowcount
|
||||
|
||||
session.commit()
|
||||
|
||||
# 2. Вставляем в ПРЯМОМ порядке FK-зависимостей
|
||||
for filename, model_class in _TABLES:
|
||||
arc_file = f"{filename}.json"
|
||||
if arc_file not in zf.namelist():
|
||||
inserted_counts[filename] = 0
|
||||
continue
|
||||
|
||||
rows: list[dict] = json.loads(zf.read(arc_file))
|
||||
if not rows:
|
||||
inserted_counts[filename] = 0
|
||||
continue
|
||||
|
||||
table = model_class.__table__
|
||||
session.execute(table.insert(), rows)
|
||||
inserted_counts[filename] = len(rows)
|
||||
|
||||
session.commit()
|
||||
|
||||
# 3. SVG-диаграммы: полностью очищаем каталог и восстанавливаем
|
||||
svg_names = [n for n in zf.namelist() if n.startswith("radar_svgs/") and n != "radar_svgs/"]
|
||||
if svg_names and _RADAR_DIR.exists():
|
||||
shutil.rmtree(_RADAR_DIR)
|
||||
_RADAR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
svg_written = 0
|
||||
for arc_name in svg_names:
|
||||
rel_path = arc_name[len("radar_svgs/"):]
|
||||
if not rel_path:
|
||||
continue
|
||||
dest = _RADAR_DIR / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(zf.read(arc_name))
|
||||
svg_written += 1
|
||||
|
||||
inserted_counts["svg_files"] = svg_written
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"source_exported_at": manifest.get("exported_at"),
|
||||
"deleted": deleted_counts,
|
||||
"inserted": inserted_counts,
|
||||
}
|
||||
Reference in New Issue
Block a user