Разделы, фильтры и просмотр текстов в UI; опция «не хранить видео»

- таблица sections (дерево через parent_id), у видео section_id
- API: CRUD /sections, фильтры /videos (поиск, раздел с потомками, метод),
  GET /videos/{id}/text, PATCH раздела, DELETE вместе с файлами
- pipeline: keep_video=false удаляет mp4 после расшифровки, section_id
- UI: панель разделов с вложенностью и счётчиками, поиск, фильтр по методу,
  диалоги полного текста/выжимки, привязка к разделу, удаление,
  переключатель «Сохранять видео на диске»
- compose: bind-mount ./web для правок без пересборки

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

api/db/connection.py

@@ -8,6 +8,7 @@ from sqlalchemy.orm import sessionmaker
 
 from api.config import settings
 from api.models.base import Base
+import api.models.section  # noqa: F401  - register model with Base.metadata
 import api.models.video  # noqa: F401  - register model with Base.metadata
 
 
@@ -61,3 +62,10 @@ async def init_db() -> None:
     await _create_database_if_missing()
     async with engine.begin() as conn:
         await conn.run_sync(Base.metadata.create_all)
+        # create_all не добавляет колонки в уже существующие таблицы
+        await conn.execute(
+            text(
+                "ALTER TABLE videos ADD COLUMN IF NOT EXISTS section_id INTEGER "
+                "REFERENCES sections(id) ON DELETE SET NULL"
+            )
+        )

api/db/section.py

@@ -0,0 +1,57 @@
+from sqlalchemy import func, select, update
+
+from api.db.connection import SessionLocal
+from api.models.section import Section
+from api.models.video import Video
+
+
+async def list_sections() -> list[dict]:
+    """Плоский список разделов + количество видео в каждом (дерево строит клиент)."""
+    async with SessionLocal() as session:
+        res = await session.execute(select(Section).order_by(Section.name))
+        sections = list(res.scalars().all())
+        counts_res = await session.execute(
+            select(Video.section_id, func.count()).group_by(Video.section_id)
+        )
+        counts = dict(counts_res.all())
+        return [
+            {
+                "id": s.id,
+                "name": s.name,
+                "parent_id": s.parent_id,
+                "video_count": counts.get(s.id, 0),
+            }
+            for s in sections
+        ]
+
+
+async def create_section(name: str, parent_id: int | None) -> Section:
+    async with SessionLocal() as session:
+        if parent_id is not None:
+            parent = await session.get(Section, parent_id)
+            if parent is None:
+                raise ValueError(f"Parent section {parent_id} not found")
+        s = Section(name=name, parent_id=parent_id)
+        session.add(s)
+        await session.commit()
+        await session.refresh(s)
+        return s
+
+
+async def delete_section(section_id: int) -> bool:
+    """Удаляет раздел: дочерние разделы поднимаются к родителю, видео — без раздела."""
+    async with SessionLocal() as session:
+        s = await session.get(Section, section_id)
+        if s is None:
+            return False
+        await session.execute(
+            update(Section)
+            .where(Section.parent_id == section_id)
+            .values(parent_id=s.parent_id)
+        )
+        await session.execute(
+            update(Video).where(Video.section_id == section_id).values(section_id=None)
+        )
+        await session.delete(s)
+        await session.commit()
+        return True

api/db/video.py

@@ -13,11 +13,13 @@ async def create_video(
     text_full_path: str,
     text_summary_path: str,
     transcription_method: str,
+    section_id: int | None = None,
 ) -> Video:
     async with SessionLocal() as session:
         v = Video(
             uuid=uuid,
             source_url=source_url,
+            section_id=section_id,
             title=title or "",
             video_path=video_path or "",
             text_full_path=text_full_path or "",
@@ -36,12 +38,39 @@ async def get_video(video_uuid: str) -> Video | None:
         return res.scalars().first()
 
 
-async def list_videos() -> list[Video]:
+async def list_videos(
+    *,
+    q: str | None = None,
+    section_ids: list[int] | None = None,
+    method: str | None = None,
+) -> list[Video]:
+    """Каталог с фильтрами: поиск по названию/URL, разделы (уже с потомками), метод."""
+    stmt = select(Video)
+    if q:
+        pattern = f"%{q}%"
+        stmt = stmt.where(Video.title.ilike(pattern) | Video.source_url.ilike(pattern))
+    if section_ids is not None:
+        stmt = stmt.where(Video.section_id.in_(section_ids))
+    if method:
+        stmt = stmt.where(Video.transcription_method.ilike(f"{method}%"))
+    stmt = stmt.order_by(Video.created_at.desc())
     async with SessionLocal() as session:
-        res = await session.execute(select(Video).order_by(Video.created_at.desc()))
+        res = await session.execute(stmt)
         return list(res.scalars().all())
 
 
+async def set_video_section(video_uuid: str, section_id: int | None) -> Video | None:
+    async with SessionLocal() as session:
+        res = await session.execute(select(Video).where(Video.uuid == video_uuid))
+        v = res.scalars().first()
+        if not v:
+            return None
+        v.section_id = section_id
+        await session.commit()
+        await session.refresh(v)
+        return v
+
+
 async def delete_video(video_uuid: str) -> bool:
     async with SessionLocal() as session:
         res = await session.execute(select(Video).where(Video.uuid == video_uuid))

api/lib/llm.py

@@ -68,7 +68,12 @@ def chat_complete(cfg: LLMConfig, messages: list[dict]) -> str:
         "max_tokens": cfg.max_tokens,
         "temperature": cfg.temperature,
     }
-    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        # urllib's default "Python-urllib/x.y" UA is blocked by some CDNs (Cloudflare 1010)
+        "User-Agent": "LLM-infa/0.3",
+    }
     if cfg.api_key:
         headers["Authorization"] = f"Bearer {cfg.api_key}"
     headers.update(cfg.extra_headers or {})
@@ -121,7 +126,7 @@ def list_models(cfg: LLMConfig) -> list[dict]:
     Compatible with OpenAI, OpenRouter, Ollama OpenAI-compat, vLLM, LM Studio.
     """
     url = cfg.base_url.rstrip("/") + "/models"
-    headers = {"Accept": "application/json"}
+    headers = {"Accept": "application/json", "User-Agent": "LLM-infa/0.3"}
     if cfg.api_key:
         headers["Authorization"] = f"Bearer {cfg.api_key}"
     headers.update(cfg.extra_headers or {})

api/lib/youtube.py

@@ -38,20 +38,30 @@ def download_video_with_subs(
 
     outtmpl = str(video_dir / f"{video_id}.%(ext)s")
 
-    ydl_opts = {
+    base_opts = {
         "outtmpl": outtmpl,
         "format": "bv*+ba/b",
         "merge_output_format": "mp4",
+        "quiet": True,
+        "no_warnings": True,
+    }
+    sub_opts = {
+        **base_opts,
         "writesubtitles": True,
         "writeautomaticsub": True,
         "subtitleslangs": langs,
         "subtitlesformat": "vtt",
-        "quiet": True,
-        "no_warnings": True,
     }
 
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        info = ydl.extract_info(url, download=True)
+    # YouTube often 429s the subtitle endpoints; a failed subtitle download must not
+    # kill the pipeline — retry without subs and let the Vosk fallback transcribe.
+    try:
+        with yt_dlp.YoutubeDL(sub_opts) as ydl:
+            info = ydl.extract_info(url, download=True)
+    except yt_dlp.utils.DownloadError as e:
+        logger.warning("Download with subtitles failed (%s); retrying without subs", e)
+        with yt_dlp.YoutubeDL(base_opts) as ydl:
+            info = ydl.extract_info(url, download=True)
 
     title = info.get("title") or video_id
 

api/main.py

@@ -4,7 +4,7 @@ from contextlib import asynccontextmanager
 from fastapi import FastAPI
 
 from api.db.connection import wait_for_postgres, init_db
-from api.route import default, llm, pipeline, videos
+from api.route import default, llm, pipeline, sections, videos
 
 
 logging.basicConfig(
@@ -25,4 +25,5 @@ app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan)
 app.include_router(default.router)
 app.include_router(pipeline.router)
 app.include_router(videos.router)
+app.include_router(sections.router)
 app.include_router(llm.router)

api/route/pipeline.py

@@ -42,6 +42,8 @@ class ProcessRequest(BaseModel):
     url: str
     model_path: str | None = None  # path inside models/, used when subtitles are unavailable
     summary_max_sentences: int = 15
+    keep_video: bool = True  # False: файл видео удаляется после расшифровки
+    section_id: int | None = None
     # If supplied, this LLM is used for the summary; otherwise env-configured LLM;
     # otherwise the built-in extractive fallback.
     llm: LLMOverride | None = None
@@ -170,7 +172,14 @@ async def process_video(req: ProcessRequest):
         raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
     text_summary_path.write_text(summary, encoding="utf-8")
 
-    rel_video = str(video_path.relative_to(_PROJECT_ROOT))
+    if req.keep_video:
+        rel_video = str(video_path.relative_to(_PROJECT_ROOT))
+    else:
+        # пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
+        video_path.unlink(missing_ok=True)
+        for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
+            p.unlink(missing_ok=True)
+        rel_video = ""
     rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
     rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
 
@@ -183,6 +192,7 @@ async def process_video(req: ProcessRequest):
             text_full_path=rel_full,
             text_summary_path=rel_summary,
             transcription_method=method,
+            section_id=req.section_id,
         )
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")

api/route/sections.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from api.db.section import create_section, delete_section, list_sections
+
+
+router = APIRouter(prefix="/sections", tags=["sections"])
+
+
+class SectionIn(BaseModel):
+    name: str
+    parent_id: int | None = None
+
+
+class SectionOut(BaseModel):
+    id: int
+    name: str
+    parent_id: int | None
+    video_count: int
+
+
+@router.get("/", response_model=list[SectionOut])
+async def list_all():
+    return await list_sections()
+
+
+@router.post("/", response_model=SectionOut)
+async def create(body: SectionIn):
+    name = body.name.strip()
+    if not name:
+        raise HTTPException(status_code=400, detail="Section name is empty")
+    try:
+        s = await create_section(name, body.parent_id)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return SectionOut(id=s.id, name=s.name, parent_id=s.parent_id, video_count=0)
+
+
+@router.delete("/{section_id}")
+async def delete(section_id: int):
+    ok = await delete_section(section_id)
+    if not ok:
+        raise HTTPException(status_code=404, detail="Section not found")
+    return {"ok": True}

api/route/videos.py

@@ -1,18 +1,28 @@
 from datetime import datetime
-from typing import List
+from pathlib import Path
+from typing import List, Literal
 
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
-from api.db.video import delete_video, get_video, list_videos
+from api.db.section import list_sections
+from api.db.video import (
+    delete_video,
+    get_video,
+    list_videos,
+    set_video_section,
+)
 
 
 router = APIRouter(prefix="/videos", tags=["videos"])
 
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+
 
 class VideoOut(BaseModel):
     uuid: str
     source_url: str
+    section_id: int | None
     title: str
     video_path: str
     text_full_path: str
@@ -21,10 +31,15 @@ class VideoOut(BaseModel):
     created_at: datetime
 
 
+class SectionPatch(BaseModel):
+    section_id: int | None
+
+
 def _to_out(v) -> VideoOut:
     return VideoOut(
         uuid=v.uuid,
         source_url=v.source_url,
+        section_id=v.section_id,
         title=v.title,
         video_path=v.video_path,
         text_full_path=v.text_full_path,
@@ -34,9 +49,31 @@ def _to_out(v) -> VideoOut:
     )
 
 
+async def _descendant_ids(section_id: int) -> list[int]:
+    """Раздел + все его потомки (фильтр по разделу захватывает подразделы)."""
+    sections = await list_sections()
+    children: dict[int | None, list[int]] = {}
+    for s in sections:
+        children.setdefault(s["parent_id"], []).append(s["id"])
+    ids, stack = [], [section_id]
+    while stack:
+        cur = stack.pop()
+        ids.append(cur)
+        stack.extend(children.get(cur, []))
+    return ids
+
+
 @router.get("/", response_model=List[VideoOut])
-async def list_all():
-    return [_to_out(v) for v in await list_videos()]
+async def list_all(
+    q: str | None = Query(default=None, description="Поиск по названию/URL"),
+    section_id: int | None = Query(default=None, description="Раздел (включая подразделы)"),
+    method: str | None = Query(default=None, description="Префикс метода: subtitles | vosk"),
+):
+    section_ids = await _descendant_ids(section_id) if section_id is not None else None
+    return [
+        _to_out(v)
+        for v in await list_videos(q=q, section_ids=section_ids, method=method)
+    ]
 
 
 @router.get("/{video_uuid}", response_model=VideoOut)
@@ -47,9 +84,49 @@ async def get_one(video_uuid: str):
     return _to_out(v)
 
 
-@router.delete("/{video_uuid}")
-async def delete_one(video_uuid: str):
-    ok = await delete_video(video_uuid)
-    if not ok:
+@router.get("/{video_uuid}/text")
+async def get_text(video_uuid: str, kind: Literal["full", "summary"] = "full"):
+    v = await get_video(video_uuid)
+    if not v:
         raise HTTPException(status_code=404, detail="Video not found")
+    rel = v.text_full_path if kind == "full" else v.text_summary_path
+    if not rel:
+        raise HTTPException(status_code=404, detail="Text file is not recorded")
+    p = (_PROJECT_ROOT / rel).resolve()
+    try:
+        p.relative_to(_PROJECT_ROOT)
+    except ValueError:
+        raise HTTPException(status_code=400, detail="Bad text path")
+    if not p.exists():
+        raise HTTPException(status_code=404, detail=f"File missing on disk: {rel}")
+    return {"uuid": video_uuid, "kind": kind, "text": p.read_text(encoding="utf-8", errors="ignore")}
+
+
+@router.patch("/{video_uuid}", response_model=VideoOut)
+async def patch_section(video_uuid: str, body: SectionPatch):
+    v = await set_video_section(video_uuid, body.section_id)
+    if not v:
+        raise HTTPException(status_code=404, detail="Video not found")
+    return _to_out(v)
+
+
+@router.delete("/{video_uuid}")
+async def delete_one(video_uuid: str, delete_files: bool = True):
+    v = await get_video(video_uuid)
+    if not v:
+        raise HTTPException(status_code=404, detail="Video not found")
+    if delete_files:
+        for rel in (v.video_path, v.text_full_path, v.text_summary_path):
+            if not rel:
+                continue
+            p = (_PROJECT_ROOT / rel).resolve()
+            try:
+                p.relative_to(_PROJECT_ROOT)
+            except ValueError:
+                continue
+            p.unlink(missing_ok=True)
+        # сопутствующие субтитры .vtt рядом с видео
+        for p in (_PROJECT_ROOT / "data" / "videos").glob(f"{video_uuid}*.vtt"):
+            p.unlink(missing_ok=True)
+    await delete_video(video_uuid)
     return {"ok": True}