jze9/LLM-infa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Фоновые задачи: плейлисты, длинные лекции, вкладка настроек

Browse Source 
- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
  плейлиста обрабатываются последовательно, есть отмена и живой прогресс
  (скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
  хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
  каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
  готовности элементов

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 15:01:11 +05:00
parent bf854f2d6e
commit e87f6cd215
18 changed files with 1059 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/db/connection.py
View File

@@ -8,7 +8,9 @@ from sqlalchemy.orm import sessionmaker
from api.config import settings
from api.models.base import Base
import api.models.job # noqa: F401 - register model with Base.metadata
import api.models.section # noqa: F401 - register model with Base.metadata
import api.models.setting # noqa: F401 - register model with Base.metadata
import api.models.video # noqa: F401 - register model with Base.metadata
@@ -69,3 +71,6 @@ async def init_db() -> None:
"REFERENCES sections(id) ON DELETE SET NULL"
)
)
await conn.execute(
text("ALTER TABLE videos ADD COLUMN IF NOT EXISTS job_id VARCHAR")
)

53
api/db/job.py Normal file
View File

@@ -0,0 +1,53 @@
from sqlalchemy import select, update
from api.db.connection import SessionLocal
from api.models.job import Job
async def create_job(job_id: str, url: str) -> Job:
async with SessionLocal() as session:
j = Job(id=job_id, url=url, status="queued")
session.add(j)
await session.commit()
await session.refresh(j)
return j
async def get_job(job_id: str) -> Job | None:
async with SessionLocal() as session:
return await session.get(Job, job_id)
async def list_jobs(limit: int = 50) -> list[Job]:
async with SessionLocal() as session:
res = await session.execute(
select(Job).order_by(Job.created_at.desc()).limit(limit)
)
return list(res.scalars().all())
async def update_job(job_id: str, **fields) -> None:
async with SessionLocal() as session:
await session.execute(update(Job).where(Job.id == job_id).values(**fields))
await session.commit()
async def delete_job(job_id: str) -> bool:
async with SessionLocal() as session:
j = await session.get(Job, job_id)
if j is None:
return False
await session.delete(j)
await session.commit()
return True
async def mark_interrupted() -> None:
"""На старте приложения: задачи, шедшие до перезапуска, отмечаем прерванными."""
async with SessionLocal() as session:
await session.execute(
update(Job)
.where(Job.status.in_(("queued", "running")))
.values(status="interrupted", current_item="")
)
await session.commit()

21
api/db/setting.py Normal file
View File

@@ -0,0 +1,21 @@
from sqlalchemy import select
from api.db.connection import SessionLocal
from api.models.setting import Setting
async def get_all_settings() -> dict[str, str]:
async with SessionLocal() as session:
res = await session.execute(select(Setting))
return {s.key: s.value for s in res.scalars().all()}
async def set_settings(values: dict[str, str]) -> None:
async with SessionLocal() as session:
for key, value in values.items():
existing = await session.get(Setting, key)
if existing is None:
session.add(Setting(key=key, value=str(value)))
else:
existing.value = str(value)
await session.commit()

2
api/db/video.py
View File

@@ -14,12 +14,14 @@ async def create_video(
text_summary_path: str,
transcription_method: str,
section_id: int | None = None,
job_id: str | None = None,
) -> Video:
async with SessionLocal() as session:
v = Video(
uuid=uuid,
source_url=source_url,
section_id=section_id,
job_id=job_id,
title=title or "",
video_path=video_path or "",
text_full_path=text_full_path or "",

119
api/lib/jobqueue.py Normal file
View File

@@ -0,0 +1,119 @@
"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются
последовательно (чтобы не долбить YouTube и LLM параллельными запросами).
Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs;
устойчивое состояние (счётчики, статус) — в таблице jobs.
"""
from __future__ import annotations
import asyncio
import logging
from api.db.job import get_job, update_job
from api.lib.processing import process_single_video
from api.lib.youtube import expand_url
logger = logging.getLogger("jobs")
_queue: asyncio.Queue[str] = asyncio.Queue()
_live: dict[str, str] = {}
_cancel: set[str] = set()
_params: dict[str, dict] = {}
def submit(job_id: str, params: dict) -> None:
_params[job_id] = params
_queue.put_nowait(job_id)
def live_status(job_id: str) -> str:
return _live.get(job_id, "")
def request_cancel(job_id: str) -> None:
_cancel.add(job_id)
def is_cancelled(job_id: str) -> bool:
return job_id in _cancel
async def worker() -> None:
"""Единственный воркер; запускается на старте приложения."""
logger.info("job worker started")
while True:
job_id = await _queue.get()
try:
await _run_job(job_id)
except Exception:
logger.exception("job %s crashed", job_id)
try:
await update_job(job_id, status="error", current_item="")
except Exception:
pass
finally:
_live.pop(job_id, None)
_cancel.discard(job_id)
_params.pop(job_id, None)
_queue.task_done()
async def _run_job(job_id: str) -> None:
job = await get_job(job_id)
if job is None or job.status == "cancelled":
return
params = _params.get(job_id, {})
loop = asyncio.get_running_loop()
await update_job(job_id, status="running", current_item="анализ ссылки…")
_live[job_id] = "анализ ссылки…"
plan = await loop.run_in_executor(None, expand_url, job.url)
entries = plan["entries"]
if not entries:
await update_job(job_id, status="error", error="Плейлист пуст", current_item="")
return
await update_job(
job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries)
)
done = failed = 0
errors: list[str] = []
n = len(entries)
for i, entry in enumerate(entries, start=1):
if is_cancelled(job_id):
await update_job(job_id, status="cancelled", current_item="")
logger.info("job %s cancelled at item %d/%d", job_id, i, n)
return
label = f"{i}/{n} · {entry['title']}"
await update_job(job_id, current_item=label)
_live[job_id] = label
def _notify(msg: str, _label: str = label) -> None:
_live[job_id] = f"{_label} — {msg}"
try:
await process_single_video(
entry["url"],
section_id=params.get("section_id"),
model_path=params.get("model_path"),
summary_max_sentences=params.get("summary_max_sentences"),
keep_video=params.get("keep_video"),
job_id=job_id,
progress=_notify,
)
done += 1
except Exception as e:
failed += 1
errors.append(f"{entry['title']}: {e}")
logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e)
await update_job(
job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000]
)
status = "error" if done == 0 and failed > 0 else "done"
await update_job(job_id, status=status, current_item="")
logger.info("job %s finished: %d ok, %d failed", job_id, done, failed)

229
api/lib/processing.py Normal file
View File

@@ -0,0 +1,229 @@
"""Ядро пайплайна: одно видео от URL до записи в БД.
Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs.
Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из
настроек в БД (таблица settings) с фолбэком на переменные окружения.
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from typing import Callable, Optional
from api.config import settings
from api.db.setting import get_all_settings
from api.db.video import create_video
from api.lib.llm import LLMConfig
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
logger = logging.getLogger("processing")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class PipelineError(RuntimeError):
def __init__(self, message: str, status: int = 500):
super().__init__(message)
self.status = status
async def effective_defaults() -> dict:
"""Настройки из БД поверх переменных окружения."""
db = await get_all_settings()
base_url = db.get("llm_base_url") or settings.LLM_BASE_URL
model = db.get("llm_model") or settings.LLM_MODEL
llm_cfg = None
if base_url and model:
llm_cfg = LLMConfig(
base_url=base_url,
api_key=db.get("llm_api_key") or settings.LLM_API_KEY,
model=model,
timeout=settings.LLM_TIMEOUT,
max_tokens=settings.LLM_MAX_TOKENS,
temperature=settings.LLM_TEMPERATURE,
)
try:
max_sentences = int(db.get("summary_max_sentences") or 15)
except ValueError:
max_sentences = 15
return {
"llm_cfg": llm_cfg,
"summary_max_sentences": max_sentences,
"keep_video": db.get("keep_video", "1") != "0",
"model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL,
}
def resolve_model_path(raw: str) -> Path:
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise PipelineError(
"model_path must point inside the project's models directory", status=400
)
if not p.exists():
raise PipelineError(f"Model not found: {p}", status=400)
return p
async def process_single_video(
url: str,
*,
model_path: str | None = None,
summary_max_sentences: int | None = None,
keep_video: bool | None = None,
section_id: int | None = None,
job_id: str | None = None,
llm_cfg: Optional[LLMConfig] = None,
system_prompt: str | None = None,
user_template: str | None = None,
progress: Optional[Callable[[str], None]] = None,
) -> dict:
"""Скачивает, расшифровывает, сокращает и сохраняет одно видео.
Параметры со значением None подтягиваются из настроек (БД/env).
progress(msg) — живой статус для UI задач.
"""
notify = progress or (lambda msg: None)
defaults = await effective_defaults()
if llm_cfg is None:
llm_cfg = defaults["llm_cfg"]
if summary_max_sentences is None:
summary_max_sentences = defaults["summary_max_sentences"]
if keep_video is None:
keep_video = defaults["keep_video"]
raw_model_path = model_path or defaults["model_path"]
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video)
notify("скачивание…")
try:
info = await loop.run_in_executor(
None,
lambda: download_video_with_subs(
url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
# видео не сохраняем — достаточно аудио (для лекций в разы быстрее)
audio_only=not keep_video,
on_progress=lambda pct: notify(f"скачивание {pct}%"),
),
)
except Exception as e:
raise PipelineError(f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise PipelineError("Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
notify("распознавание речи (Vosk)…")
resolved_model = resolve_model_path(raw_model_path)
try:
full_text = await loop.run_in_executor(
None,
lambda: transcribe_via_ffmpeg(
video_path,
resolved_model,
on_progress=lambda m: notify(f"распознано {m} мин аудио"),
),
)
method = f"vosk:{resolved_model.name}"
except Exception as e:
raise PipelineError(f"Transcription failed: {e}")
if not full_text.strip():
raise PipelineError("No subtitles and transcription returned empty text")
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
notify("выжимка (LLM)…" if llm_cfg else "выжимка…")
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=summary_max_sentences,
),
)
except Exception as e:
raise PipelineError(f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
section_id=section_id,
job_id=job_id,
)
except Exception as e:
raise PipelineError(f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return {
"uuid": video_uuid,
"title": title,
"source_url": url,
"video_path": rel_video,
"text_full_path": rel_full,
"text_summary_path": rel_summary,
"transcription_method": method,
"summary_preview": summary[:300],
}

14
api/lib/summarize.py
View File

@@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = (
DEFAULT_USER_TEMPLATE = (
"Сделай связную краткую выжимку следующего текста. "
"Выдели основные тезисы и логику изложения. Объём — 5–12 предложений.\n\n"
"Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n"
"---\n{text}\n---"
)
@@ -154,6 +154,7 @@ def summarize_llm(
user_template: Optional[str] = None,
reduce_template: Optional[str] = None,
chunk_chars: int = 10000,
max_sentences: int = 15,
) -> str:
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
text = (text or "").strip()
@@ -164,12 +165,19 @@ def summarize_llm(
user_t = user_template or DEFAULT_USER_TEMPLATE
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
def _fmt(template: str, chunk: str) -> str:
try:
return template.format(text=chunk, n=max_sentences)
except (KeyError, IndexError):
# пользовательский шаблон без {n} / с лишними скобками
return template.replace("{text}", chunk)
def _one(chunk: str, template: str) -> str:
return chat_complete(
cfg,
[
{"role": "system", "content": sys_p},
{"role": "user", "content": template.format(text=chunk)},
{"role": "user", "content": _fmt(template, chunk)},
],
)
@@ -194,6 +202,7 @@ def summarize_llm(
user_template=reduce_t,
reduce_template=reduce_t,
chunk_chars=chunk_chars,
max_sentences=max_sentences,
)
logger.info("LLM reduce step over %d partials", len(partials))
return _one(combined, reduce_t)
@@ -215,6 +224,7 @@ def summarize(
llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=max_sentences,
)
except LLMError as e:
logger.warning("LLM summarization failed, falling back to extractive: %s", e)

26
api/lib/transcribe.py
View File

@@ -53,8 +53,16 @@ class VoskModelManager:
t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
def transcribe_via_ffmpeg(
media_path: Path,
model_path: Path,
sample_rate: int = 16000,
on_progress=None,
) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.
on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио.
"""
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None:
@@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int =
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024)
if mb >= last_logged_mb + 5:
last_logged_mb = mb
logger.info("Transcribed %d MB of audio so far", mb)
# PCM s16le mono: sample_rate * 2 байта в секунду
minutes = bytes_read // (sample_rate * 2 * 60)
if minutes >= last_logged_mb + 5:
last_logged_mb = minutes
logger.info("Transcribed %d minutes of audio so far", minutes)
if on_progress is not None:
try:
on_progress(minutes)
except Exception:
pass
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")

72
api/lib/youtube.py
View File

@@ -12,7 +12,44 @@ import yt_dlp
logger = logging.getLogger("youtube")
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"}
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"}
def expand_url(url: str) -> dict:
"""Определяет, видео это или плейлист, и возвращает список элементов.
Returns: {"kind": "video"|"playlist", "title": str,
"entries": [{"url": str, "title": str}, ...]}
"""
opts = {
"extract_flat": "in_playlist",
"quiet": True,
"no_warnings": True,
"skip_download": True,
}
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info.get("_type") == "playlist":
entries = []
for e in info.get("entries") or []:
if not e:
continue
u = e.get("url") or ""
if u and not u.startswith("http"):
u = f"https://www.youtube.com/watch?v={u}"
if not u and e.get("id"):
u = f"https://www.youtube.com/watch?v={e['id']}"
if u:
entries.append({"url": u, "title": e.get("title") or u})
return {
"kind": "playlist",
"title": info.get("title") or url,
"entries": entries,
}
title = info.get("title") or url
return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]}
def download_video_with_subs(
@@ -20,9 +57,15 @@ def download_video_with_subs(
video_id: str,
video_dir: Path,
langs: Optional[list[str]] = None,
audio_only: bool = False,
on_progress=None,
) -> dict:
"""Download a single video, also requesting subtitles for the given languages.
audio_only: качает только аудиодорожку (для длинных лекций, когда видео
не сохраняется — в разы быстрее и меньше по объёму).
on_progress: callback(percent: int), вызывается с шагом ~5%.
Returns:
{
"title": str,
@@ -40,11 +83,34 @@ def download_video_with_subs(
base_opts = {
"outtmpl": outtmpl,
"format": "bv*+ba/b",
"merge_output_format": "mp4",
"quiet": True,
"no_warnings": True,
"noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист
}
if audio_only:
base_opts["format"] = "ba/b"
else:
base_opts["format"] = "bv*+ba/b"
base_opts["merge_output_format"] = "mp4"
if on_progress is not None:
state = {"last": -5}
def _hook(dd: dict):
if dd.get("status") != "downloading":
return
total = dd.get("total_bytes") or dd.get("total_bytes_estimate")
if not total:
return
pct = int(dd.get("downloaded_bytes", 0) * 100 / total)
if pct >= state["last"] + 5:
state["last"] = pct
try:
on_progress(pct)
except Exception:
pass
base_opts["progress_hooks"] = [_hook]
sub_opts = {
**base_opts,
"writesubtitles": True,

12
api/main.py
View File

@@ -1,10 +1,13 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from api.db.connection import wait_for_postgres, init_db
from api.route import default, llm, pipeline, sections, videos
from api.db.job import mark_interrupted
from api.lib import jobqueue
from api.route import app_settings, default, jobs, llm, pipeline, sections, videos
logging.basicConfig(
@@ -17,13 +20,18 @@ logging.basicConfig(
async def lifespan(app: FastAPI):
await wait_for_postgres()
await init_db()
await mark_interrupted()
worker_task = asyncio.create_task(jobqueue.worker())
yield
worker_task.cancel()
app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan)
app = FastAPI(title="LLM-infa API", version="0.4.0", lifespan=lifespan)
app.include_router(default.router)
app.include_router(pipeline.router)
app.include_router(jobs.router)
app.include_router(videos.router)
app.include_router(sections.router)
app.include_router(app_settings.router)
app.include_router(llm.router)

59
api/route/app_settings.py Normal file
View File

@@ -0,0 +1,59 @@
"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres,
переживают перезапуск контейнеров и перезагрузку страницы.
"""
from __future__ import annotations
from fastapi import APIRouter
from pydantic import BaseModel
from api.db.setting import get_all_settings, set_settings
router = APIRouter(prefix="/settings", tags=["settings"])
_KEYS = (
"llm_base_url",
"llm_api_key",
"llm_model",
"summary_max_sentences",
"keep_video",
"vosk_model_path",
)
class SettingsPayload(BaseModel):
llm_base_url: str | None = None
llm_api_key: str | None = None
llm_model: str | None = None
summary_max_sentences: int | None = None
keep_video: bool | None = None
vosk_model_path: str | None = None
@router.get("/")
async def get_settings():
db = await get_all_settings()
return {
"llm_base_url": db.get("llm_base_url", ""),
"llm_api_key": db.get("llm_api_key", ""),
"llm_model": db.get("llm_model", ""),
"summary_max_sentences": int(db.get("summary_max_sentences") or 15),
"keep_video": db.get("keep_video", "1") != "0",
"vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"),
}
@router.put("/")
async def put_settings(body: SettingsPayload):
values: dict[str, str] = {}
for key in _KEYS:
v = getattr(body, key)
if v is None:
continue
if key == "keep_video":
values[key] = "1" if v else "0"
else:
values[key] = str(v)
if values:
await set_settings(values)
return await get_settings()

112
api/route/jobs.py Normal file
View File

@@ -0,0 +1,112 @@
"""Фоновые задачи: видео или плейлист любой длины.
POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs.
"""
from __future__ import annotations
import uuid as _uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.db.job import create_job, delete_job, get_job, list_jobs
from api.lib import jobqueue
router = APIRouter(prefix="/jobs", tags=["jobs"])
class JobIn(BaseModel):
url: str
section_id: int | None = None
# None → берутся из сохранённых настроек
keep_video: bool | None = None
summary_max_sentences: int | None = None
model_path: str | None = None
class JobOut(BaseModel):
id: str
url: str
kind: str
title: str
status: str
total_items: int
done_items: int
failed_items: int
current_item: str
live: str
error: str
created_at: datetime
def _to_out(j) -> JobOut:
return JobOut(
id=j.id,
url=j.url,
kind=j.kind,
title=j.title,
status=j.status,
total_items=j.total_items,
done_items=j.done_items,
failed_items=j.failed_items,
current_item=j.current_item,
live=jobqueue.live_status(j.id) or j.current_item,
error=j.error,
created_at=j.created_at,
)
@router.post("/", response_model=JobOut)
async def create(body: JobIn):
url = body.url.strip()
if not url:
raise HTTPException(status_code=400, detail="URL is empty")
job_id = _uuid.uuid4().hex
j = await create_job(job_id, url)
jobqueue.submit(
job_id,
{
"section_id": body.section_id,
"keep_video": body.keep_video,
"summary_max_sentences": body.summary_max_sentences,
"model_path": body.model_path,
},
)
return _to_out(j)
@router.get("/", response_model=list[JobOut])
async def list_all(limit: int = 50):
return [_to_out(j) for j in await list_jobs(limit)]
@router.get("/{job_id}", response_model=JobOut)
async def get_one(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
return _to_out(j)
@router.post("/{job_id}/cancel")
async def cancel(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status not in ("queued", "running"):
raise HTTPException(status_code=400, detail=f"Job is {j.status}")
jobqueue.request_cancel(job_id)
return {"ok": True, "note": "остановится после текущего элемента"}
@router.delete("/{job_id}")
async def delete(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status in ("queued", "running"):
raise HTTPException(status_code=400, detail="Сначала отмените задачу")
await delete_job(job_id)
return {"ok": True}

177
api/route/pipeline.py
View File

@@ -1,29 +1,16 @@
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist."""
"""Синхронный запуск пайплайна для одного видео (для длинных видео и
плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым).
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.config import settings
from api.db.video import create_video
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
from api.lib.llm import LLMConfig
from api.lib.processing import PipelineError, process_single_video
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
logger = logging.getLogger("pipeline")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class LLMOverride(BaseModel):
@@ -40,12 +27,12 @@ class LLMOverride(BaseModel):
class ProcessRequest(BaseModel):
url: str
model_path: str | None = None # path inside models/, used when subtitles are unavailable
summary_max_sentences: int = 15
keep_video: bool = True # False: файл видео удаляется после расшифровки
model_path: str | None = None # None → из настроек
summary_max_sentences: int | None = None
keep_video: bool | None = None
section_id: int | None = None
# If supplied, this LLM is used for the summary; otherwise env-configured LLM;
# otherwise the built-in extractive fallback.
# Если задано — используется этот LLM; иначе сохранённые настройки/env;
# иначе встроенная экстрактивная выжимка.
llm: LLMOverride | None = None
@@ -60,86 +47,10 @@ class ProcessResponse(BaseModel):
summary_preview: str
def _resolve_model_path(req_model_path: str | None) -> Path:
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise HTTPException(
status_code=400,
detail="model_path must point inside the project's models directory",
)
if not p.exists():
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
return p
@router.post("/process", response_model=ProcessResponse)
async def process_video(req: ProcessRequest):
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s", video_uuid, req.url)
try:
info = await loop.run_in_executor(
None,
download_video_with_subs,
req.url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise HTTPException(status_code=500, detail="Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
model_path = _resolve_model_path(req.model_path)
try:
full_text = await loop.run_in_executor(
None, transcribe_via_ffmpeg, video_path, model_path
)
method = f"vosk:{model_path.name}"
except Exception as e:
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
if not full_text.strip():
raise HTTPException(
status_code=500, detail="No subtitles and transcription returned empty text"
)
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
llm_cfg = None
sys_p = user_t = None
if req.llm is not None:
llm_cfg = LLMConfig(
base_url=req.llm.base_url,
@@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest):
)
sys_p = req.llm.system_prompt
user_t = req.llm.user_template
else:
llm_cfg = llm_from_settings()
sys_p = None
user_t = None
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
max_sentences=req.summary_max_sentences,
),
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if req.keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=req.url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
result = await process_single_video(
req.url,
model_path=req.model_path,
summary_max_sentences=req.summary_max_sentences,
keep_video=req.keep_video,
section_id=req.section_id,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return ProcessResponse(
uuid=video_uuid,
title=title,
source_url=req.url,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
summary_preview=summary[:300],
)
except PipelineError as e:
raise HTTPException(status_code=e.status, detail=str(e))
return ProcessResponse(**result)