jze9/LLM-infa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Фоновые задачи: плейлисты, длинные лекции, вкладка настроек

Browse Source 
- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
  плейлиста обрабатываются последовательно, есть отмена и живой прогресс
  (скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
  хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
  каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
  готовности элементов

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 15:01:11 +05:00
parent bf854f2d6e
commit e87f6cd215
18 changed files with 1059 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
api/lib/jobqueue.py Normal file
View File

@@ -0,0 +1,119 @@
"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются
последовательно (чтобы не долбить YouTube и LLM параллельными запросами).
Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs;
устойчивое состояние (счётчики, статус) — в таблице jobs.
"""
from __future__ import annotations
import asyncio
import logging
from api.db.job import get_job, update_job
from api.lib.processing import process_single_video
from api.lib.youtube import expand_url
logger = logging.getLogger("jobs")
_queue: asyncio.Queue[str] = asyncio.Queue()
_live: dict[str, str] = {}
_cancel: set[str] = set()
_params: dict[str, dict] = {}
def submit(job_id: str, params: dict) -> None:
_params[job_id] = params
_queue.put_nowait(job_id)
def live_status(job_id: str) -> str:
return _live.get(job_id, "")
def request_cancel(job_id: str) -> None:
_cancel.add(job_id)
def is_cancelled(job_id: str) -> bool:
return job_id in _cancel
async def worker() -> None:
"""Единственный воркер; запускается на старте приложения."""
logger.info("job worker started")
while True:
job_id = await _queue.get()
try:
await _run_job(job_id)
except Exception:
logger.exception("job %s crashed", job_id)
try:
await update_job(job_id, status="error", current_item="")
except Exception:
pass
finally:
_live.pop(job_id, None)
_cancel.discard(job_id)
_params.pop(job_id, None)
_queue.task_done()
async def _run_job(job_id: str) -> None:
job = await get_job(job_id)
if job is None or job.status == "cancelled":
return
params = _params.get(job_id, {})
loop = asyncio.get_running_loop()
await update_job(job_id, status="running", current_item="анализ ссылки…")
_live[job_id] = "анализ ссылки…"
plan = await loop.run_in_executor(None, expand_url, job.url)
entries = plan["entries"]
if not entries:
await update_job(job_id, status="error", error="Плейлист пуст", current_item="")
return
await update_job(
job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries)
)
done = failed = 0
errors: list[str] = []
n = len(entries)
for i, entry in enumerate(entries, start=1):
if is_cancelled(job_id):
await update_job(job_id, status="cancelled", current_item="")
logger.info("job %s cancelled at item %d/%d", job_id, i, n)
return
label = f"{i}/{n} · {entry['title']}"
await update_job(job_id, current_item=label)
_live[job_id] = label
def _notify(msg: str, _label: str = label) -> None:
_live[job_id] = f"{_label} — {msg}"
try:
await process_single_video(
entry["url"],
section_id=params.get("section_id"),
model_path=params.get("model_path"),
summary_max_sentences=params.get("summary_max_sentences"),
keep_video=params.get("keep_video"),
job_id=job_id,
progress=_notify,
)
done += 1
except Exception as e:
failed += 1
errors.append(f"{entry['title']}: {e}")
logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e)
await update_job(
job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000]
)
status = "error" if done == 0 and failed > 0 else "done"
await update_job(job_id, status=status, current_item="")
logger.info("job %s finished: %d ok, %d failed", job_id, done, failed)

229
api/lib/processing.py Normal file
View File

@@ -0,0 +1,229 @@
"""Ядро пайплайна: одно видео от URL до записи в БД.
Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs.
Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из
настроек в БД (таблица settings) с фолбэком на переменные окружения.
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from typing import Callable, Optional
from api.config import settings
from api.db.setting import get_all_settings
from api.db.video import create_video
from api.lib.llm import LLMConfig
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
logger = logging.getLogger("processing")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class PipelineError(RuntimeError):
def __init__(self, message: str, status: int = 500):
super().__init__(message)
self.status = status
async def effective_defaults() -> dict:
"""Настройки из БД поверх переменных окружения."""
db = await get_all_settings()
base_url = db.get("llm_base_url") or settings.LLM_BASE_URL
model = db.get("llm_model") or settings.LLM_MODEL
llm_cfg = None
if base_url and model:
llm_cfg = LLMConfig(
base_url=base_url,
api_key=db.get("llm_api_key") or settings.LLM_API_KEY,
model=model,
timeout=settings.LLM_TIMEOUT,
max_tokens=settings.LLM_MAX_TOKENS,
temperature=settings.LLM_TEMPERATURE,
)
try:
max_sentences = int(db.get("summary_max_sentences") or 15)
except ValueError:
max_sentences = 15
return {
"llm_cfg": llm_cfg,
"summary_max_sentences": max_sentences,
"keep_video": db.get("keep_video", "1") != "0",
"model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL,
}
def resolve_model_path(raw: str) -> Path:
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise PipelineError(
"model_path must point inside the project's models directory", status=400
)
if not p.exists():
raise PipelineError(f"Model not found: {p}", status=400)
return p
async def process_single_video(
url: str,
*,
model_path: str | None = None,
summary_max_sentences: int | None = None,
keep_video: bool | None = None,
section_id: int | None = None,
job_id: str | None = None,
llm_cfg: Optional[LLMConfig] = None,
system_prompt: str | None = None,
user_template: str | None = None,
progress: Optional[Callable[[str], None]] = None,
) -> dict:
"""Скачивает, расшифровывает, сокращает и сохраняет одно видео.
Параметры со значением None подтягиваются из настроек (БД/env).
progress(msg) — живой статус для UI задач.
"""
notify = progress or (lambda msg: None)
defaults = await effective_defaults()
if llm_cfg is None:
llm_cfg = defaults["llm_cfg"]
if summary_max_sentences is None:
summary_max_sentences = defaults["summary_max_sentences"]
if keep_video is None:
keep_video = defaults["keep_video"]
raw_model_path = model_path or defaults["model_path"]
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video)
notify("скачивание…")
try:
info = await loop.run_in_executor(
None,
lambda: download_video_with_subs(
url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
# видео не сохраняем — достаточно аудио (для лекций в разы быстрее)
audio_only=not keep_video,
on_progress=lambda pct: notify(f"скачивание {pct}%"),
),
)
except Exception as e:
raise PipelineError(f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise PipelineError("Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
notify("распознавание речи (Vosk)…")
resolved_model = resolve_model_path(raw_model_path)
try:
full_text = await loop.run_in_executor(
None,
lambda: transcribe_via_ffmpeg(
video_path,
resolved_model,
on_progress=lambda m: notify(f"распознано {m} мин аудио"),
),
)
method = f"vosk:{resolved_model.name}"
except Exception as e:
raise PipelineError(f"Transcription failed: {e}")
if not full_text.strip():
raise PipelineError("No subtitles and transcription returned empty text")
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
notify("выжимка (LLM)…" if llm_cfg else "выжимка…")
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=summary_max_sentences,
),
)
except Exception as e:
raise PipelineError(f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
section_id=section_id,
job_id=job_id,
)
except Exception as e:
raise PipelineError(f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return {
"uuid": video_uuid,
"title": title,
"source_url": url,
"video_path": rel_video,
"text_full_path": rel_full,
"text_summary_path": rel_summary,
"transcription_method": method,
"summary_preview": summary[:300],
}

14
api/lib/summarize.py
View File

@@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = (
DEFAULT_USER_TEMPLATE = (
"Сделай связную краткую выжимку следующего текста. "
"Выдели основные тезисы и логику изложения. Объём — 5–12 предложений.\n\n"
"Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n"
"---\n{text}\n---"
)
@@ -154,6 +154,7 @@ def summarize_llm(
user_template: Optional[str] = None,
reduce_template: Optional[str] = None,
chunk_chars: int = 10000,
max_sentences: int = 15,
) -> str:
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
text = (text or "").strip()
@@ -164,12 +165,19 @@ def summarize_llm(
user_t = user_template or DEFAULT_USER_TEMPLATE
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
def _fmt(template: str, chunk: str) -> str:
try:
return template.format(text=chunk, n=max_sentences)
except (KeyError, IndexError):
# пользовательский шаблон без {n} / с лишними скобками
return template.replace("{text}", chunk)
def _one(chunk: str, template: str) -> str:
return chat_complete(
cfg,
[
{"role": "system", "content": sys_p},
{"role": "user", "content": template.format(text=chunk)},
{"role": "user", "content": _fmt(template, chunk)},
],
)
@@ -194,6 +202,7 @@ def summarize_llm(
user_template=reduce_t,
reduce_template=reduce_t,
chunk_chars=chunk_chars,
max_sentences=max_sentences,
)
logger.info("LLM reduce step over %d partials", len(partials))
return _one(combined, reduce_t)
@@ -215,6 +224,7 @@ def summarize(
llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=max_sentences,
)
except LLMError as e:
logger.warning("LLM summarization failed, falling back to extractive: %s", e)

26
api/lib/transcribe.py
View File

@@ -53,8 +53,16 @@ class VoskModelManager:
t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
def transcribe_via_ffmpeg(
media_path: Path,
model_path: Path,
sample_rate: int = 16000,
on_progress=None,
) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.
on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио.
"""
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None:
@@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int =
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024)
if mb >= last_logged_mb + 5:
last_logged_mb = mb
logger.info("Transcribed %d MB of audio so far", mb)
# PCM s16le mono: sample_rate * 2 байта в секунду
minutes = bytes_read // (sample_rate * 2 * 60)
if minutes >= last_logged_mb + 5:
last_logged_mb = minutes
logger.info("Transcribed %d minutes of audio so far", minutes)
if on_progress is not None:
try:
on_progress(minutes)
except Exception:
pass
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")

72
api/lib/youtube.py
View File

@@ -12,7 +12,44 @@ import yt_dlp
logger = logging.getLogger("youtube")
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"}
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"}
def expand_url(url: str) -> dict:
"""Определяет, видео это или плейлист, и возвращает список элементов.
Returns: {"kind": "video"|"playlist", "title": str,
"entries": [{"url": str, "title": str}, ...]}
"""
opts = {
"extract_flat": "in_playlist",
"quiet": True,
"no_warnings": True,
"skip_download": True,
}
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info.get("_type") == "playlist":
entries = []
for e in info.get("entries") or []:
if not e:
continue
u = e.get("url") or ""
if u and not u.startswith("http"):
u = f"https://www.youtube.com/watch?v={u}"
if not u and e.get("id"):
u = f"https://www.youtube.com/watch?v={e['id']}"
if u:
entries.append({"url": u, "title": e.get("title") or u})
return {
"kind": "playlist",
"title": info.get("title") or url,
"entries": entries,
}
title = info.get("title") or url
return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]}
def download_video_with_subs(
@@ -20,9 +57,15 @@ def download_video_with_subs(
video_id: str,
video_dir: Path,
langs: Optional[list[str]] = None,
audio_only: bool = False,
on_progress=None,
) -> dict:
"""Download a single video, also requesting subtitles for the given languages.
audio_only: качает только аудиодорожку (для длинных лекций, когда видео
не сохраняется — в разы быстрее и меньше по объёму).
on_progress: callback(percent: int), вызывается с шагом ~5%.
Returns:
{
"title": str,
@@ -40,11 +83,34 @@ def download_video_with_subs(
base_opts = {
"outtmpl": outtmpl,
"format": "bv*+ba/b",
"merge_output_format": "mp4",
"quiet": True,
"no_warnings": True,
"noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист
}
if audio_only:
base_opts["format"] = "ba/b"
else:
base_opts["format"] = "bv*+ba/b"
base_opts["merge_output_format"] = "mp4"
if on_progress is not None:
state = {"last": -5}
def _hook(dd: dict):
if dd.get("status") != "downloading":
return
total = dd.get("total_bytes") or dd.get("total_bytes_estimate")
if not total:
return
pct = int(dd.get("downloaded_bytes", 0) * 100 / total)
if pct >= state["last"] + 5:
state["last"] = pct
try:
on_progress(pct)
except Exception:
pass
base_opts["progress_hooks"] = [_hook]
sub_opts = {
**base_opts,
"writesubtitles": True,