Фоновые задачи: плейлисты, длинные лекции, вкладка настроек

- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
  плейлиста обрабатываются последовательно, есть отмена и живой прогресс
  (скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
  хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
  каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
  готовности элементов

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 15:01:11 +05:00
parent bf854f2d6e
commit e87f6cd215
18 changed files with 1059 additions and 292 deletions

View File

@@ -48,19 +48,38 @@ OpenAI-совместимый эндпоинт. Два способа:
Если LLM не настроена, работает встроенная экстрактивная суммаризация Если LLM не настроена, работает встроенная экстрактивная суммаризация
(выбор ключевых предложений, без нейросети). (выбор ключевых предложений, без нейросети).
## Плейлисты и длинные лекции
Обработка идёт через фоновые задачи: `POST /jobs` сразу возвращает id, элементы
плейлиста обрабатываются по очереди, прогресс виден в UI (скачивание %,
минуты распознанного аудио, номер видео в плейлисте). Если в настройках
выключено «Сохранять видео», для лекций качается только аудиодорожка —
в разы быстрее и легче.
Настройки (ключ LLM, модель, длина выжимки, хранение видео) сохраняются
в Postgres — переживают перезапуск контейнеров и перезагрузку страницы.
Вкладка «Настройки» в UI.
## API ## API
```bash ```bash
# обработать видео целиком # фоновая задача: видео или плейлист
curl -X POST localhost:8000/jobs/ \
-H 'Content-Type: application/json' \
-d '{"url": "https://www.youtube.com/playlist?list=..."}'
curl localhost:8000/jobs/ # статусы и прогресс
# синхронно, одно видео (для коротких роликов)
curl -X POST localhost:8000/pipeline/process \ curl -X POST localhost:8000/pipeline/process \
-H 'Content-Type: application/json' \ -H 'Content-Type: application/json' \
-d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}' -d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}'
# каталог обработанных видео # каталог обработанных видео (фильтры: ?q=, ?section_id=, ?method=)
curl localhost:8000/videos/ curl localhost:8000/videos/
# проверить связь с LLM # настройки
curl -X POST localhost:8000/llm/test curl -X PUT localhost:8000/settings/ -H 'Content-Type: application/json' \
-d '{"llm_base_url": "https://openrouter.ai/api/v1", "llm_api_key": "sk-...", "llm_model": "..."}'
``` ```
## Заметки ## Заметки

View File

@@ -8,7 +8,9 @@ from sqlalchemy.orm import sessionmaker
from api.config import settings from api.config import settings
from api.models.base import Base from api.models.base import Base
import api.models.job # noqa: F401 - register model with Base.metadata
import api.models.section # noqa: F401 - register model with Base.metadata import api.models.section # noqa: F401 - register model with Base.metadata
import api.models.setting # noqa: F401 - register model with Base.metadata
import api.models.video # noqa: F401 - register model with Base.metadata import api.models.video # noqa: F401 - register model with Base.metadata
@@ -69,3 +71,6 @@ async def init_db() -> None:
"REFERENCES sections(id) ON DELETE SET NULL" "REFERENCES sections(id) ON DELETE SET NULL"
) )
) )
await conn.execute(
text("ALTER TABLE videos ADD COLUMN IF NOT EXISTS job_id VARCHAR")
)

53
api/db/job.py Normal file
View File

@@ -0,0 +1,53 @@
from sqlalchemy import select, update
from api.db.connection import SessionLocal
from api.models.job import Job
async def create_job(job_id: str, url: str) -> Job:
async with SessionLocal() as session:
j = Job(id=job_id, url=url, status="queued")
session.add(j)
await session.commit()
await session.refresh(j)
return j
async def get_job(job_id: str) -> Job | None:
async with SessionLocal() as session:
return await session.get(Job, job_id)
async def list_jobs(limit: int = 50) -> list[Job]:
async with SessionLocal() as session:
res = await session.execute(
select(Job).order_by(Job.created_at.desc()).limit(limit)
)
return list(res.scalars().all())
async def update_job(job_id: str, **fields) -> None:
async with SessionLocal() as session:
await session.execute(update(Job).where(Job.id == job_id).values(**fields))
await session.commit()
async def delete_job(job_id: str) -> bool:
async with SessionLocal() as session:
j = await session.get(Job, job_id)
if j is None:
return False
await session.delete(j)
await session.commit()
return True
async def mark_interrupted() -> None:
"""На старте приложения: задачи, шедшие до перезапуска, отмечаем прерванными."""
async with SessionLocal() as session:
await session.execute(
update(Job)
.where(Job.status.in_(("queued", "running")))
.values(status="interrupted", current_item="")
)
await session.commit()

21
api/db/setting.py Normal file
View File

@@ -0,0 +1,21 @@
from sqlalchemy import select
from api.db.connection import SessionLocal
from api.models.setting import Setting
async def get_all_settings() -> dict[str, str]:
async with SessionLocal() as session:
res = await session.execute(select(Setting))
return {s.key: s.value for s in res.scalars().all()}
async def set_settings(values: dict[str, str]) -> None:
async with SessionLocal() as session:
for key, value in values.items():
existing = await session.get(Setting, key)
if existing is None:
session.add(Setting(key=key, value=str(value)))
else:
existing.value = str(value)
await session.commit()

View File

@@ -14,12 +14,14 @@ async def create_video(
text_summary_path: str, text_summary_path: str,
transcription_method: str, transcription_method: str,
section_id: int | None = None, section_id: int | None = None,
job_id: str | None = None,
) -> Video: ) -> Video:
async with SessionLocal() as session: async with SessionLocal() as session:
v = Video( v = Video(
uuid=uuid, uuid=uuid,
source_url=source_url, source_url=source_url,
section_id=section_id, section_id=section_id,
job_id=job_id,
title=title or "", title=title or "",
video_path=video_path or "", video_path=video_path or "",
text_full_path=text_full_path or "", text_full_path=text_full_path or "",

119
api/lib/jobqueue.py Normal file
View File

@@ -0,0 +1,119 @@
"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются
последовательно (чтобы не долбить YouTube и LLM параллельными запросами).
Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs;
устойчивое состояние (счётчики, статус) — в таблице jobs.
"""
from __future__ import annotations
import asyncio
import logging
from api.db.job import get_job, update_job
from api.lib.processing import process_single_video
from api.lib.youtube import expand_url
logger = logging.getLogger("jobs")
_queue: asyncio.Queue[str] = asyncio.Queue()
_live: dict[str, str] = {}
_cancel: set[str] = set()
_params: dict[str, dict] = {}
def submit(job_id: str, params: dict) -> None:
_params[job_id] = params
_queue.put_nowait(job_id)
def live_status(job_id: str) -> str:
return _live.get(job_id, "")
def request_cancel(job_id: str) -> None:
_cancel.add(job_id)
def is_cancelled(job_id: str) -> bool:
return job_id in _cancel
async def worker() -> None:
"""Единственный воркер; запускается на старте приложения."""
logger.info("job worker started")
while True:
job_id = await _queue.get()
try:
await _run_job(job_id)
except Exception:
logger.exception("job %s crashed", job_id)
try:
await update_job(job_id, status="error", current_item="")
except Exception:
pass
finally:
_live.pop(job_id, None)
_cancel.discard(job_id)
_params.pop(job_id, None)
_queue.task_done()
async def _run_job(job_id: str) -> None:
job = await get_job(job_id)
if job is None or job.status == "cancelled":
return
params = _params.get(job_id, {})
loop = asyncio.get_running_loop()
await update_job(job_id, status="running", current_item="анализ ссылки…")
_live[job_id] = "анализ ссылки…"
plan = await loop.run_in_executor(None, expand_url, job.url)
entries = plan["entries"]
if not entries:
await update_job(job_id, status="error", error="Плейлист пуст", current_item="")
return
await update_job(
job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries)
)
done = failed = 0
errors: list[str] = []
n = len(entries)
for i, entry in enumerate(entries, start=1):
if is_cancelled(job_id):
await update_job(job_id, status="cancelled", current_item="")
logger.info("job %s cancelled at item %d/%d", job_id, i, n)
return
label = f"{i}/{n} · {entry['title']}"
await update_job(job_id, current_item=label)
_live[job_id] = label
def _notify(msg: str, _label: str = label) -> None:
_live[job_id] = f"{_label}{msg}"
try:
await process_single_video(
entry["url"],
section_id=params.get("section_id"),
model_path=params.get("model_path"),
summary_max_sentences=params.get("summary_max_sentences"),
keep_video=params.get("keep_video"),
job_id=job_id,
progress=_notify,
)
done += 1
except Exception as e:
failed += 1
errors.append(f"{entry['title']}: {e}")
logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e)
await update_job(
job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000]
)
status = "error" if done == 0 and failed > 0 else "done"
await update_job(job_id, status=status, current_item="")
logger.info("job %s finished: %d ok, %d failed", job_id, done, failed)

229
api/lib/processing.py Normal file
View File

@@ -0,0 +1,229 @@
"""Ядро пайплайна: одно видео от URL до записи в БД.
Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs.
Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из
настроек в БД (таблица settings) с фолбэком на переменные окружения.
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from typing import Callable, Optional
from api.config import settings
from api.db.setting import get_all_settings
from api.db.video import create_video
from api.lib.llm import LLMConfig
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
logger = logging.getLogger("processing")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class PipelineError(RuntimeError):
def __init__(self, message: str, status: int = 500):
super().__init__(message)
self.status = status
async def effective_defaults() -> dict:
"""Настройки из БД поверх переменных окружения."""
db = await get_all_settings()
base_url = db.get("llm_base_url") or settings.LLM_BASE_URL
model = db.get("llm_model") or settings.LLM_MODEL
llm_cfg = None
if base_url and model:
llm_cfg = LLMConfig(
base_url=base_url,
api_key=db.get("llm_api_key") or settings.LLM_API_KEY,
model=model,
timeout=settings.LLM_TIMEOUT,
max_tokens=settings.LLM_MAX_TOKENS,
temperature=settings.LLM_TEMPERATURE,
)
try:
max_sentences = int(db.get("summary_max_sentences") or 15)
except ValueError:
max_sentences = 15
return {
"llm_cfg": llm_cfg,
"summary_max_sentences": max_sentences,
"keep_video": db.get("keep_video", "1") != "0",
"model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL,
}
def resolve_model_path(raw: str) -> Path:
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise PipelineError(
"model_path must point inside the project's models directory", status=400
)
if not p.exists():
raise PipelineError(f"Model not found: {p}", status=400)
return p
async def process_single_video(
url: str,
*,
model_path: str | None = None,
summary_max_sentences: int | None = None,
keep_video: bool | None = None,
section_id: int | None = None,
job_id: str | None = None,
llm_cfg: Optional[LLMConfig] = None,
system_prompt: str | None = None,
user_template: str | None = None,
progress: Optional[Callable[[str], None]] = None,
) -> dict:
"""Скачивает, расшифровывает, сокращает и сохраняет одно видео.
Параметры со значением None подтягиваются из настроек (БД/env).
progress(msg) — живой статус для UI задач.
"""
notify = progress or (lambda msg: None)
defaults = await effective_defaults()
if llm_cfg is None:
llm_cfg = defaults["llm_cfg"]
if summary_max_sentences is None:
summary_max_sentences = defaults["summary_max_sentences"]
if keep_video is None:
keep_video = defaults["keep_video"]
raw_model_path = model_path or defaults["model_path"]
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video)
notify("скачивание…")
try:
info = await loop.run_in_executor(
None,
lambda: download_video_with_subs(
url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
# видео не сохраняем — достаточно аудио (для лекций в разы быстрее)
audio_only=not keep_video,
on_progress=lambda pct: notify(f"скачивание {pct}%"),
),
)
except Exception as e:
raise PipelineError(f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise PipelineError("Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
notify("распознавание речи (Vosk)…")
resolved_model = resolve_model_path(raw_model_path)
try:
full_text = await loop.run_in_executor(
None,
lambda: transcribe_via_ffmpeg(
video_path,
resolved_model,
on_progress=lambda m: notify(f"распознано {m} мин аудио"),
),
)
method = f"vosk:{resolved_model.name}"
except Exception as e:
raise PipelineError(f"Transcription failed: {e}")
if not full_text.strip():
raise PipelineError("No subtitles and transcription returned empty text")
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
notify("выжимка (LLM)…" if llm_cfg else "выжимка…")
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=summary_max_sentences,
),
)
except Exception as e:
raise PipelineError(f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
section_id=section_id,
job_id=job_id,
)
except Exception as e:
raise PipelineError(f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return {
"uuid": video_uuid,
"title": title,
"source_url": url,
"video_path": rel_video,
"text_full_path": rel_full,
"text_summary_path": rel_summary,
"transcription_method": method,
"summary_preview": summary[:300],
}

View File

@@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = (
DEFAULT_USER_TEMPLATE = ( DEFAULT_USER_TEMPLATE = (
"Сделай связную краткую выжимку следующего текста. " "Сделай связную краткую выжимку следующего текста. "
"Выдели основные тезисы и логику изложения. Объём — 512 предложений.\n\n" "Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n"
"---\n{text}\n---" "---\n{text}\n---"
) )
@@ -154,6 +154,7 @@ def summarize_llm(
user_template: Optional[str] = None, user_template: Optional[str] = None,
reduce_template: Optional[str] = None, reduce_template: Optional[str] = None,
chunk_chars: int = 10000, chunk_chars: int = 10000,
max_sentences: int = 15,
) -> str: ) -> str:
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts.""" """Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
text = (text or "").strip() text = (text or "").strip()
@@ -164,12 +165,19 @@ def summarize_llm(
user_t = user_template or DEFAULT_USER_TEMPLATE user_t = user_template or DEFAULT_USER_TEMPLATE
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
def _fmt(template: str, chunk: str) -> str:
try:
return template.format(text=chunk, n=max_sentences)
except (KeyError, IndexError):
# пользовательский шаблон без {n} / с лишними скобками
return template.replace("{text}", chunk)
def _one(chunk: str, template: str) -> str: def _one(chunk: str, template: str) -> str:
return chat_complete( return chat_complete(
cfg, cfg,
[ [
{"role": "system", "content": sys_p}, {"role": "system", "content": sys_p},
{"role": "user", "content": template.format(text=chunk)}, {"role": "user", "content": _fmt(template, chunk)},
], ],
) )
@@ -194,6 +202,7 @@ def summarize_llm(
user_template=reduce_t, user_template=reduce_t,
reduce_template=reduce_t, reduce_template=reduce_t,
chunk_chars=chunk_chars, chunk_chars=chunk_chars,
max_sentences=max_sentences,
) )
logger.info("LLM reduce step over %d partials", len(partials)) logger.info("LLM reduce step over %d partials", len(partials))
return _one(combined, reduce_t) return _one(combined, reduce_t)
@@ -215,6 +224,7 @@ def summarize(
llm_cfg, llm_cfg,
system_prompt=system_prompt, system_prompt=system_prompt,
user_template=user_template, user_template=user_template,
max_sentences=max_sentences,
) )
except LLMError as e: except LLMError as e:
logger.warning("LLM summarization failed, falling back to extractive: %s", e) logger.warning("LLM summarization failed, falling back to extractive: %s", e)

View File

@@ -53,8 +53,16 @@ class VoskModelManager:
t.join(timeout=2) t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str: def transcribe_via_ffmpeg(
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.""" media_path: Path,
model_path: Path,
sample_rate: int = 16000,
on_progress=None,
) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.
on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио.
"""
if Model is None or KaldiRecognizer is None: if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime") raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None: if shutil.which("ffmpeg") is None:
@@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int =
break break
recognizer.AcceptWaveform(chunk) recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk) bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024) # PCM s16le mono: sample_rate * 2 байта в секунду
if mb >= last_logged_mb + 5: minutes = bytes_read // (sample_rate * 2 * 60)
last_logged_mb = mb if minutes >= last_logged_mb + 5:
logger.info("Transcribed %d MB of audio so far", mb) last_logged_mb = minutes
logger.info("Transcribed %d minutes of audio so far", minutes)
if on_progress is not None:
try:
on_progress(minutes)
except Exception:
pass
result = json.loads(recognizer.FinalResult()) result = json.loads(recognizer.FinalResult())
text = result.get("text", "") text = result.get("text", "")

View File

@@ -12,7 +12,44 @@ import yt_dlp
logger = logging.getLogger("youtube") logger = logging.getLogger("youtube")
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"} _VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"}
def expand_url(url: str) -> dict:
"""Определяет, видео это или плейлист, и возвращает список элементов.
Returns: {"kind": "video"|"playlist", "title": str,
"entries": [{"url": str, "title": str}, ...]}
"""
opts = {
"extract_flat": "in_playlist",
"quiet": True,
"no_warnings": True,
"skip_download": True,
}
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info.get("_type") == "playlist":
entries = []
for e in info.get("entries") or []:
if not e:
continue
u = e.get("url") or ""
if u and not u.startswith("http"):
u = f"https://www.youtube.com/watch?v={u}"
if not u and e.get("id"):
u = f"https://www.youtube.com/watch?v={e['id']}"
if u:
entries.append({"url": u, "title": e.get("title") or u})
return {
"kind": "playlist",
"title": info.get("title") or url,
"entries": entries,
}
title = info.get("title") or url
return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]}
def download_video_with_subs( def download_video_with_subs(
@@ -20,9 +57,15 @@ def download_video_with_subs(
video_id: str, video_id: str,
video_dir: Path, video_dir: Path,
langs: Optional[list[str]] = None, langs: Optional[list[str]] = None,
audio_only: bool = False,
on_progress=None,
) -> dict: ) -> dict:
"""Download a single video, also requesting subtitles for the given languages. """Download a single video, also requesting subtitles for the given languages.
audio_only: качает только аудиодорожку (для длинных лекций, когда видео
не сохраняется — в разы быстрее и меньше по объёму).
on_progress: callback(percent: int), вызывается с шагом ~5%.
Returns: Returns:
{ {
"title": str, "title": str,
@@ -40,11 +83,34 @@ def download_video_with_subs(
base_opts = { base_opts = {
"outtmpl": outtmpl, "outtmpl": outtmpl,
"format": "bv*+ba/b",
"merge_output_format": "mp4",
"quiet": True, "quiet": True,
"no_warnings": True, "no_warnings": True,
"noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист
} }
if audio_only:
base_opts["format"] = "ba/b"
else:
base_opts["format"] = "bv*+ba/b"
base_opts["merge_output_format"] = "mp4"
if on_progress is not None:
state = {"last": -5}
def _hook(dd: dict):
if dd.get("status") != "downloading":
return
total = dd.get("total_bytes") or dd.get("total_bytes_estimate")
if not total:
return
pct = int(dd.get("downloaded_bytes", 0) * 100 / total)
if pct >= state["last"] + 5:
state["last"] = pct
try:
on_progress(pct)
except Exception:
pass
base_opts["progress_hooks"] = [_hook]
sub_opts = { sub_opts = {
**base_opts, **base_opts,
"writesubtitles": True, "writesubtitles": True,

View File

@@ -1,10 +1,13 @@
import asyncio
import logging import logging
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI from fastapi import FastAPI
from api.db.connection import wait_for_postgres, init_db from api.db.connection import wait_for_postgres, init_db
from api.route import default, llm, pipeline, sections, videos from api.db.job import mark_interrupted
from api.lib import jobqueue
from api.route import app_settings, default, jobs, llm, pipeline, sections, videos
logging.basicConfig( logging.basicConfig(
@@ -17,13 +20,18 @@ logging.basicConfig(
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
await wait_for_postgres() await wait_for_postgres()
await init_db() await init_db()
await mark_interrupted()
worker_task = asyncio.create_task(jobqueue.worker())
yield yield
worker_task.cancel()
app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan) app = FastAPI(title="LLM-infa API", version="0.4.0", lifespan=lifespan)
app.include_router(default.router) app.include_router(default.router)
app.include_router(pipeline.router) app.include_router(pipeline.router)
app.include_router(jobs.router)
app.include_router(videos.router) app.include_router(videos.router)
app.include_router(sections.router) app.include_router(sections.router)
app.include_router(app_settings.router)
app.include_router(llm.router) app.include_router(llm.router)

59
api/route/app_settings.py Normal file
View File

@@ -0,0 +1,59 @@
"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres,
переживают перезапуск контейнеров и перезагрузку страницы.
"""
from __future__ import annotations
from fastapi import APIRouter
from pydantic import BaseModel
from api.db.setting import get_all_settings, set_settings
router = APIRouter(prefix="/settings", tags=["settings"])
_KEYS = (
"llm_base_url",
"llm_api_key",
"llm_model",
"summary_max_sentences",
"keep_video",
"vosk_model_path",
)
class SettingsPayload(BaseModel):
llm_base_url: str | None = None
llm_api_key: str | None = None
llm_model: str | None = None
summary_max_sentences: int | None = None
keep_video: bool | None = None
vosk_model_path: str | None = None
@router.get("/")
async def get_settings():
db = await get_all_settings()
return {
"llm_base_url": db.get("llm_base_url", ""),
"llm_api_key": db.get("llm_api_key", ""),
"llm_model": db.get("llm_model", ""),
"summary_max_sentences": int(db.get("summary_max_sentences") or 15),
"keep_video": db.get("keep_video", "1") != "0",
"vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"),
}
@router.put("/")
async def put_settings(body: SettingsPayload):
values: dict[str, str] = {}
for key in _KEYS:
v = getattr(body, key)
if v is None:
continue
if key == "keep_video":
values[key] = "1" if v else "0"
else:
values[key] = str(v)
if values:
await set_settings(values)
return await get_settings()

112
api/route/jobs.py Normal file
View File

@@ -0,0 +1,112 @@
"""Фоновые задачи: видео или плейлист любой длины.
POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs.
"""
from __future__ import annotations
import uuid as _uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.db.job import create_job, delete_job, get_job, list_jobs
from api.lib import jobqueue
router = APIRouter(prefix="/jobs", tags=["jobs"])
class JobIn(BaseModel):
url: str
section_id: int | None = None
# None → берутся из сохранённых настроек
keep_video: bool | None = None
summary_max_sentences: int | None = None
model_path: str | None = None
class JobOut(BaseModel):
id: str
url: str
kind: str
title: str
status: str
total_items: int
done_items: int
failed_items: int
current_item: str
live: str
error: str
created_at: datetime
def _to_out(j) -> JobOut:
return JobOut(
id=j.id,
url=j.url,
kind=j.kind,
title=j.title,
status=j.status,
total_items=j.total_items,
done_items=j.done_items,
failed_items=j.failed_items,
current_item=j.current_item,
live=jobqueue.live_status(j.id) or j.current_item,
error=j.error,
created_at=j.created_at,
)
@router.post("/", response_model=JobOut)
async def create(body: JobIn):
url = body.url.strip()
if not url:
raise HTTPException(status_code=400, detail="URL is empty")
job_id = _uuid.uuid4().hex
j = await create_job(job_id, url)
jobqueue.submit(
job_id,
{
"section_id": body.section_id,
"keep_video": body.keep_video,
"summary_max_sentences": body.summary_max_sentences,
"model_path": body.model_path,
},
)
return _to_out(j)
@router.get("/", response_model=list[JobOut])
async def list_all(limit: int = 50):
return [_to_out(j) for j in await list_jobs(limit)]
@router.get("/{job_id}", response_model=JobOut)
async def get_one(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
return _to_out(j)
@router.post("/{job_id}/cancel")
async def cancel(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status not in ("queued", "running"):
raise HTTPException(status_code=400, detail=f"Job is {j.status}")
jobqueue.request_cancel(job_id)
return {"ok": True, "note": "остановится после текущего элемента"}
@router.delete("/{job_id}")
async def delete(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status in ("queued", "running"):
raise HTTPException(status_code=400, detail="Сначала отмените задачу")
await delete_job(job_id)
return {"ok": True}

View File

@@ -1,29 +1,16 @@
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist.""" """Синхронный запуск пайплайна для одного видео (для длинных видео и
плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым).
"""
from __future__ import annotations from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from fastapi import APIRouter, HTTPException from fastapi import APIRouter, HTTPException
from pydantic import BaseModel from pydantic import BaseModel
from api.config import settings from api.lib.llm import LLMConfig
from api.db.video import create_video from api.lib.processing import PipelineError, process_single_video
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
router = APIRouter(prefix="/pipeline", tags=["pipeline"]) router = APIRouter(prefix="/pipeline", tags=["pipeline"])
logger = logging.getLogger("pipeline")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class LLMOverride(BaseModel): class LLMOverride(BaseModel):
@@ -40,12 +27,12 @@ class LLMOverride(BaseModel):
class ProcessRequest(BaseModel): class ProcessRequest(BaseModel):
url: str url: str
model_path: str | None = None # path inside models/, used when subtitles are unavailable model_path: str | None = None # None → из настроек
summary_max_sentences: int = 15 summary_max_sentences: int | None = None
keep_video: bool = True # False: файл видео удаляется после расшифровки keep_video: bool | None = None
section_id: int | None = None section_id: int | None = None
# If supplied, this LLM is used for the summary; otherwise env-configured LLM; # Если задано — используется этот LLM; иначе сохранённые настройки/env;
# otherwise the built-in extractive fallback. # иначе встроенная экстрактивная выжимка.
llm: LLMOverride | None = None llm: LLMOverride | None = None
@@ -60,86 +47,10 @@ class ProcessResponse(BaseModel):
summary_preview: str summary_preview: str
def _resolve_model_path(req_model_path: str | None) -> Path:
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise HTTPException(
status_code=400,
detail="model_path must point inside the project's models directory",
)
if not p.exists():
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
return p
@router.post("/process", response_model=ProcessResponse) @router.post("/process", response_model=ProcessResponse)
async def process_video(req: ProcessRequest): async def process_video(req: ProcessRequest):
video_uuid = _uuid.uuid4().hex llm_cfg = None
_VIDEO_DIR.mkdir(parents=True, exist_ok=True) sys_p = user_t = None
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s", video_uuid, req.url)
try:
info = await loop.run_in_executor(
None,
download_video_with_subs,
req.url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise HTTPException(status_code=500, detail="Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
model_path = _resolve_model_path(req.model_path)
try:
full_text = await loop.run_in_executor(
None, transcribe_via_ffmpeg, video_path, model_path
)
method = f"vosk:{model_path.name}"
except Exception as e:
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
if not full_text.strip():
raise HTTPException(
status_code=500, detail="No subtitles and transcription returned empty text"
)
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
if req.llm is not None: if req.llm is not None:
llm_cfg = LLMConfig( llm_cfg = LLMConfig(
base_url=req.llm.base_url, base_url=req.llm.base_url,
@@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest):
) )
sys_p = req.llm.system_prompt sys_p = req.llm.system_prompt
user_t = req.llm.user_template user_t = req.llm.user_template
else:
llm_cfg = llm_from_settings()
sys_p = None
user_t = None
try: try:
summary = await loop.run_in_executor( result = await process_single_video(
None, req.url,
lambda: summarize( model_path=req.model_path,
full_text, summary_max_sentences=req.summary_max_sentences,
llm_cfg=llm_cfg, keep_video=req.keep_video,
system_prompt=sys_p,
user_template=user_t,
max_sentences=req.summary_max_sentences,
),
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if req.keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=req.url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
section_id=req.section_id, section_id=req.section_id,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
) )
except Exception as e: except PipelineError as e:
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}") raise HTTPException(status_code=e.status, detail=str(e))
return ProcessResponse(**result)
logger.info("[%s] done (method=%s)", video_uuid, method)
return ProcessResponse(
uuid=video_uuid,
title=title,
source_url=req.url,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
summary_preview=summary[:300],
)

View File

@@ -1,6 +1,6 @@
[project] [project]
name = "llm-infa" name = "llm-infa"
version = "0.3.0" version = "0.4.0"
description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres" description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres"
readme = "README.md" readme = "README.md"
requires-python = ">=3.14" requires-python = ">=3.14"

View File

@@ -21,10 +21,15 @@ class ApiClient:
def close(self) -> None: def close(self) -> None:
self._client.close() self._client.close()
# --- LLM --- # --- Settings (persisted server-side in Postgres) ---
def llm_status(self) -> dict: def get_settings(self) -> dict:
return self._get("/llm/config") return self._get("/settings/")
def save_settings(self, values: dict) -> dict:
return self._request("PUT", "/settings/", values)
# --- LLM ---
def llm_models(self, base_url: str, api_key: str) -> list[dict]: def llm_models(self, base_url: str, api_key: str) -> list[dict]:
data = self._post( data = self._post(
@@ -39,10 +44,21 @@ class ApiClient:
{"base_url": base_url, "api_key": api_key, "model": model}, {"base_url": base_url, "api_key": api_key, "model": model},
) )
# --- Pipeline / videos --- # --- Jobs (видео и плейлисты, в фоне) ---
def pipeline_process(self, payload: dict) -> dict: def create_job(self, payload: dict) -> dict:
return self._post("/pipeline/process", payload) return self._post("/jobs/", payload)
def list_jobs(self) -> list[dict]:
return self._get("/jobs/")
def cancel_job(self, job_id: str) -> dict:
return self._post(f"/jobs/{job_id}/cancel", None)
def delete_job(self, job_id: str) -> dict:
return self._request("DELETE", f"/jobs/{job_id}")
# --- Videos ---
def list_videos( def list_videos(
self, self,

View File

@@ -19,20 +19,14 @@ def main(page: ft.Page) -> None:
view = MainView(page, client) view = MainView(page, client)
page.add(view.build()) page.add(view.build())
# probe env-configured LLM and initial library load # настройки живут на сервере (БД) — заполняем поля сохранёнными значениями
try: try:
status = client.llm_status() view.apply_settings(client.get_settings())
if status.get("configured"):
view._set_status(ok=True)
if status.get("base_url") and not view.base_url.value:
view.base_url.value = status["base_url"]
if status.get("model") and not view.selected_model:
view.selected_model = status["model"]
view.model_input.value = status["model"]
except Exception: except Exception:
pass pass
view._refresh_library() view._refresh_library()
view._start_jobs_poll()
page.update() page.update()

View File

@@ -1,7 +1,8 @@
"""Main page of the LLM-infa Flet web app.""" """Main page of the LLM-infa Flet web app: вкладки «Главная» и «Настройки»."""
from __future__ import annotations from __future__ import annotations
import threading import threading
import time
import flet as ft import flet as ft
@@ -9,6 +10,16 @@ from web.api_client import ApiClient, ApiError
import web.designer as d import web.designer as d
_JOB_STATUS = {
"queued": ("в очереди", None),
"running": ("выполняется", None),
"done": ("готово", "ok"),
"error": ("ошибка", "danger"),
"cancelled": ("отменена", None),
"interrupted": ("прервана", "danger"),
}
class MainView: class MainView:
def __init__(self, page: ft.Page, client: ApiClient): def __init__(self, page: ft.Page, client: ApiClient):
self.page = page self.page = page
@@ -20,14 +31,15 @@ class MainView:
self.sections: list[dict] = [] self.sections: list[dict] = []
self.current_section_id: int | None = None # None = все видео self.current_section_id: int | None = None # None = все видео
# ── LLM config fields ────────────────────────────────────────────── self._polling = False
# ── Settings tab fields ────────────────────────────────────────────
self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True) self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True)
self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True) self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True)
# Searchable model combobox
self.model_input = d.text_field( self.model_input = d.text_field(
"Модель", "Модель",
hint="Сначала нажмите «Загрузить модели»", hint="Нажмите «Загрузить модели» и выберите из списка",
on_change=self._on_model_search, on_change=self._on_model_search,
expand=True, expand=True,
) )
@@ -44,7 +56,7 @@ class MainView:
self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True) self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.llm_status_chip = ft.Container( self.llm_status_chip = ft.Container(
content=ft.Text("LLM не подключена", size=12, color=d.MUTED), content=ft.Text("LLM не настроена", size=12, color=d.MUTED),
padding=ft.padding.symmetric(horizontal=10, vertical=4), padding=ft.padding.symmetric(horizontal=10, vertical=4),
border=ft.border.all(1, d.BORDER), border=ft.border.all(1, d.BORDER),
border_radius=99, border_radius=99,
@@ -52,12 +64,6 @@ class MainView:
self.btn_load = d.primary_button("Загрузить модели", self._load_models) self.btn_load = d.primary_button("Загрузить модели", self._load_models)
self.btn_test = d.secondary_button("Проверить связь", self._test_conn) self.btn_test = d.secondary_button("Проверить связь", self._test_conn)
# ── Pipeline fields ────────────────────────────────────────────────
self.yt_url = d.text_field(
"Ссылка на YouTube",
hint="https://youtu.be/…",
expand=True,
)
self.vosk_path = d.text_field( self.vosk_path = d.text_field(
"Vosk-модель (fallback, когда нет субтитров)", "Vosk-модель (fallback, когда нет субтитров)",
value="models/vosk-model-small-ru-0.22", value="models/vosk-model-small-ru-0.22",
@@ -76,19 +82,28 @@ class MainView:
width=220, width=220,
) )
self.keep_video_sw = ft.Switch( self.keep_video_sw = ft.Switch(
label="Сохранять видео на диске", label="Сохранять видео на диске (иначе качается только аудио)",
value=True, value=True,
active_color=d.ACCENT, active_color=d.ACCENT,
label_style=ft.TextStyle(color=d.TEXT, size=13), label_style=ft.TextStyle(color=d.TEXT, size=13),
) )
self.pipeline_section_dd = self._dropdown("Раздел для нового видео", width=280) self.settings_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.btn_save = d.primary_button("💾 Сохранить настройки", self._save_settings)
# ── Pipeline (main tab) ────────────────────────────────────────────
self.yt_url = d.text_field(
"Ссылка на YouTube: видео или плейлист",
hint="https://youtu.be/… или https://www.youtube.com/playlist?list=…",
expand=True,
)
self.pipeline_section_dd = self._dropdown("Раздел для новых видео", width=280)
self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True) self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.btn_run = d.primary_button("Запустить", self._run_pipeline) self.btn_run = d.primary_button("В очередь", self._run_pipeline)
# Result block (initially hidden) # ── Jobs ───────────────────────────────────────────────────────────
self._result_col = ft.Column(spacing=6, visible=False) self._jobs_col = ft.Column(spacing=6)
# ── Library controls ─────────────────────────────────────────────── # ── Library ────────────────────────────────────────────────────────
self.search_field = d.text_field( self.search_field = d.text_field(
"Поиск по названию / URL", on_change=None, expand=True "Поиск по названию / URL", on_change=None, expand=True
) )
@@ -124,24 +139,47 @@ class MainView:
# ─────────────────────── build ─────────────────────────────────────── # ─────────────────────── build ───────────────────────────────────────
def build(self) -> ft.Control: def build(self) -> ft.Control:
return ft.Column( main_tab = ft.Container(
controls=[ content=ft.Column(
self._build_topbar(), [
ft.Container( self._build_pipeline_card(),
content=ft.Column( self._build_jobs_card(),
[ self._build_library_card(),
self._build_llm_card(), ],
self._build_pipeline_card(), spacing=20,
self._build_library_card(), scroll=ft.ScrollMode.AUTO,
], ),
spacing=20, padding=ft.padding.symmetric(horizontal=24, vertical=20),
scroll=ft.ScrollMode.AUTO, expand=True,
), )
padding=ft.padding.symmetric(horizontal=24, vertical=20), settings_tab = ft.Container(
expand=True, content=ft.Column(
), [
self._build_llm_card(),
self._build_defaults_card(),
],
spacing=20,
scroll=ft.ScrollMode.AUTO,
),
padding=ft.padding.symmetric(horizontal=24, vertical=20),
expand=True,
)
tabs = ft.Tabs(
selected_index=0,
animation_duration=200,
label_color=d.ACCENT_2,
unselected_label_color=d.MUTED,
indicator_color=d.ACCENT,
divider_color=d.BORDER,
tabs=[
ft.Tab(text="Главная", content=main_tab),
ft.Tab(text="Настройки", content=settings_tab),
], ],
expand=True, expand=True,
)
return ft.Column(
controls=[self._build_topbar(), tabs],
expand=True,
spacing=0, spacing=0,
) )
@@ -174,13 +212,14 @@ class MainView:
border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)), border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)),
) )
# ── Settings tab cards ────────────────────────────────────────────────
def _build_llm_card(self) -> ft.Container: def _build_llm_card(self) -> ft.Container:
return d.card( return d.card(
d.section_title("Подключение к LLM"), d.section_title("Подключение к LLM"),
d.hint( d.hint(
"Укажите любой OpenAI-совместимый провайдер. " "Любой OpenAI-совместимый провайдер (OpenRouter, OpenAI, локальная Ollama…). "
"Нейронка внутри Docker не разворачивается — " "Ключ и модель хранятся на сервере в БД и переживают перезапуск."
"указываем URL внешнего сервиса."
), ),
ft.Row([self.base_url, self.api_key], spacing=12), ft.Row([self.base_url, self.api_key], spacing=12),
ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10), ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10),
@@ -189,22 +228,43 @@ class MainView:
self._model_list_wrap, self._model_list_wrap,
) )
def _build_defaults_card(self) -> ft.Container:
return d.card(
d.section_title("Параметры обработки"),
d.hint("Применяются ко всем новым задачам."),
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
self.keep_video_sw,
ft.Divider(color=d.BORDER, height=1),
ft.Row([self.btn_save, self.settings_msg], spacing=10),
)
# ── Main tab cards ────────────────────────────────────────────────────
def _build_pipeline_card(self) -> ft.Container: def _build_pipeline_card(self) -> ft.Container:
return d.card( return d.card(
d.section_title("Обработать видео"), d.section_title("Обработать видео или плейлист"),
d.hint( d.hint(
"Pipeline: yt-dlp → субтитры (или Vosk fallback) " "Задача уходит в фон: качаем, расшифровываем (субтитры или Vosk), "
"→ выжимка через выбранную LLM → запись в БД." "сокращаем через LLM, пишем в БД. Плейлисты и лекции на 45 часов — ок."
), ),
self.yt_url, self.yt_url,
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
ft.Row( ft.Row(
[self.pipeline_section_dd, self.keep_video_sw], [self.btn_run, self.pipeline_section_dd, self.pipeline_msg],
spacing=18, spacing=12,
vertical_alignment=ft.CrossAxisAlignment.CENTER, vertical_alignment=ft.CrossAxisAlignment.CENTER,
), ),
ft.Row([self.btn_run, self.pipeline_msg], spacing=10), )
self._result_col,
def _build_jobs_card(self) -> ft.Container:
return d.card(
ft.Row(
[
d.section_title("Задачи"),
ft.Container(expand=True),
d.secondary_button("Обновить", lambda _: self._start_jobs_poll()),
]
),
self._jobs_col,
) )
def _build_library_card(self) -> ft.Container: def _build_library_card(self) -> ft.Container:
@@ -256,6 +316,39 @@ class MainView:
), ),
) )
# ─────────────────── settings load/save ──────────────────────────────
def apply_settings(self, s: dict) -> None:
"""Заполняет поля значениями с сервера (вызывается при старте сессии)."""
self.base_url.value = s.get("llm_base_url", "")
self.api_key.value = s.get("llm_api_key", "")
self.selected_model = s.get("llm_model", "")
self.model_input.value = self.selected_model
self.n_sentences.value = str(s.get("summary_max_sentences", 15))
self.keep_video_sw.value = bool(s.get("keep_video", True))
self.vosk_path.value = s.get("vosk_model_path", "models/vosk-model-small-ru-0.22")
self._set_status(ok=bool(self.base_url.value and self.selected_model))
def _save_settings(self, _):
model = self.selected_model or (self.model_input.value or "").strip()
payload = {
"llm_base_url": self.base_url.value.strip(),
"llm_api_key": self.api_key.value.strip(),
"llm_model": model,
"summary_max_sentences": int(self.n_sentences.value or 15),
"keep_video": bool(self.keep_video_sw.value),
"vosk_model_path": self.vosk_path.value.strip(),
}
try:
self.client.save_settings(payload)
self.settings_msg.value = "Сохранено ✓ (хранится в БД, переживёт перезапуск)"
self.settings_msg.color = d.OK
self._set_status(ok=bool(payload["llm_base_url"] and model))
except ApiError as e:
self.settings_msg.value = str(e)
self.settings_msg.color = d.DANGER
self.page.update()
# ─────────────────── model combobox ────────────────────────────────── # ─────────────────── model combobox ──────────────────────────────────
def _on_model_search(self, e): def _on_model_search(self, e):
@@ -292,6 +385,8 @@ class MainView:
self.selected_model = mid self.selected_model = mid
self.model_input.value = mid self.model_input.value = mid
self._model_list_wrap.visible = False self._model_list_wrap.visible = False
self.llm_msg.value = "Не забудьте нажать «Сохранить настройки»"
self.llm_msg.color = d.MUTED
self.page.update() self.page.update()
# ─────────────────── LLM actions ───────────────────────────────────── # ─────────────────── LLM actions ─────────────────────────────────────
@@ -311,10 +406,8 @@ class MainView:
self.models = self.client.llm_models(base_url, self.api_key.value.strip()) self.models = self.client.llm_models(base_url, self.api_key.value.strip())
self._render_model_list(self.models) self._render_model_list(self.models)
self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True) self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True)
self._set_status(ok=True)
except ApiError as e: except ApiError as e:
self._set_llm_msg(str(e), error=True) self._set_llm_msg(str(e), error=True)
self._set_status(ok=False)
finally: finally:
self.btn_load.disabled = False self.btn_load.disabled = False
self.page.update() self.page.update()
@@ -322,7 +415,8 @@ class MainView:
threading.Thread(target=_task, daemon=True).start() threading.Thread(target=_task, daemon=True).start()
def _test_conn(self, _): def _test_conn(self, _):
if not self.selected_model: model = self.selected_model or (self.model_input.value or "").strip()
if not model:
self._set_llm_msg("Сначала выберите модель", error=True) self._set_llm_msg("Сначала выберите модель", error=True)
return return
@@ -335,87 +429,164 @@ class MainView:
r = self.client.llm_test( r = self.client.llm_test(
self.base_url.value.strip(), self.base_url.value.strip(),
self.api_key.value.strip(), self.api_key.value.strip(),
self.selected_model, model,
) )
preview = (r.get("reply") or "")[:80] preview = (r.get("reply") or "")[:80]
self._set_llm_msg(f"OK · {preview}", ok=True) self._set_llm_msg(f"OK · {preview}", ok=True)
self._set_status(ok=True)
except ApiError as e: except ApiError as e:
self._set_llm_msg(str(e), error=True) self._set_llm_msg(str(e), error=True)
self._set_status(ok=False)
finally: finally:
self.btn_test.disabled = False self.btn_test.disabled = False
self.page.update() self.page.update()
threading.Thread(target=_task, daemon=True).start() threading.Thread(target=_task, daemon=True).start()
# ─────────────────── pipeline ───────────────────────────────────────── # ─────────────────── pipeline → jobs ──────────────────────────────────
def _run_pipeline(self, _): def _run_pipeline(self, _):
url = self.yt_url.value.strip() url = self.yt_url.value.strip()
if not url: if not url:
self._set_pipeline_msg("Укажите ссылку на видео", error=True) self._set_pipeline_msg("Укажите ссылку на видео или плейлист", error=True)
return return
payload: dict = { payload: dict = {"url": url}
"url": url,
"model_path": self.vosk_path.value.strip() or None,
"summary_max_sentences": int(self.n_sentences.value or 15),
"keep_video": bool(self.keep_video_sw.value),
}
if self.pipeline_section_dd.value: if self.pipeline_section_dd.value:
payload["section_id"] = int(self.pipeline_section_dd.value) payload["section_id"] = int(self.pipeline_section_dd.value)
if self.base_url.value.strip() and self.selected_model:
payload["llm"] = {
"base_url": self.base_url.value.strip(),
"api_key": self.api_key.value.strip(),
"model": self.selected_model,
}
self._set_pipeline_msg("Обработка видео… (может занять несколько минут)") try:
self._result_col.visible = False self.client.create_job(payload)
self.btn_run.disabled = True self.yt_url.value = ""
self._set_pipeline_msg("Задача поставлена в очередь ✓", ok=True)
except ApiError as e:
self._set_pipeline_msg(str(e), error=True)
self.page.update() self.page.update()
self._start_jobs_poll()
def _task(): # ─────────────────── jobs ─────────────────────────────────────────────
try:
r = self.client.pipeline_process(payload)
self._render_result(r)
self._set_pipeline_msg("Готово ✓", ok=True)
self._refresh_library()
except ApiError as e:
self._set_pipeline_msg(str(e), error=True)
finally:
self.btn_run.disabled = False
self.page.update()
threading.Thread(target=_task, daemon=True).start() def _start_jobs_poll(self):
if self._polling:
return
self._polling = True
threading.Thread(target=self._poll_loop, daemon=True).start()
def _render_result(self, r: dict): def _poll_loop(self):
self._result_col.controls = [ prev: dict = {}
ft.Divider(color=d.BORDER, height=1), try:
ft.Text(r.get("title") or r.get("uuid", ""), color=d.ACCENT_2, size=15, weight=ft.FontWeight.W_600), while True:
*[ try:
self._kv(k, v) jobs = self.client.list_jobs()
for k, v in [ except (ApiError, Exception):
("UUID", r.get("uuid", "")), break
("Источник", r.get("source_url", "")), self._render_jobs(jobs)
("Метод", r.get("transcription_method", "")), state = {j["id"]: (j["done_items"], j["status"]) for j in jobs}
("Видео", r.get("video_path") or "не сохранено"), if state != prev:
("Полный текст", r.get("text_full_path", "")), prev = state
("Выжимка", r.get("text_summary_path", "")), self._refresh_library() # там же page.update()
] else:
], self.page.update()
ft.Text("Превью выжимки:", color=d.MUTED, size=12), if not any(j["status"] in ("queued", "running") for j in jobs):
ft.Container( break
content=ft.Text(r.get("summary_preview", ""), color=d.TEXT, size=13, selectable=True, no_wrap=False), time.sleep(3)
padding=12, finally:
bgcolor=d.SURFACE_3, self._polling = False
border=ft.border.all(1, d.BORDER),
border_radius=8, def _render_jobs(self, jobs: list[dict]):
if not jobs:
self._jobs_col.controls = [
ft.Text("Задач ещё не было", color=d.MUTED, size=13)
]
return
self._jobs_col.controls = [self._job_row(j) for j in jobs]
def _job_row(self, j: dict) -> ft.Container:
label, tone = _JOB_STATUS.get(j["status"], (j["status"], None))
color = d.OK if tone == "ok" else (d.DANGER if tone == "danger" else d.MUTED)
active = j["status"] in ("queued", "running")
chip = ft.Container(
content=ft.Text(label, size=11, color=color),
padding=ft.padding.symmetric(horizontal=8, vertical=2),
border=ft.border.all(1, color),
border_radius=99,
)
head = [
chip,
ft.Text(
(j["title"] or j["url"])[:90],
color=d.TEXT,
size=13,
weight=ft.FontWeight.W_600,
expand=True,
no_wrap=True,
), ),
] ]
self._result_col.visible = True if j["kind"] == "playlist":
head.append(ft.Text("плейлист", color=d.ACCENT_2, size=11))
if j["total_items"]:
head.append(
ft.Text(f"{j['done_items']}/{j['total_items']}", color=d.MUTED, size=12)
)
if active:
head.append(
ft.IconButton(
icon=ft.Icons.STOP_CIRCLE_OUTLINED,
icon_color=d.DANGER,
icon_size=17,
tooltip="Отменить (после текущего элемента)",
on_click=lambda e, jid=j["id"]: self._cancel_job(jid),
)
)
else:
head.append(
ft.IconButton(
icon=ft.Icons.CLOSE,
icon_color=d.MUTED,
icon_size=15,
tooltip="Убрать из списка",
on_click=lambda e, jid=j["id"]: self._delete_job(jid),
)
)
rows: list[ft.Control] = [ft.Row(head, spacing=8)]
if active:
total = j["total_items"] or 0
value = (j["done_items"] / total) if total else None
rows.append(
ft.ProgressBar(value=value, color=d.ACCENT, bgcolor=d.SURFACE_3, height=4)
)
if j.get("live"):
rows.append(ft.Text(j["live"][:140], color=d.ACCENT_2, size=11))
if j.get("error"):
rows.append(
ft.Text(j["error"][:300], color=d.DANGER, size=11, no_wrap=False)
)
if j["failed_items"]:
rows.append(
ft.Text(f"с ошибкой: {j['failed_items']}", color=d.DANGER, size=11)
)
return ft.Container(
content=ft.Column(rows, spacing=5, tight=True),
padding=10,
bgcolor=d.SURFACE_2,
border=ft.border.all(1, d.BORDER),
border_radius=10,
)
def _cancel_job(self, job_id: str):
try:
self.client.cancel_job(job_id)
except ApiError as e:
self.lib_msg.value = str(e)
self._start_jobs_poll()
def _delete_job(self, job_id: str):
try:
self.client.delete_job(job_id)
except ApiError as e:
self.lib_msg.value = str(e)
self._start_jobs_poll()
# ─────────────────── sections ───────────────────────────────────────── # ─────────────────── sections ─────────────────────────────────────────
@@ -688,6 +859,6 @@ class MainView:
def _set_status(self, *, ok: bool): def _set_status(self, *, ok: bool):
chip = self.llm_status_chip chip = self.llm_status_chip
chip.content.value = "LLM подключена" if ok else "Ошибка подключения" chip.content.value = "LLM настроена" if ok else "LLM не настроена"
chip.content.color = d.OK if ok else d.DANGER chip.content.color = d.OK if ok else d.MUTED
chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else "rgba(227,90,114,.4)") chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else d.BORDER)