jze9/LLM-infa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Фоновые задачи: плейлисты, длинные лекции, вкладка настроек

Browse Source 
- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
  плейлиста обрабатываются последовательно, есть отмена и живой прогресс
  (скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
  хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
  каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
  готовности элементов

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 15:01:11 +05:00
parent bf854f2d6e
commit e87f6cd215
18 changed files with 1059 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
README.md
View File

@@ -48,19 +48,38 @@ OpenAI-совместимый эндпоинт. Два способа:
Если LLM не настроена, работает встроенная экстрактивная суммаризация
(выбор ключевых предложений, без нейросети).
## Плейлисты и длинные лекции
Обработка идёт через фоновые задачи: `POST /jobs` сразу возвращает id, элементы
плейлиста обрабатываются по очереди, прогресс виден в UI (скачивание %,
минуты распознанного аудио, номер видео в плейлисте). Если в настройках
выключено «Сохранять видео», для лекций качается только аудиодорожка —
в разы быстрее и легче.
Настройки (ключ LLM, модель, длина выжимки, хранение видео) сохраняются
в Postgres — переживают перезапуск контейнеров и перезагрузку страницы.
Вкладка «Настройки» в UI.
## API
```bash
# обработать видео целиком
# фоновая задача: видео или плейлист
curl -X POST localhost:8000/jobs/ \
-H 'Content-Type: application/json' \
-d '{"url": "https://www.youtube.com/playlist?list=..."}'
curl localhost:8000/jobs/ # статусы и прогресс
# синхронно, одно видео (для коротких роликов)
curl -X POST localhost:8000/pipeline/process \
-H 'Content-Type: application/json' \
-d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}'
# каталог обработанных видео
# каталог обработанных видео (фильтры: ?q=, ?section_id=, ?method=)
curl localhost:8000/videos/
# проверить связь с LLM
curl -X POST localhost:8000/llm/test
# настройки
curl -X PUT localhost:8000/settings/ -H 'Content-Type: application/json' \
-d '{"llm_base_url": "https://openrouter.ai/api/v1", "llm_api_key": "sk-...", "llm_model": "..."}'
```
## Заметки

5
api/db/connection.py
View File

@@ -8,7 +8,9 @@ from sqlalchemy.orm import sessionmaker
from api.config import settings
from api.models.base import Base
import api.models.job # noqa: F401 - register model with Base.metadata
import api.models.section # noqa: F401 - register model with Base.metadata
import api.models.setting # noqa: F401 - register model with Base.metadata
import api.models.video # noqa: F401 - register model with Base.metadata
@@ -69,3 +71,6 @@ async def init_db() -> None:
"REFERENCES sections(id) ON DELETE SET NULL"
)
)
await conn.execute(
text("ALTER TABLE videos ADD COLUMN IF NOT EXISTS job_id VARCHAR")
)

53
api/db/job.py Normal file
View File

@@ -0,0 +1,53 @@
from sqlalchemy import select, update
from api.db.connection import SessionLocal
from api.models.job import Job
async def create_job(job_id: str, url: str) -> Job:
async with SessionLocal() as session:
j = Job(id=job_id, url=url, status="queued")
session.add(j)
await session.commit()
await session.refresh(j)
return j
async def get_job(job_id: str) -> Job | None:
async with SessionLocal() as session:
return await session.get(Job, job_id)
async def list_jobs(limit: int = 50) -> list[Job]:
async with SessionLocal() as session:
res = await session.execute(
select(Job).order_by(Job.created_at.desc()).limit(limit)
)
return list(res.scalars().all())
async def update_job(job_id: str, **fields) -> None:
async with SessionLocal() as session:
await session.execute(update(Job).where(Job.id == job_id).values(**fields))
await session.commit()
async def delete_job(job_id: str) -> bool:
async with SessionLocal() as session:
j = await session.get(Job, job_id)
if j is None:
return False
await session.delete(j)
await session.commit()
return True
async def mark_interrupted() -> None:
"""На старте приложения: задачи, шедшие до перезапуска, отмечаем прерванными."""
async with SessionLocal() as session:
await session.execute(
update(Job)
.where(Job.status.in_(("queued", "running")))
.values(status="interrupted", current_item="")
)
await session.commit()

21
api/db/setting.py Normal file
View File

@@ -0,0 +1,21 @@
from sqlalchemy import select
from api.db.connection import SessionLocal
from api.models.setting import Setting
async def get_all_settings() -> dict[str, str]:
async with SessionLocal() as session:
res = await session.execute(select(Setting))
return {s.key: s.value for s in res.scalars().all()}
async def set_settings(values: dict[str, str]) -> None:
async with SessionLocal() as session:
for key, value in values.items():
existing = await session.get(Setting, key)
if existing is None:
session.add(Setting(key=key, value=str(value)))
else:
existing.value = str(value)
await session.commit()

2
api/db/video.py
View File

@@ -14,12 +14,14 @@ async def create_video(
text_summary_path: str,
transcription_method: str,
section_id: int | None = None,
job_id: str | None = None,
) -> Video:
async with SessionLocal() as session:
v = Video(
uuid=uuid,
source_url=source_url,
section_id=section_id,
job_id=job_id,
title=title or "",
video_path=video_path or "",
text_full_path=text_full_path or "",

119
api/lib/jobqueue.py Normal file
View File

@@ -0,0 +1,119 @@
"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются
последовательно (чтобы не долбить YouTube и LLM параллельными запросами).
Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs;
устойчивое состояние (счётчики, статус) — в таблице jobs.
"""
from __future__ import annotations
import asyncio
import logging
from api.db.job import get_job, update_job
from api.lib.processing import process_single_video
from api.lib.youtube import expand_url
logger = logging.getLogger("jobs")
_queue: asyncio.Queue[str] = asyncio.Queue()
_live: dict[str, str] = {}
_cancel: set[str] = set()
_params: dict[str, dict] = {}
def submit(job_id: str, params: dict) -> None:
_params[job_id] = params
_queue.put_nowait(job_id)
def live_status(job_id: str) -> str:
return _live.get(job_id, "")
def request_cancel(job_id: str) -> None:
_cancel.add(job_id)
def is_cancelled(job_id: str) -> bool:
return job_id in _cancel
async def worker() -> None:
"""Единственный воркер; запускается на старте приложения."""
logger.info("job worker started")
while True:
job_id = await _queue.get()
try:
await _run_job(job_id)
except Exception:
logger.exception("job %s crashed", job_id)
try:
await update_job(job_id, status="error", current_item="")
except Exception:
pass
finally:
_live.pop(job_id, None)
_cancel.discard(job_id)
_params.pop(job_id, None)
_queue.task_done()
async def _run_job(job_id: str) -> None:
job = await get_job(job_id)
if job is None or job.status == "cancelled":
return
params = _params.get(job_id, {})
loop = asyncio.get_running_loop()
await update_job(job_id, status="running", current_item="анализ ссылки…")
_live[job_id] = "анализ ссылки…"
plan = await loop.run_in_executor(None, expand_url, job.url)
entries = plan["entries"]
if not entries:
await update_job(job_id, status="error", error="Плейлист пуст", current_item="")
return
await update_job(
job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries)
)
done = failed = 0
errors: list[str] = []
n = len(entries)
for i, entry in enumerate(entries, start=1):
if is_cancelled(job_id):
await update_job(job_id, status="cancelled", current_item="")
logger.info("job %s cancelled at item %d/%d", job_id, i, n)
return
label = f"{i}/{n} · {entry['title']}"
await update_job(job_id, current_item=label)
_live[job_id] = label
def _notify(msg: str, _label: str = label) -> None:
_live[job_id] = f"{_label} — {msg}"
try:
await process_single_video(
entry["url"],
section_id=params.get("section_id"),
model_path=params.get("model_path"),
summary_max_sentences=params.get("summary_max_sentences"),
keep_video=params.get("keep_video"),
job_id=job_id,
progress=_notify,
)
done += 1
except Exception as e:
failed += 1
errors.append(f"{entry['title']}: {e}")
logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e)
await update_job(
job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000]
)
status = "error" if done == 0 and failed > 0 else "done"
await update_job(job_id, status=status, current_item="")
logger.info("job %s finished: %d ok, %d failed", job_id, done, failed)

229
api/lib/processing.py Normal file
View File

@@ -0,0 +1,229 @@
"""Ядро пайплайна: одно видео от URL до записи в БД.
Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs.
Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из
настроек в БД (таблица settings) с фолбэком на переменные окружения.
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from typing import Callable, Optional
from api.config import settings
from api.db.setting import get_all_settings
from api.db.video import create_video
from api.lib.llm import LLMConfig
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
logger = logging.getLogger("processing")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class PipelineError(RuntimeError):
def __init__(self, message: str, status: int = 500):
super().__init__(message)
self.status = status
async def effective_defaults() -> dict:
"""Настройки из БД поверх переменных окружения."""
db = await get_all_settings()
base_url = db.get("llm_base_url") or settings.LLM_BASE_URL
model = db.get("llm_model") or settings.LLM_MODEL
llm_cfg = None
if base_url and model:
llm_cfg = LLMConfig(
base_url=base_url,
api_key=db.get("llm_api_key") or settings.LLM_API_KEY,
model=model,
timeout=settings.LLM_TIMEOUT,
max_tokens=settings.LLM_MAX_TOKENS,
temperature=settings.LLM_TEMPERATURE,
)
try:
max_sentences = int(db.get("summary_max_sentences") or 15)
except ValueError:
max_sentences = 15
return {
"llm_cfg": llm_cfg,
"summary_max_sentences": max_sentences,
"keep_video": db.get("keep_video", "1") != "0",
"model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL,
}
def resolve_model_path(raw: str) -> Path:
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise PipelineError(
"model_path must point inside the project's models directory", status=400
)
if not p.exists():
raise PipelineError(f"Model not found: {p}", status=400)
return p
async def process_single_video(
url: str,
*,
model_path: str | None = None,
summary_max_sentences: int | None = None,
keep_video: bool | None = None,
section_id: int | None = None,
job_id: str | None = None,
llm_cfg: Optional[LLMConfig] = None,
system_prompt: str | None = None,
user_template: str | None = None,
progress: Optional[Callable[[str], None]] = None,
) -> dict:
"""Скачивает, расшифровывает, сокращает и сохраняет одно видео.
Параметры со значением None подтягиваются из настроек (БД/env).
progress(msg) — живой статус для UI задач.
"""
notify = progress or (lambda msg: None)
defaults = await effective_defaults()
if llm_cfg is None:
llm_cfg = defaults["llm_cfg"]
if summary_max_sentences is None:
summary_max_sentences = defaults["summary_max_sentences"]
if keep_video is None:
keep_video = defaults["keep_video"]
raw_model_path = model_path or defaults["model_path"]
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video)
notify("скачивание…")
try:
info = await loop.run_in_executor(
None,
lambda: download_video_with_subs(
url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
# видео не сохраняем — достаточно аудио (для лекций в разы быстрее)
audio_only=not keep_video,
on_progress=lambda pct: notify(f"скачивание {pct}%"),
),
)
except Exception as e:
raise PipelineError(f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise PipelineError("Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
notify("распознавание речи (Vosk)…")
resolved_model = resolve_model_path(raw_model_path)
try:
full_text = await loop.run_in_executor(
None,
lambda: transcribe_via_ffmpeg(
video_path,
resolved_model,
on_progress=lambda m: notify(f"распознано {m} мин аудио"),
),
)
method = f"vosk:{resolved_model.name}"
except Exception as e:
raise PipelineError(f"Transcription failed: {e}")
if not full_text.strip():
raise PipelineError("No subtitles and transcription returned empty text")
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
notify("выжимка (LLM)…" if llm_cfg else "выжимка…")
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=summary_max_sentences,
),
)
except Exception as e:
raise PipelineError(f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
section_id=section_id,
job_id=job_id,
)
except Exception as e:
raise PipelineError(f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return {
"uuid": video_uuid,
"title": title,
"source_url": url,
"video_path": rel_video,
"text_full_path": rel_full,
"text_summary_path": rel_summary,
"transcription_method": method,
"summary_preview": summary[:300],
}

14
api/lib/summarize.py
View File

@@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = (
DEFAULT_USER_TEMPLATE = (
"Сделай связную краткую выжимку следующего текста. "
"Выдели основные тезисы и логику изложения. Объём — 5–12 предложений.\n\n"
"Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n"
"---\n{text}\n---"
)
@@ -154,6 +154,7 @@ def summarize_llm(
user_template: Optional[str] = None,
reduce_template: Optional[str] = None,
chunk_chars: int = 10000,
max_sentences: int = 15,
) -> str:
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
text = (text or "").strip()
@@ -164,12 +165,19 @@ def summarize_llm(
user_t = user_template or DEFAULT_USER_TEMPLATE
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
def _fmt(template: str, chunk: str) -> str:
try:
return template.format(text=chunk, n=max_sentences)
except (KeyError, IndexError):
# пользовательский шаблон без {n} / с лишними скобками
return template.replace("{text}", chunk)
def _one(chunk: str, template: str) -> str:
return chat_complete(
cfg,
[
{"role": "system", "content": sys_p},
{"role": "user", "content": template.format(text=chunk)},
{"role": "user", "content": _fmt(template, chunk)},
],
)
@@ -194,6 +202,7 @@ def summarize_llm(
user_template=reduce_t,
reduce_template=reduce_t,
chunk_chars=chunk_chars,
max_sentences=max_sentences,
)
logger.info("LLM reduce step over %d partials", len(partials))
return _one(combined, reduce_t)
@@ -215,6 +224,7 @@ def summarize(
llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
max_sentences=max_sentences,
)
except LLMError as e:
logger.warning("LLM summarization failed, falling back to extractive: %s", e)

26
api/lib/transcribe.py
View File

@@ -53,8 +53,16 @@ class VoskModelManager:
t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
def transcribe_via_ffmpeg(
media_path: Path,
model_path: Path,
sample_rate: int = 16000,
on_progress=None,
) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.
on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио.
"""
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None:
@@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int =
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024)
if mb >= last_logged_mb + 5:
last_logged_mb = mb
logger.info("Transcribed %d MB of audio so far", mb)
# PCM s16le mono: sample_rate * 2 байта в секунду
minutes = bytes_read // (sample_rate * 2 * 60)
if minutes >= last_logged_mb + 5:
last_logged_mb = minutes
logger.info("Transcribed %d minutes of audio so far", minutes)
if on_progress is not None:
try:
on_progress(minutes)
except Exception:
pass
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")

72
api/lib/youtube.py
View File

@@ -12,7 +12,44 @@ import yt_dlp
logger = logging.getLogger("youtube")
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"}
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"}
def expand_url(url: str) -> dict:
"""Определяет, видео это или плейлист, и возвращает список элементов.
Returns: {"kind": "video"|"playlist", "title": str,
"entries": [{"url": str, "title": str}, ...]}
"""
opts = {
"extract_flat": "in_playlist",
"quiet": True,
"no_warnings": True,
"skip_download": True,
}
with yt_dlp.YoutubeDL(opts) as ydl:
info = ydl.extract_info(url, download=False)
if info.get("_type") == "playlist":
entries = []
for e in info.get("entries") or []:
if not e:
continue
u = e.get("url") or ""
if u and not u.startswith("http"):
u = f"https://www.youtube.com/watch?v={u}"
if not u and e.get("id"):
u = f"https://www.youtube.com/watch?v={e['id']}"
if u:
entries.append({"url": u, "title": e.get("title") or u})
return {
"kind": "playlist",
"title": info.get("title") or url,
"entries": entries,
}
title = info.get("title") or url
return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]}
def download_video_with_subs(
@@ -20,9 +57,15 @@ def download_video_with_subs(
video_id: str,
video_dir: Path,
langs: Optional[list[str]] = None,
audio_only: bool = False,
on_progress=None,
) -> dict:
"""Download a single video, also requesting subtitles for the given languages.
audio_only: качает только аудиодорожку (для длинных лекций, когда видео
не сохраняется — в разы быстрее и меньше по объёму).
on_progress: callback(percent: int), вызывается с шагом ~5%.
Returns:
{
"title": str,
@@ -40,11 +83,34 @@ def download_video_with_subs(
base_opts = {
"outtmpl": outtmpl,
"format": "bv*+ba/b",
"merge_output_format": "mp4",
"quiet": True,
"no_warnings": True,
"noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист
}
if audio_only:
base_opts["format"] = "ba/b"
else:
base_opts["format"] = "bv*+ba/b"
base_opts["merge_output_format"] = "mp4"
if on_progress is not None:
state = {"last": -5}
def _hook(dd: dict):
if dd.get("status") != "downloading":
return
total = dd.get("total_bytes") or dd.get("total_bytes_estimate")
if not total:
return
pct = int(dd.get("downloaded_bytes", 0) * 100 / total)
if pct >= state["last"] + 5:
state["last"] = pct
try:
on_progress(pct)
except Exception:
pass
base_opts["progress_hooks"] = [_hook]
sub_opts = {
**base_opts,
"writesubtitles": True,

12
api/main.py
View File

@@ -1,10 +1,13 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from api.db.connection import wait_for_postgres, init_db
from api.route import default, llm, pipeline, sections, videos
from api.db.job import mark_interrupted
from api.lib import jobqueue
from api.route import app_settings, default, jobs, llm, pipeline, sections, videos
logging.basicConfig(
@@ -17,13 +20,18 @@ logging.basicConfig(
async def lifespan(app: FastAPI):
await wait_for_postgres()
await init_db()
await mark_interrupted()
worker_task = asyncio.create_task(jobqueue.worker())
yield
worker_task.cancel()
app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan)
app = FastAPI(title="LLM-infa API", version="0.4.0", lifespan=lifespan)
app.include_router(default.router)
app.include_router(pipeline.router)
app.include_router(jobs.router)
app.include_router(videos.router)
app.include_router(sections.router)
app.include_router(app_settings.router)
app.include_router(llm.router)

59
api/route/app_settings.py Normal file
View File

@@ -0,0 +1,59 @@
"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres,
переживают перезапуск контейнеров и перезагрузку страницы.
"""
from __future__ import annotations
from fastapi import APIRouter
from pydantic import BaseModel
from api.db.setting import get_all_settings, set_settings
router = APIRouter(prefix="/settings", tags=["settings"])
_KEYS = (
"llm_base_url",
"llm_api_key",
"llm_model",
"summary_max_sentences",
"keep_video",
"vosk_model_path",
)
class SettingsPayload(BaseModel):
llm_base_url: str | None = None
llm_api_key: str | None = None
llm_model: str | None = None
summary_max_sentences: int | None = None
keep_video: bool | None = None
vosk_model_path: str | None = None
@router.get("/")
async def get_settings():
db = await get_all_settings()
return {
"llm_base_url": db.get("llm_base_url", ""),
"llm_api_key": db.get("llm_api_key", ""),
"llm_model": db.get("llm_model", ""),
"summary_max_sentences": int(db.get("summary_max_sentences") or 15),
"keep_video": db.get("keep_video", "1") != "0",
"vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"),
}
@router.put("/")
async def put_settings(body: SettingsPayload):
values: dict[str, str] = {}
for key in _KEYS:
v = getattr(body, key)
if v is None:
continue
if key == "keep_video":
values[key] = "1" if v else "0"
else:
values[key] = str(v)
if values:
await set_settings(values)
return await get_settings()

112
api/route/jobs.py Normal file
View File

@@ -0,0 +1,112 @@
"""Фоновые задачи: видео или плейлист любой длины.
POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs.
"""
from __future__ import annotations
import uuid as _uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.db.job import create_job, delete_job, get_job, list_jobs
from api.lib import jobqueue
router = APIRouter(prefix="/jobs", tags=["jobs"])
class JobIn(BaseModel):
url: str
section_id: int | None = None
# None → берутся из сохранённых настроек
keep_video: bool | None = None
summary_max_sentences: int | None = None
model_path: str | None = None
class JobOut(BaseModel):
id: str
url: str
kind: str
title: str
status: str
total_items: int
done_items: int
failed_items: int
current_item: str
live: str
error: str
created_at: datetime
def _to_out(j) -> JobOut:
return JobOut(
id=j.id,
url=j.url,
kind=j.kind,
title=j.title,
status=j.status,
total_items=j.total_items,
done_items=j.done_items,
failed_items=j.failed_items,
current_item=j.current_item,
live=jobqueue.live_status(j.id) or j.current_item,
error=j.error,
created_at=j.created_at,
)
@router.post("/", response_model=JobOut)
async def create(body: JobIn):
url = body.url.strip()
if not url:
raise HTTPException(status_code=400, detail="URL is empty")
job_id = _uuid.uuid4().hex
j = await create_job(job_id, url)
jobqueue.submit(
job_id,
{
"section_id": body.section_id,
"keep_video": body.keep_video,
"summary_max_sentences": body.summary_max_sentences,
"model_path": body.model_path,
},
)
return _to_out(j)
@router.get("/", response_model=list[JobOut])
async def list_all(limit: int = 50):
return [_to_out(j) for j in await list_jobs(limit)]
@router.get("/{job_id}", response_model=JobOut)
async def get_one(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
return _to_out(j)
@router.post("/{job_id}/cancel")
async def cancel(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status not in ("queued", "running"):
raise HTTPException(status_code=400, detail=f"Job is {j.status}")
jobqueue.request_cancel(job_id)
return {"ok": True, "note": "остановится после текущего элемента"}
@router.delete("/{job_id}")
async def delete(job_id: str):
j = await get_job(job_id)
if j is None:
raise HTTPException(status_code=404, detail="Job not found")
if j.status in ("queued", "running"):
raise HTTPException(status_code=400, detail="Сначала отмените задачу")
await delete_job(job_id)
return {"ok": True}

177
api/route/pipeline.py
View File

@@ -1,29 +1,16 @@
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist."""
"""Синхронный запуск пайплайна для одного видео (для длинных видео и
плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым).
"""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.config import settings
from api.db.video import create_video
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
from api.lib.llm import LLMConfig
from api.lib.processing import PipelineError, process_single_video
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
logger = logging.getLogger("pipeline")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class LLMOverride(BaseModel):
@@ -40,12 +27,12 @@ class LLMOverride(BaseModel):
class ProcessRequest(BaseModel):
url: str
model_path: str | None = None # path inside models/, used when subtitles are unavailable
summary_max_sentences: int = 15
keep_video: bool = True # False: файл видео удаляется после расшифровки
model_path: str | None = None # None → из настроек
summary_max_sentences: int | None = None
keep_video: bool | None = None
section_id: int | None = None
# If supplied, this LLM is used for the summary; otherwise env-configured LLM;
# otherwise the built-in extractive fallback.
# Если задано — используется этот LLM; иначе сохранённые настройки/env;
# иначе встроенная экстрактивная выжимка.
llm: LLMOverride | None = None
@@ -60,86 +47,10 @@ class ProcessResponse(BaseModel):
summary_preview: str
def _resolve_model_path(req_model_path: str | None) -> Path:
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise HTTPException(
status_code=400,
detail="model_path must point inside the project's models directory",
)
if not p.exists():
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
return p
@router.post("/process", response_model=ProcessResponse)
async def process_video(req: ProcessRequest):
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s", video_uuid, req.url)
try:
info = await loop.run_in_executor(
None,
download_video_with_subs,
req.url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise HTTPException(status_code=500, detail="Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
model_path = _resolve_model_path(req.model_path)
try:
full_text = await loop.run_in_executor(
None, transcribe_via_ffmpeg, video_path, model_path
)
method = f"vosk:{model_path.name}"
except Exception as e:
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
if not full_text.strip():
raise HTTPException(
status_code=500, detail="No subtitles and transcription returned empty text"
)
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
llm_cfg = None
sys_p = user_t = None
if req.llm is not None:
llm_cfg = LLMConfig(
base_url=req.llm.base_url,
@@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest):
)
sys_p = req.llm.system_prompt
user_t = req.llm.user_template
else:
llm_cfg = llm_from_settings()
sys_p = None
user_t = None
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
max_sentences=req.summary_max_sentences,
),
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
if req.keep_video:
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
else:
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
video_path.unlink(missing_ok=True)
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
p.unlink(missing_ok=True)
rel_video = ""
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=req.url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
result = await process_single_video(
req.url,
model_path=req.model_path,
summary_max_sentences=req.summary_max_sentences,
keep_video=req.keep_video,
section_id=req.section_id,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return ProcessResponse(
uuid=video_uuid,
title=title,
source_url=req.url,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
summary_preview=summary[:300],
)
except PipelineError as e:
raise HTTPException(status_code=e.status, detail=str(e))
return ProcessResponse(**result)

2
pyproject.toml
View File

@@ -1,6 +1,6 @@
[project]
name = "llm-infa"
version = "0.3.0"
version = "0.4.0"
description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres"
readme = "README.md"
requires-python = ">=3.14"

28
web/api_client.py
View File

@@ -21,10 +21,15 @@ class ApiClient:
def close(self) -> None:
self._client.close()
# --- LLM ---
# --- Settings (persisted server-side in Postgres) ---
def llm_status(self) -> dict:
return self._get("/llm/config")
def get_settings(self) -> dict:
return self._get("/settings/")
def save_settings(self, values: dict) -> dict:
return self._request("PUT", "/settings/", values)
# --- LLM ---
def llm_models(self, base_url: str, api_key: str) -> list[dict]:
data = self._post(
@@ -39,10 +44,21 @@ class ApiClient:
{"base_url": base_url, "api_key": api_key, "model": model},
)
# --- Pipeline / videos ---
# --- Jobs (видео и плейлисты, в фоне) ---
def pipeline_process(self, payload: dict) -> dict:
return self._post("/pipeline/process", payload)
def create_job(self, payload: dict) -> dict:
return self._post("/jobs/", payload)
def list_jobs(self) -> list[dict]:
return self._get("/jobs/")
def cancel_job(self, job_id: str) -> dict:
return self._post(f"/jobs/{job_id}/cancel", None)
def delete_job(self, job_id: str) -> dict:
return self._request("DELETE", f"/jobs/{job_id}")
# --- Videos ---
def list_videos(
self,

12
web/main.py
View File

@@ -19,20 +19,14 @@ def main(page: ft.Page) -> None:
view = MainView(page, client)
page.add(view.build())
# probe env-configured LLM and initial library load
# настройки живут на сервере (БД) — заполняем поля сохранёнными значениями
try:
status = client.llm_status()
if status.get("configured"):
view._set_status(ok=True)
if status.get("base_url") and not view.base_url.value:
view.base_url.value = status["base_url"]
if status.get("model") and not view.selected_model:
view.selected_model = status["model"]
view.model_input.value = status["model"]
view.apply_settings(client.get_settings())
except Exception:
pass
view._refresh_library()
view._start_jobs_poll()
page.update()

381
web/views/main_view.py
View File

@@ -1,7 +1,8 @@
"""Main page of the LLM-infa Flet web app."""
"""Main page of the LLM-infa Flet web app: вкладки «Главная» и «Настройки»."""
from __future__ import annotations
import threading
import time
import flet as ft
@@ -9,6 +10,16 @@ from web.api_client import ApiClient, ApiError
import web.designer as d
_JOB_STATUS = {
"queued": ("в очереди", None),
"running": ("выполняется", None),
"done": ("готово", "ok"),
"error": ("ошибка", "danger"),
"cancelled": ("отменена", None),
"interrupted": ("прервана", "danger"),
}
class MainView:
def __init__(self, page: ft.Page, client: ApiClient):
self.page = page
@@ -20,14 +31,15 @@ class MainView:
self.sections: list[dict] = []
self.current_section_id: int | None = None # None = все видео
# ── LLM config fields ──────────────────────────────────────────────
self._polling = False
# ── Settings tab fields ────────────────────────────────────────────
self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True)
self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True)
# Searchable model combobox
self.model_input = d.text_field(
"Модель",
hint="Сначала нажмите «Загрузить модели»",
hint="Нажмите «Загрузить модели» и выберите из списка",
on_change=self._on_model_search,
expand=True,
)
@@ -44,7 +56,7 @@ class MainView:
self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.llm_status_chip = ft.Container(
content=ft.Text("LLM не подключена", size=12, color=d.MUTED),
content=ft.Text("LLM не настроена", size=12, color=d.MUTED),
padding=ft.padding.symmetric(horizontal=10, vertical=4),
border=ft.border.all(1, d.BORDER),
border_radius=99,
@@ -52,12 +64,6 @@ class MainView:
self.btn_load = d.primary_button("Загрузить модели", self._load_models)
self.btn_test = d.secondary_button("Проверить связь", self._test_conn)
# ── Pipeline fields ────────────────────────────────────────────────
self.yt_url = d.text_field(
"Ссылка на YouTube",
hint="https://youtu.be/…",
expand=True,
)
self.vosk_path = d.text_field(
"Vosk-модель (fallback, когда нет субтитров)",
value="models/vosk-model-small-ru-0.22",
@@ -76,19 +82,28 @@ class MainView:
width=220,
)
self.keep_video_sw = ft.Switch(
label="Сохранять видео на диске",
label="Сохранять видео на диске (иначе качается только аудио)",
value=True,
active_color=d.ACCENT,
label_style=ft.TextStyle(color=d.TEXT, size=13),
)
self.pipeline_section_dd = self._dropdown("Раздел для нового видео", width=280)
self.settings_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.btn_save = d.primary_button("💾 Сохранить настройки", self._save_settings)
# ── Pipeline (main tab) ────────────────────────────────────────────
self.yt_url = d.text_field(
"Ссылка на YouTube: видео или плейлист",
hint="https://youtu.be/… или https://www.youtube.com/playlist?list=…",
expand=True,
)
self.pipeline_section_dd = self._dropdown("Раздел для новых видео", width=280)
self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
self.btn_run = d.primary_button("▶ Запустить", self._run_pipeline)
self.btn_run = d.primary_button("▶ В очередь", self._run_pipeline)
# Result block (initially hidden)
self._result_col = ft.Column(spacing=6, visible=False)
# ── Jobs ───────────────────────────────────────────────────────────
self._jobs_col = ft.Column(spacing=6)
# ── Library controls ───────────────────────────────────────────────
# ── Library ────────────────────────────────────────────────────────
self.search_field = d.text_field(
"Поиск по названию / URL", on_change=None, expand=True
)
@@ -124,24 +139,47 @@ class MainView:
# ─────────────────────── build ───────────────────────────────────────
def build(self) -> ft.Control:
return ft.Column(
controls=[
self._build_topbar(),
ft.Container(
content=ft.Column(
[
self._build_llm_card(),
self._build_pipeline_card(),
self._build_library_card(),
],
spacing=20,
scroll=ft.ScrollMode.AUTO,
),
padding=ft.padding.symmetric(horizontal=24, vertical=20),
expand=True,
),
main_tab = ft.Container(
content=ft.Column(
[
self._build_pipeline_card(),
self._build_jobs_card(),
self._build_library_card(),
],
spacing=20,
scroll=ft.ScrollMode.AUTO,
),
padding=ft.padding.symmetric(horizontal=24, vertical=20),
expand=True,
)
settings_tab = ft.Container(
content=ft.Column(
[
self._build_llm_card(),
self._build_defaults_card(),
],
spacing=20,
scroll=ft.ScrollMode.AUTO,
),
padding=ft.padding.symmetric(horizontal=24, vertical=20),
expand=True,
)
tabs = ft.Tabs(
selected_index=0,
animation_duration=200,
label_color=d.ACCENT_2,
unselected_label_color=d.MUTED,
indicator_color=d.ACCENT,
divider_color=d.BORDER,
tabs=[
ft.Tab(text="Главная", content=main_tab),
ft.Tab(text="Настройки", content=settings_tab),
],
expand=True,
)
return ft.Column(
controls=[self._build_topbar(), tabs],
expand=True,
spacing=0,
)
@@ -174,13 +212,14 @@ class MainView:
border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)),
)
# ── Settings tab cards ────────────────────────────────────────────────
def _build_llm_card(self) -> ft.Container:
return d.card(
d.section_title("Подключение к LLM"),
d.hint(
"Укажите любой OpenAI-совместимый провайдер. "
"Нейронка внутри Docker не разворачивается — "
"указываем URL внешнего сервиса."
"Любой OpenAI-совместимый провайдер (OpenRouter, OpenAI, локальная Ollama…). "
"Ключ и модель хранятся на сервере в БД и переживают перезапуск."
),
ft.Row([self.base_url, self.api_key], spacing=12),
ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10),
@@ -189,22 +228,43 @@ class MainView:
self._model_list_wrap,
)
def _build_defaults_card(self) -> ft.Container:
return d.card(
d.section_title("Параметры обработки"),
d.hint("Применяются ко всем новым задачам."),
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
self.keep_video_sw,
ft.Divider(color=d.BORDER, height=1),
ft.Row([self.btn_save, self.settings_msg], spacing=10),
)
# ── Main tab cards ────────────────────────────────────────────────────
def _build_pipeline_card(self) -> ft.Container:
return d.card(
d.section_title("Обработать видео"),
d.section_title("Обработать видео или плейлист"),
d.hint(
"Pipeline: yt-dlp → субтитры (или Vosk fallback) "
"→ выжимка через выбранную LLM → запись в БД."
"Задача уходит в фон: качаем, расшифровываем (субтитры или Vosk), "
"сокращаем через LLM, пишем в БД. Плейлисты и лекции на 4–5 часов — ок."
),
self.yt_url,
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
ft.Row(
[self.pipeline_section_dd, self.keep_video_sw],
spacing=18,
[self.btn_run, self.pipeline_section_dd, self.pipeline_msg],
spacing=12,
vertical_alignment=ft.CrossAxisAlignment.CENTER,
),
ft.Row([self.btn_run, self.pipeline_msg], spacing=10),
self._result_col,
)
def _build_jobs_card(self) -> ft.Container:
return d.card(
ft.Row(
[
d.section_title("Задачи"),
ft.Container(expand=True),
d.secondary_button("Обновить", lambda _: self._start_jobs_poll()),
]
),
self._jobs_col,
)
def _build_library_card(self) -> ft.Container:
@@ -256,6 +316,39 @@ class MainView:
),
)
# ─────────────────── settings load/save ──────────────────────────────
def apply_settings(self, s: dict) -> None:
"""Заполняет поля значениями с сервера (вызывается при старте сессии)."""
self.base_url.value = s.get("llm_base_url", "")
self.api_key.value = s.get("llm_api_key", "")
self.selected_model = s.get("llm_model", "")
self.model_input.value = self.selected_model
self.n_sentences.value = str(s.get("summary_max_sentences", 15))
self.keep_video_sw.value = bool(s.get("keep_video", True))
self.vosk_path.value = s.get("vosk_model_path", "models/vosk-model-small-ru-0.22")
self._set_status(ok=bool(self.base_url.value and self.selected_model))
def _save_settings(self, _):
model = self.selected_model or (self.model_input.value or "").strip()
payload = {
"llm_base_url": self.base_url.value.strip(),
"llm_api_key": self.api_key.value.strip(),
"llm_model": model,
"summary_max_sentences": int(self.n_sentences.value or 15),
"keep_video": bool(self.keep_video_sw.value),
"vosk_model_path": self.vosk_path.value.strip(),
}
try:
self.client.save_settings(payload)
self.settings_msg.value = "Сохранено ✓ (хранится в БД, переживёт перезапуск)"
self.settings_msg.color = d.OK
self._set_status(ok=bool(payload["llm_base_url"] and model))
except ApiError as e:
self.settings_msg.value = str(e)
self.settings_msg.color = d.DANGER
self.page.update()
# ─────────────────── model combobox ──────────────────────────────────
def _on_model_search(self, e):
@@ -292,6 +385,8 @@ class MainView:
self.selected_model = mid
self.model_input.value = mid
self._model_list_wrap.visible = False
self.llm_msg.value = "Не забудьте нажать «Сохранить настройки»"
self.llm_msg.color = d.MUTED
self.page.update()
# ─────────────────── LLM actions ─────────────────────────────────────
@@ -311,10 +406,8 @@ class MainView:
self.models = self.client.llm_models(base_url, self.api_key.value.strip())
self._render_model_list(self.models)
self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True)
self._set_status(ok=True)
except ApiError as e:
self._set_llm_msg(str(e), error=True)
self._set_status(ok=False)
finally:
self.btn_load.disabled = False
self.page.update()
@@ -322,7 +415,8 @@ class MainView:
threading.Thread(target=_task, daemon=True).start()
def _test_conn(self, _):
if not self.selected_model:
model = self.selected_model or (self.model_input.value or "").strip()
if not model:
self._set_llm_msg("Сначала выберите модель", error=True)
return
@@ -335,87 +429,164 @@ class MainView:
r = self.client.llm_test(
self.base_url.value.strip(),
self.api_key.value.strip(),
self.selected_model,
model,
)
preview = (r.get("reply") or "")[:80]
self._set_llm_msg(f"OK · {preview}", ok=True)
self._set_status(ok=True)
except ApiError as e:
self._set_llm_msg(str(e), error=True)
self._set_status(ok=False)
finally:
self.btn_test.disabled = False
self.page.update()
threading.Thread(target=_task, daemon=True).start()
# ─────────────────── pipeline ─────────────────────────────────────────
# ─────────────────── pipeline → jobs ──────────────────────────────────
def _run_pipeline(self, _):
url = self.yt_url.value.strip()
if not url:
self._set_pipeline_msg("Укажите ссылку на видео", error=True)
self._set_pipeline_msg("Укажите ссылку на видео или плейлист", error=True)
return
payload: dict = {
"url": url,
"model_path": self.vosk_path.value.strip() or None,
"summary_max_sentences": int(self.n_sentences.value or 15),
"keep_video": bool(self.keep_video_sw.value),
}
payload: dict = {"url": url}
if self.pipeline_section_dd.value:
payload["section_id"] = int(self.pipeline_section_dd.value)
if self.base_url.value.strip() and self.selected_model:
payload["llm"] = {
"base_url": self.base_url.value.strip(),
"api_key": self.api_key.value.strip(),
"model": self.selected_model,
}
self._set_pipeline_msg("Обработка видео… (может занять несколько минут)")
self._result_col.visible = False
self.btn_run.disabled = True
try:
self.client.create_job(payload)
self.yt_url.value = ""
self._set_pipeline_msg("Задача поставлена в очередь ✓", ok=True)
except ApiError as e:
self._set_pipeline_msg(str(e), error=True)
self.page.update()
self._start_jobs_poll()
def _task():
try:
r = self.client.pipeline_process(payload)
self._render_result(r)
self._set_pipeline_msg("Готово ✓", ok=True)
self._refresh_library()
except ApiError as e:
self._set_pipeline_msg(str(e), error=True)
finally:
self.btn_run.disabled = False
self.page.update()
# ─────────────────── jobs ─────────────────────────────────────────────
threading.Thread(target=_task, daemon=True).start()
def _start_jobs_poll(self):
if self._polling:
return
self._polling = True
threading.Thread(target=self._poll_loop, daemon=True).start()
def _render_result(self, r: dict):
self._result_col.controls = [
ft.Divider(color=d.BORDER, height=1),
ft.Text(r.get("title") or r.get("uuid", ""), color=d.ACCENT_2, size=15, weight=ft.FontWeight.W_600),
*[
self._kv(k, v)
for k, v in [
("UUID", r.get("uuid", "")),
("Источник", r.get("source_url", "")),
("Метод", r.get("transcription_method", "")),
("Видео", r.get("video_path") or "не сохранено"),
("Полный текст", r.get("text_full_path", "")),
("Выжимка", r.get("text_summary_path", "")),
]
],
ft.Text("Превью выжимки:", color=d.MUTED, size=12),
ft.Container(
content=ft.Text(r.get("summary_preview", ""), color=d.TEXT, size=13, selectable=True, no_wrap=False),
padding=12,
bgcolor=d.SURFACE_3,
border=ft.border.all(1, d.BORDER),
border_radius=8,
def _poll_loop(self):
prev: dict = {}
try:
while True:
try:
jobs = self.client.list_jobs()
except (ApiError, Exception):
break
self._render_jobs(jobs)
state = {j["id"]: (j["done_items"], j["status"]) for j in jobs}
if state != prev:
prev = state
self._refresh_library() # там же page.update()
else:
self.page.update()
if not any(j["status"] in ("queued", "running") for j in jobs):
break
time.sleep(3)
finally:
self._polling = False
def _render_jobs(self, jobs: list[dict]):
if not jobs:
self._jobs_col.controls = [
ft.Text("Задач ещё не было", color=d.MUTED, size=13)
]
return
self._jobs_col.controls = [self._job_row(j) for j in jobs]
def _job_row(self, j: dict) -> ft.Container:
label, tone = _JOB_STATUS.get(j["status"], (j["status"], None))
color = d.OK if tone == "ok" else (d.DANGER if tone == "danger" else d.MUTED)
active = j["status"] in ("queued", "running")
chip = ft.Container(
content=ft.Text(label, size=11, color=color),
padding=ft.padding.symmetric(horizontal=8, vertical=2),
border=ft.border.all(1, color),
border_radius=99,
)
head = [
chip,
ft.Text(
(j["title"] or j["url"])[:90],
color=d.TEXT,
size=13,
weight=ft.FontWeight.W_600,
expand=True,
no_wrap=True,
),
]
self._result_col.visible = True
if j["kind"] == "playlist":
head.append(ft.Text("плейлист", color=d.ACCENT_2, size=11))
if j["total_items"]:
head.append(
ft.Text(f"{j['done_items']}/{j['total_items']}", color=d.MUTED, size=12)
)
if active:
head.append(
ft.IconButton(
icon=ft.Icons.STOP_CIRCLE_OUTLINED,
icon_color=d.DANGER,
icon_size=17,
tooltip="Отменить (после текущего элемента)",
on_click=lambda e, jid=j["id"]: self._cancel_job(jid),
)
)
else:
head.append(
ft.IconButton(
icon=ft.Icons.CLOSE,
icon_color=d.MUTED,
icon_size=15,
tooltip="Убрать из списка",
on_click=lambda e, jid=j["id"]: self._delete_job(jid),
)
)
rows: list[ft.Control] = [ft.Row(head, spacing=8)]
if active:
total = j["total_items"] or 0
value = (j["done_items"] / total) if total else None
rows.append(
ft.ProgressBar(value=value, color=d.ACCENT, bgcolor=d.SURFACE_3, height=4)
)
if j.get("live"):
rows.append(ft.Text(j["live"][:140], color=d.ACCENT_2, size=11))
if j.get("error"):
rows.append(
ft.Text(j["error"][:300], color=d.DANGER, size=11, no_wrap=False)
)
if j["failed_items"]:
rows.append(
ft.Text(f"с ошибкой: {j['failed_items']}", color=d.DANGER, size=11)
)
return ft.Container(
content=ft.Column(rows, spacing=5, tight=True),
padding=10,
bgcolor=d.SURFACE_2,
border=ft.border.all(1, d.BORDER),
border_radius=10,
)
def _cancel_job(self, job_id: str):
try:
self.client.cancel_job(job_id)
except ApiError as e:
self.lib_msg.value = str(e)
self._start_jobs_poll()
def _delete_job(self, job_id: str):
try:
self.client.delete_job(job_id)
except ApiError as e:
self.lib_msg.value = str(e)
self._start_jobs_poll()
# ─────────────────── sections ─────────────────────────────────────────
@@ -688,6 +859,6 @@ class MainView:
def _set_status(self, *, ok: bool):
chip = self.llm_status_chip
chip.content.value = "LLM подключена" if ok else "Ошибка подключения"
chip.content.color = d.OK if ok else d.DANGER
chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else "rgba(227,90,114,.4)")
chip.content.value = "LLM настроена" if ok else "LLM не настроена"
chip.content.color = d.OK if ok else d.MUTED
chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else d.BORDER)