diff --git a/README.md b/README.md index 9c19025..e83a719 100644 --- a/README.md +++ b/README.md @@ -48,19 +48,38 @@ OpenAI-совместимый эндпоинт. Два способа: Если LLM не настроена, работает встроенная экстрактивная суммаризация (выбор ключевых предложений, без нейросети). +## Плейлисты и длинные лекции + +Обработка идёт через фоновые задачи: `POST /jobs` сразу возвращает id, элементы +плейлиста обрабатываются по очереди, прогресс виден в UI (скачивание %, +минуты распознанного аудио, номер видео в плейлисте). Если в настройках +выключено «Сохранять видео», для лекций качается только аудиодорожка — +в разы быстрее и легче. + +Настройки (ключ LLM, модель, длина выжимки, хранение видео) сохраняются +в Postgres — переживают перезапуск контейнеров и перезагрузку страницы. +Вкладка «Настройки» в UI. + ## API ```bash -# обработать видео целиком +# фоновая задача: видео или плейлист +curl -X POST localhost:8000/jobs/ \ + -H 'Content-Type: application/json' \ + -d '{"url": "https://www.youtube.com/playlist?list=..."}' +curl localhost:8000/jobs/ # статусы и прогресс + +# синхронно, одно видео (для коротких роликов) curl -X POST localhost:8000/pipeline/process \ -H 'Content-Type: application/json' \ -d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}' -# каталог обработанных видео +# каталог обработанных видео (фильтры: ?q=, ?section_id=, ?method=) curl localhost:8000/videos/ -# проверить связь с LLM -curl -X POST localhost:8000/llm/test +# настройки +curl -X PUT localhost:8000/settings/ -H 'Content-Type: application/json' \ + -d '{"llm_base_url": "https://openrouter.ai/api/v1", "llm_api_key": "sk-...", "llm_model": "..."}' ``` ## Заметки diff --git a/api/db/connection.py b/api/db/connection.py index 1895d26..1343561 100644 --- a/api/db/connection.py +++ b/api/db/connection.py @@ -8,7 +8,9 @@ from sqlalchemy.orm import sessionmaker from api.config import settings from api.models.base import Base +import api.models.job # noqa: F401 - register model with Base.metadata import api.models.section # noqa: F401 - register model with Base.metadata +import api.models.setting # noqa: F401 - register model with Base.metadata import api.models.video # noqa: F401 - register model with Base.metadata @@ -69,3 +71,6 @@ async def init_db() -> None: "REFERENCES sections(id) ON DELETE SET NULL" ) ) + await conn.execute( + text("ALTER TABLE videos ADD COLUMN IF NOT EXISTS job_id VARCHAR") + ) diff --git a/api/db/job.py b/api/db/job.py new file mode 100644 index 0000000..18d1a89 --- /dev/null +++ b/api/db/job.py @@ -0,0 +1,53 @@ +from sqlalchemy import select, update + +from api.db.connection import SessionLocal +from api.models.job import Job + + +async def create_job(job_id: str, url: str) -> Job: + async with SessionLocal() as session: + j = Job(id=job_id, url=url, status="queued") + session.add(j) + await session.commit() + await session.refresh(j) + return j + + +async def get_job(job_id: str) -> Job | None: + async with SessionLocal() as session: + return await session.get(Job, job_id) + + +async def list_jobs(limit: int = 50) -> list[Job]: + async with SessionLocal() as session: + res = await session.execute( + select(Job).order_by(Job.created_at.desc()).limit(limit) + ) + return list(res.scalars().all()) + + +async def update_job(job_id: str, **fields) -> None: + async with SessionLocal() as session: + await session.execute(update(Job).where(Job.id == job_id).values(**fields)) + await session.commit() + + +async def delete_job(job_id: str) -> bool: + async with SessionLocal() as session: + j = await session.get(Job, job_id) + if j is None: + return False + await session.delete(j) + await session.commit() + return True + + +async def mark_interrupted() -> None: + """На старте приложения: задачи, шедшие до перезапуска, отмечаем прерванными.""" + async with SessionLocal() as session: + await session.execute( + update(Job) + .where(Job.status.in_(("queued", "running"))) + .values(status="interrupted", current_item="") + ) + await session.commit() diff --git a/api/db/setting.py b/api/db/setting.py new file mode 100644 index 0000000..22ca3ff --- /dev/null +++ b/api/db/setting.py @@ -0,0 +1,21 @@ +from sqlalchemy import select + +from api.db.connection import SessionLocal +from api.models.setting import Setting + + +async def get_all_settings() -> dict[str, str]: + async with SessionLocal() as session: + res = await session.execute(select(Setting)) + return {s.key: s.value for s in res.scalars().all()} + + +async def set_settings(values: dict[str, str]) -> None: + async with SessionLocal() as session: + for key, value in values.items(): + existing = await session.get(Setting, key) + if existing is None: + session.add(Setting(key=key, value=str(value))) + else: + existing.value = str(value) + await session.commit() diff --git a/api/db/video.py b/api/db/video.py index d1b7264..34bc1a0 100644 --- a/api/db/video.py +++ b/api/db/video.py @@ -14,12 +14,14 @@ async def create_video( text_summary_path: str, transcription_method: str, section_id: int | None = None, + job_id: str | None = None, ) -> Video: async with SessionLocal() as session: v = Video( uuid=uuid, source_url=source_url, section_id=section_id, + job_id=job_id, title=title or "", video_path=video_path or "", text_full_path=text_full_path or "", diff --git a/api/lib/jobqueue.py b/api/lib/jobqueue.py new file mode 100644 index 0000000..4c75919 --- /dev/null +++ b/api/lib/jobqueue.py @@ -0,0 +1,119 @@ +"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются +последовательно (чтобы не долбить YouTube и LLM параллельными запросами). + +Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs; +устойчивое состояние (счётчики, статус) — в таблице jobs. +""" +from __future__ import annotations + +import asyncio +import logging + +from api.db.job import get_job, update_job +from api.lib.processing import process_single_video +from api.lib.youtube import expand_url + + +logger = logging.getLogger("jobs") + +_queue: asyncio.Queue[str] = asyncio.Queue() +_live: dict[str, str] = {} +_cancel: set[str] = set() +_params: dict[str, dict] = {} + + +def submit(job_id: str, params: dict) -> None: + _params[job_id] = params + _queue.put_nowait(job_id) + + +def live_status(job_id: str) -> str: + return _live.get(job_id, "") + + +def request_cancel(job_id: str) -> None: + _cancel.add(job_id) + + +def is_cancelled(job_id: str) -> bool: + return job_id in _cancel + + +async def worker() -> None: + """Единственный воркер; запускается на старте приложения.""" + logger.info("job worker started") + while True: + job_id = await _queue.get() + try: + await _run_job(job_id) + except Exception: + logger.exception("job %s crashed", job_id) + try: + await update_job(job_id, status="error", current_item="") + except Exception: + pass + finally: + _live.pop(job_id, None) + _cancel.discard(job_id) + _params.pop(job_id, None) + _queue.task_done() + + +async def _run_job(job_id: str) -> None: + job = await get_job(job_id) + if job is None or job.status == "cancelled": + return + params = _params.get(job_id, {}) + loop = asyncio.get_running_loop() + + await update_job(job_id, status="running", current_item="анализ ссылки…") + _live[job_id] = "анализ ссылки…" + + plan = await loop.run_in_executor(None, expand_url, job.url) + entries = plan["entries"] + if not entries: + await update_job(job_id, status="error", error="Плейлист пуст", current_item="") + return + await update_job( + job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries) + ) + + done = failed = 0 + errors: list[str] = [] + n = len(entries) + + for i, entry in enumerate(entries, start=1): + if is_cancelled(job_id): + await update_job(job_id, status="cancelled", current_item="") + logger.info("job %s cancelled at item %d/%d", job_id, i, n) + return + + label = f"{i}/{n} · {entry['title']}" + await update_job(job_id, current_item=label) + _live[job_id] = label + + def _notify(msg: str, _label: str = label) -> None: + _live[job_id] = f"{_label} — {msg}" + + try: + await process_single_video( + entry["url"], + section_id=params.get("section_id"), + model_path=params.get("model_path"), + summary_max_sentences=params.get("summary_max_sentences"), + keep_video=params.get("keep_video"), + job_id=job_id, + progress=_notify, + ) + done += 1 + except Exception as e: + failed += 1 + errors.append(f"{entry['title']}: {e}") + logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e) + await update_job( + job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000] + ) + + status = "error" if done == 0 and failed > 0 else "done" + await update_job(job_id, status=status, current_item="") + logger.info("job %s finished: %d ok, %d failed", job_id, done, failed) diff --git a/api/lib/processing.py b/api/lib/processing.py new file mode 100644 index 0000000..b951000 --- /dev/null +++ b/api/lib/processing.py @@ -0,0 +1,229 @@ +"""Ядро пайплайна: одно видео от URL до записи в БД. + +Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs. +Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из +настроек в БД (таблица settings) с фолбэком на переменные окружения. +""" +from __future__ import annotations + +import asyncio +import logging +import uuid as _uuid +from pathlib import Path +from typing import Callable, Optional + +from api.config import settings +from api.db.setting import get_all_settings +from api.db.video import create_video +from api.lib.llm import LLMConfig +from api.lib.summarize import summarize +from api.lib.transcribe import transcribe_via_ffmpeg +from api.lib.youtube import download_video_with_subs, vtt_to_text + + +logger = logging.getLogger("processing") + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos" +_TEXT_DIR = _PROJECT_ROOT / "data" / "text" +_MODELS_DIR = _PROJECT_ROOT / "models" + + +class PipelineError(RuntimeError): + def __init__(self, message: str, status: int = 500): + super().__init__(message) + self.status = status + + +async def effective_defaults() -> dict: + """Настройки из БД поверх переменных окружения.""" + db = await get_all_settings() + base_url = db.get("llm_base_url") or settings.LLM_BASE_URL + model = db.get("llm_model") or settings.LLM_MODEL + llm_cfg = None + if base_url and model: + llm_cfg = LLMConfig( + base_url=base_url, + api_key=db.get("llm_api_key") or settings.LLM_API_KEY, + model=model, + timeout=settings.LLM_TIMEOUT, + max_tokens=settings.LLM_MAX_TOKENS, + temperature=settings.LLM_TEMPERATURE, + ) + try: + max_sentences = int(db.get("summary_max_sentences") or 15) + except ValueError: + max_sentences = 15 + return { + "llm_cfg": llm_cfg, + "summary_max_sentences": max_sentences, + "keep_video": db.get("keep_video", "1") != "0", + "model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL, + } + + +def resolve_model_path(raw: str) -> Path: + p = Path(raw) + if not p.is_absolute(): + p = _PROJECT_ROOT / raw + p = p.resolve() + try: + p.relative_to(_MODELS_DIR.resolve()) + except ValueError: + raise PipelineError( + "model_path must point inside the project's models directory", status=400 + ) + if not p.exists(): + raise PipelineError(f"Model not found: {p}", status=400) + return p + + +async def process_single_video( + url: str, + *, + model_path: str | None = None, + summary_max_sentences: int | None = None, + keep_video: bool | None = None, + section_id: int | None = None, + job_id: str | None = None, + llm_cfg: Optional[LLMConfig] = None, + system_prompt: str | None = None, + user_template: str | None = None, + progress: Optional[Callable[[str], None]] = None, +) -> dict: + """Скачивает, расшифровывает, сокращает и сохраняет одно видео. + + Параметры со значением None подтягиваются из настроек (БД/env). + progress(msg) — живой статус для UI задач. + """ + notify = progress or (lambda msg: None) + defaults = await effective_defaults() + if llm_cfg is None: + llm_cfg = defaults["llm_cfg"] + if summary_max_sentences is None: + summary_max_sentences = defaults["summary_max_sentences"] + if keep_video is None: + keep_video = defaults["keep_video"] + raw_model_path = model_path or defaults["model_path"] + + video_uuid = _uuid.uuid4().hex + _VIDEO_DIR.mkdir(parents=True, exist_ok=True) + _TEXT_DIR.mkdir(parents=True, exist_ok=True) + loop = asyncio.get_running_loop() + + logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video) + notify("скачивание…") + try: + info = await loop.run_in_executor( + None, + lambda: download_video_with_subs( + url, + video_uuid, + _VIDEO_DIR, + settings.subtitle_langs_list, + # видео не сохраняем — достаточно аудио (для лекций в разы быстрее) + audio_only=not keep_video, + on_progress=lambda pct: notify(f"скачивание {pct}%"), + ), + ) + except Exception as e: + raise PipelineError(f"Download failed: {e}") + + title: str = info["title"] + video_path: Path = info["video_path"] + sub_path: Path | None = info["subtitle_path"] + + if not video_path.exists(): + raise PipelineError("Video file missing after download") + + full_text = "" + method = "" + + if sub_path is not None and sub_path.exists(): + try: + content = sub_path.read_text(encoding="utf-8", errors="ignore") + full_text = vtt_to_text(content) + if full_text.strip(): + method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}" + logger.info("[%s] using subtitles (%s)", video_uuid, method) + except Exception as e: + logger.warning("[%s] subtitle parse failed: %s", video_uuid, e) + full_text = "" + + if not full_text.strip(): + logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid) + notify("распознавание речи (Vosk)…") + resolved_model = resolve_model_path(raw_model_path) + try: + full_text = await loop.run_in_executor( + None, + lambda: transcribe_via_ffmpeg( + video_path, + resolved_model, + on_progress=lambda m: notify(f"распознано {m} мин аудио"), + ), + ) + method = f"vosk:{resolved_model.name}" + except Exception as e: + raise PipelineError(f"Transcription failed: {e}") + + if not full_text.strip(): + raise PipelineError("No subtitles and transcription returned empty text") + + text_full_path = _TEXT_DIR / f"{video_uuid}.txt" + text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt" + text_full_path.write_text(full_text, encoding="utf-8") + + notify("выжимка (LLM)…" if llm_cfg else "выжимка…") + try: + summary = await loop.run_in_executor( + None, + lambda: summarize( + full_text, + llm_cfg=llm_cfg, + system_prompt=system_prompt, + user_template=user_template, + max_sentences=summary_max_sentences, + ), + ) + except Exception as e: + raise PipelineError(f"Summarization failed: {e}") + text_summary_path.write_text(summary, encoding="utf-8") + + if keep_video: + rel_video = str(video_path.relative_to(_PROJECT_ROOT)) + else: + # пользователь просил не хранить видео: текст уже извлечён, файлы не нужны + video_path.unlink(missing_ok=True) + for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"): + p.unlink(missing_ok=True) + rel_video = "" + rel_full = str(text_full_path.relative_to(_PROJECT_ROOT)) + rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT)) + + try: + await create_video( + uuid=video_uuid, + source_url=url, + title=title, + video_path=rel_video, + text_full_path=rel_full, + text_summary_path=rel_summary, + transcription_method=method, + section_id=section_id, + job_id=job_id, + ) + except Exception as e: + raise PipelineError(f"DB insert failed: {e}") + + logger.info("[%s] done (method=%s)", video_uuid, method) + return { + "uuid": video_uuid, + "title": title, + "source_url": url, + "video_path": rel_video, + "text_full_path": rel_full, + "text_summary_path": rel_summary, + "transcription_method": method, + "summary_preview": summary[:300], + } diff --git a/api/lib/summarize.py b/api/lib/summarize.py index 2f481c2..72f882e 100644 --- a/api/lib/summarize.py +++ b/api/lib/summarize.py @@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = ( DEFAULT_USER_TEMPLATE = ( "Сделай связную краткую выжимку следующего текста. " - "Выдели основные тезисы и логику изложения. Объём — 5–12 предложений.\n\n" + "Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n" "---\n{text}\n---" ) @@ -154,6 +154,7 @@ def summarize_llm( user_template: Optional[str] = None, reduce_template: Optional[str] = None, chunk_chars: int = 10000, + max_sentences: int = 15, ) -> str: """Summarize via an external OpenAI-compatible API. Map-reduce for long texts.""" text = (text or "").strip() @@ -164,12 +165,19 @@ def summarize_llm( user_t = user_template or DEFAULT_USER_TEMPLATE reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE + def _fmt(template: str, chunk: str) -> str: + try: + return template.format(text=chunk, n=max_sentences) + except (KeyError, IndexError): + # пользовательский шаблон без {n} / с лишними скобками + return template.replace("{text}", chunk) + def _one(chunk: str, template: str) -> str: return chat_complete( cfg, [ {"role": "system", "content": sys_p}, - {"role": "user", "content": template.format(text=chunk)}, + {"role": "user", "content": _fmt(template, chunk)}, ], ) @@ -194,6 +202,7 @@ def summarize_llm( user_template=reduce_t, reduce_template=reduce_t, chunk_chars=chunk_chars, + max_sentences=max_sentences, ) logger.info("LLM reduce step over %d partials", len(partials)) return _one(combined, reduce_t) @@ -215,6 +224,7 @@ def summarize( llm_cfg, system_prompt=system_prompt, user_template=user_template, + max_sentences=max_sentences, ) except LLMError as e: logger.warning("LLM summarization failed, falling back to extractive: %s", e) diff --git a/api/lib/transcribe.py b/api/lib/transcribe.py index a7b170d..e3fa84d 100644 --- a/api/lib/transcribe.py +++ b/api/lib/transcribe.py @@ -53,8 +53,16 @@ class VoskModelManager: t.join(timeout=2) -def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str: - """Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.""" +def transcribe_via_ffmpeg( + media_path: Path, + model_path: Path, + sample_rate: int = 16000, + on_progress=None, +) -> str: + """Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks. + + on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио. + """ if Model is None or KaldiRecognizer is None: raise RuntimeError("vosk is not installed in runtime") if shutil.which("ffmpeg") is None: @@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = break recognizer.AcceptWaveform(chunk) bytes_read += len(chunk) - mb = bytes_read // (1024 * 1024) - if mb >= last_logged_mb + 5: - last_logged_mb = mb - logger.info("Transcribed %d MB of audio so far", mb) + # PCM s16le mono: sample_rate * 2 байта в секунду + minutes = bytes_read // (sample_rate * 2 * 60) + if minutes >= last_logged_mb + 5: + last_logged_mb = minutes + logger.info("Transcribed %d minutes of audio so far", minutes) + if on_progress is not None: + try: + on_progress(minutes) + except Exception: + pass result = json.loads(recognizer.FinalResult()) text = result.get("text", "") diff --git a/api/lib/youtube.py b/api/lib/youtube.py index 44bc7c9..a2f2f2e 100644 --- a/api/lib/youtube.py +++ b/api/lib/youtube.py @@ -12,7 +12,44 @@ import yt_dlp logger = logging.getLogger("youtube") -_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"} +_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"} + + +def expand_url(url: str) -> dict: + """Определяет, видео это или плейлист, и возвращает список элементов. + + Returns: {"kind": "video"|"playlist", "title": str, + "entries": [{"url": str, "title": str}, ...]} + """ + opts = { + "extract_flat": "in_playlist", + "quiet": True, + "no_warnings": True, + "skip_download": True, + } + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=False) + + if info.get("_type") == "playlist": + entries = [] + for e in info.get("entries") or []: + if not e: + continue + u = e.get("url") or "" + if u and not u.startswith("http"): + u = f"https://www.youtube.com/watch?v={u}" + if not u and e.get("id"): + u = f"https://www.youtube.com/watch?v={e['id']}" + if u: + entries.append({"url": u, "title": e.get("title") or u}) + return { + "kind": "playlist", + "title": info.get("title") or url, + "entries": entries, + } + + title = info.get("title") or url + return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]} def download_video_with_subs( @@ -20,9 +57,15 @@ def download_video_with_subs( video_id: str, video_dir: Path, langs: Optional[list[str]] = None, + audio_only: bool = False, + on_progress=None, ) -> dict: """Download a single video, also requesting subtitles for the given languages. + audio_only: качает только аудиодорожку (для длинных лекций, когда видео + не сохраняется — в разы быстрее и меньше по объёму). + on_progress: callback(percent: int), вызывается с шагом ~5%. + Returns: { "title": str, @@ -40,11 +83,34 @@ def download_video_with_subs( base_opts = { "outtmpl": outtmpl, - "format": "bv*+ba/b", - "merge_output_format": "mp4", "quiet": True, "no_warnings": True, + "noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист } + if audio_only: + base_opts["format"] = "ba/b" + else: + base_opts["format"] = "bv*+ba/b" + base_opts["merge_output_format"] = "mp4" + + if on_progress is not None: + state = {"last": -5} + + def _hook(dd: dict): + if dd.get("status") != "downloading": + return + total = dd.get("total_bytes") or dd.get("total_bytes_estimate") + if not total: + return + pct = int(dd.get("downloaded_bytes", 0) * 100 / total) + if pct >= state["last"] + 5: + state["last"] = pct + try: + on_progress(pct) + except Exception: + pass + + base_opts["progress_hooks"] = [_hook] sub_opts = { **base_opts, "writesubtitles": True, diff --git a/api/main.py b/api/main.py index df4e0cd..08f1afc 100644 --- a/api/main.py +++ b/api/main.py @@ -1,10 +1,13 @@ +import asyncio import logging from contextlib import asynccontextmanager from fastapi import FastAPI from api.db.connection import wait_for_postgres, init_db -from api.route import default, llm, pipeline, sections, videos +from api.db.job import mark_interrupted +from api.lib import jobqueue +from api.route import app_settings, default, jobs, llm, pipeline, sections, videos logging.basicConfig( @@ -17,13 +20,18 @@ logging.basicConfig( async def lifespan(app: FastAPI): await wait_for_postgres() await init_db() + await mark_interrupted() + worker_task = asyncio.create_task(jobqueue.worker()) yield + worker_task.cancel() -app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan) +app = FastAPI(title="LLM-infa API", version="0.4.0", lifespan=lifespan) app.include_router(default.router) app.include_router(pipeline.router) +app.include_router(jobs.router) app.include_router(videos.router) app.include_router(sections.router) +app.include_router(app_settings.router) app.include_router(llm.router) diff --git a/api/route/app_settings.py b/api/route/app_settings.py new file mode 100644 index 0000000..7c25f14 --- /dev/null +++ b/api/route/app_settings.py @@ -0,0 +1,59 @@ +"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres, +переживают перезапуск контейнеров и перезагрузку страницы. +""" +from __future__ import annotations + +from fastapi import APIRouter +from pydantic import BaseModel + +from api.db.setting import get_all_settings, set_settings + + +router = APIRouter(prefix="/settings", tags=["settings"]) + +_KEYS = ( + "llm_base_url", + "llm_api_key", + "llm_model", + "summary_max_sentences", + "keep_video", + "vosk_model_path", +) + + +class SettingsPayload(BaseModel): + llm_base_url: str | None = None + llm_api_key: str | None = None + llm_model: str | None = None + summary_max_sentences: int | None = None + keep_video: bool | None = None + vosk_model_path: str | None = None + + +@router.get("/") +async def get_settings(): + db = await get_all_settings() + return { + "llm_base_url": db.get("llm_base_url", ""), + "llm_api_key": db.get("llm_api_key", ""), + "llm_model": db.get("llm_model", ""), + "summary_max_sentences": int(db.get("summary_max_sentences") or 15), + "keep_video": db.get("keep_video", "1") != "0", + "vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"), + } + + +@router.put("/") +async def put_settings(body: SettingsPayload): + values: dict[str, str] = {} + for key in _KEYS: + v = getattr(body, key) + if v is None: + continue + if key == "keep_video": + values[key] = "1" if v else "0" + else: + values[key] = str(v) + if values: + await set_settings(values) + return await get_settings() diff --git a/api/route/jobs.py b/api/route/jobs.py new file mode 100644 index 0000000..43d3879 --- /dev/null +++ b/api/route/jobs.py @@ -0,0 +1,112 @@ +"""Фоновые задачи: видео или плейлист любой длины. + +POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs. +""" +from __future__ import annotations + +import uuid as _uuid +from datetime import datetime + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from api.db.job import create_job, delete_job, get_job, list_jobs +from api.lib import jobqueue + + +router = APIRouter(prefix="/jobs", tags=["jobs"]) + + +class JobIn(BaseModel): + url: str + section_id: int | None = None + # None → берутся из сохранённых настроек + keep_video: bool | None = None + summary_max_sentences: int | None = None + model_path: str | None = None + + +class JobOut(BaseModel): + id: str + url: str + kind: str + title: str + status: str + total_items: int + done_items: int + failed_items: int + current_item: str + live: str + error: str + created_at: datetime + + +def _to_out(j) -> JobOut: + return JobOut( + id=j.id, + url=j.url, + kind=j.kind, + title=j.title, + status=j.status, + total_items=j.total_items, + done_items=j.done_items, + failed_items=j.failed_items, + current_item=j.current_item, + live=jobqueue.live_status(j.id) or j.current_item, + error=j.error, + created_at=j.created_at, + ) + + +@router.post("/", response_model=JobOut) +async def create(body: JobIn): + url = body.url.strip() + if not url: + raise HTTPException(status_code=400, detail="URL is empty") + job_id = _uuid.uuid4().hex + j = await create_job(job_id, url) + jobqueue.submit( + job_id, + { + "section_id": body.section_id, + "keep_video": body.keep_video, + "summary_max_sentences": body.summary_max_sentences, + "model_path": body.model_path, + }, + ) + return _to_out(j) + + +@router.get("/", response_model=list[JobOut]) +async def list_all(limit: int = 50): + return [_to_out(j) for j in await list_jobs(limit)] + + +@router.get("/{job_id}", response_model=JobOut) +async def get_one(job_id: str): + j = await get_job(job_id) + if j is None: + raise HTTPException(status_code=404, detail="Job not found") + return _to_out(j) + + +@router.post("/{job_id}/cancel") +async def cancel(job_id: str): + j = await get_job(job_id) + if j is None: + raise HTTPException(status_code=404, detail="Job not found") + if j.status not in ("queued", "running"): + raise HTTPException(status_code=400, detail=f"Job is {j.status}") + jobqueue.request_cancel(job_id) + return {"ok": True, "note": "остановится после текущего элемента"} + + +@router.delete("/{job_id}") +async def delete(job_id: str): + j = await get_job(job_id) + if j is None: + raise HTTPException(status_code=404, detail="Job not found") + if j.status in ("queued", "running"): + raise HTTPException(status_code=400, detail="Сначала отмените задачу") + await delete_job(job_id) + return {"ok": True} diff --git a/api/route/pipeline.py b/api/route/pipeline.py index c49c77e..6ecde63 100644 --- a/api/route/pipeline.py +++ b/api/route/pipeline.py @@ -1,29 +1,16 @@ -"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist.""" +"""Синхронный запуск пайплайна для одного видео (для длинных видео и +плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым). +""" from __future__ import annotations -import asyncio -import logging -import uuid as _uuid -from pathlib import Path - from fastapi import APIRouter, HTTPException from pydantic import BaseModel -from api.config import settings -from api.db.video import create_video -from api.lib.llm import LLMConfig, from_settings as llm_from_settings -from api.lib.summarize import summarize -from api.lib.transcribe import transcribe_via_ffmpeg -from api.lib.youtube import download_video_with_subs, vtt_to_text +from api.lib.llm import LLMConfig +from api.lib.processing import PipelineError, process_single_video router = APIRouter(prefix="/pipeline", tags=["pipeline"]) -logger = logging.getLogger("pipeline") - -_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos" -_TEXT_DIR = _PROJECT_ROOT / "data" / "text" -_MODELS_DIR = _PROJECT_ROOT / "models" class LLMOverride(BaseModel): @@ -40,12 +27,12 @@ class LLMOverride(BaseModel): class ProcessRequest(BaseModel): url: str - model_path: str | None = None # path inside models/, used when subtitles are unavailable - summary_max_sentences: int = 15 - keep_video: bool = True # False: файл видео удаляется после расшифровки + model_path: str | None = None # None → из настроек + summary_max_sentences: int | None = None + keep_video: bool | None = None section_id: int | None = None - # If supplied, this LLM is used for the summary; otherwise env-configured LLM; - # otherwise the built-in extractive fallback. + # Если задано — используется этот LLM; иначе сохранённые настройки/env; + # иначе встроенная экстрактивная выжимка. llm: LLMOverride | None = None @@ -60,86 +47,10 @@ class ProcessResponse(BaseModel): summary_preview: str -def _resolve_model_path(req_model_path: str | None) -> Path: - raw = req_model_path or settings.DEFAULT_VOSK_MODEL - p = Path(raw) - if not p.is_absolute(): - p = _PROJECT_ROOT / raw - p = p.resolve() - try: - p.relative_to(_MODELS_DIR.resolve()) - except ValueError: - raise HTTPException( - status_code=400, - detail="model_path must point inside the project's models directory", - ) - if not p.exists(): - raise HTTPException(status_code=400, detail=f"Model not found: {p}") - return p - - @router.post("/process", response_model=ProcessResponse) async def process_video(req: ProcessRequest): - video_uuid = _uuid.uuid4().hex - _VIDEO_DIR.mkdir(parents=True, exist_ok=True) - _TEXT_DIR.mkdir(parents=True, exist_ok=True) - - loop = asyncio.get_running_loop() - - logger.info("[%s] downloading %s", video_uuid, req.url) - try: - info = await loop.run_in_executor( - None, - download_video_with_subs, - req.url, - video_uuid, - _VIDEO_DIR, - settings.subtitle_langs_list, - ) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Download failed: {e}") - - title: str = info["title"] - video_path: Path = info["video_path"] - sub_path: Path | None = info["subtitle_path"] - - if not video_path.exists(): - raise HTTPException(status_code=500, detail="Video file missing after download") - - full_text = "" - method = "" - - if sub_path is not None and sub_path.exists(): - try: - content = sub_path.read_text(encoding="utf-8", errors="ignore") - full_text = vtt_to_text(content) - if full_text.strip(): - method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}" - logger.info("[%s] using subtitles (%s)", video_uuid, method) - except Exception as e: - logger.warning("[%s] subtitle parse failed: %s", video_uuid, e) - full_text = "" - - if not full_text.strip(): - logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid) - model_path = _resolve_model_path(req.model_path) - try: - full_text = await loop.run_in_executor( - None, transcribe_via_ffmpeg, video_path, model_path - ) - method = f"vosk:{model_path.name}" - except Exception as e: - raise HTTPException(status_code=500, detail=f"Transcription failed: {e}") - - if not full_text.strip(): - raise HTTPException( - status_code=500, detail="No subtitles and transcription returned empty text" - ) - - text_full_path = _TEXT_DIR / f"{video_uuid}.txt" - text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt" - text_full_path.write_text(full_text, encoding="utf-8") - + llm_cfg = None + sys_p = user_t = None if req.llm is not None: llm_cfg = LLMConfig( base_url=req.llm.base_url, @@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest): ) sys_p = req.llm.system_prompt user_t = req.llm.user_template - else: - llm_cfg = llm_from_settings() - sys_p = None - user_t = None try: - summary = await loop.run_in_executor( - None, - lambda: summarize( - full_text, - llm_cfg=llm_cfg, - system_prompt=sys_p, - user_template=user_t, - max_sentences=req.summary_max_sentences, - ), - ) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Summarization failed: {e}") - text_summary_path.write_text(summary, encoding="utf-8") - - if req.keep_video: - rel_video = str(video_path.relative_to(_PROJECT_ROOT)) - else: - # пользователь просил не хранить видео: текст уже извлечён, файлы не нужны - video_path.unlink(missing_ok=True) - for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"): - p.unlink(missing_ok=True) - rel_video = "" - rel_full = str(text_full_path.relative_to(_PROJECT_ROOT)) - rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT)) - - try: - await create_video( - uuid=video_uuid, - source_url=req.url, - title=title, - video_path=rel_video, - text_full_path=rel_full, - text_summary_path=rel_summary, - transcription_method=method, + result = await process_single_video( + req.url, + model_path=req.model_path, + summary_max_sentences=req.summary_max_sentences, + keep_video=req.keep_video, section_id=req.section_id, + llm_cfg=llm_cfg, + system_prompt=sys_p, + user_template=user_t, ) - except Exception as e: - raise HTTPException(status_code=500, detail=f"DB insert failed: {e}") - - logger.info("[%s] done (method=%s)", video_uuid, method) - - return ProcessResponse( - uuid=video_uuid, - title=title, - source_url=req.url, - video_path=rel_video, - text_full_path=rel_full, - text_summary_path=rel_summary, - transcription_method=method, - summary_preview=summary[:300], - ) + except PipelineError as e: + raise HTTPException(status_code=e.status, detail=str(e)) + return ProcessResponse(**result) diff --git a/pyproject.toml b/pyproject.toml index 1de0e6a..16f3156 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm-infa" -version = "0.3.0" +version = "0.4.0" description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres" readme = "README.md" requires-python = ">=3.14" diff --git a/web/api_client.py b/web/api_client.py index b348110..5e68fc0 100644 --- a/web/api_client.py +++ b/web/api_client.py @@ -21,10 +21,15 @@ class ApiClient: def close(self) -> None: self._client.close() - # --- LLM --- + # --- Settings (persisted server-side in Postgres) --- - def llm_status(self) -> dict: - return self._get("/llm/config") + def get_settings(self) -> dict: + return self._get("/settings/") + + def save_settings(self, values: dict) -> dict: + return self._request("PUT", "/settings/", values) + + # --- LLM --- def llm_models(self, base_url: str, api_key: str) -> list[dict]: data = self._post( @@ -39,10 +44,21 @@ class ApiClient: {"base_url": base_url, "api_key": api_key, "model": model}, ) - # --- Pipeline / videos --- + # --- Jobs (видео и плейлисты, в фоне) --- - def pipeline_process(self, payload: dict) -> dict: - return self._post("/pipeline/process", payload) + def create_job(self, payload: dict) -> dict: + return self._post("/jobs/", payload) + + def list_jobs(self) -> list[dict]: + return self._get("/jobs/") + + def cancel_job(self, job_id: str) -> dict: + return self._post(f"/jobs/{job_id}/cancel", None) + + def delete_job(self, job_id: str) -> dict: + return self._request("DELETE", f"/jobs/{job_id}") + + # --- Videos --- def list_videos( self, diff --git a/web/main.py b/web/main.py index eb8213e..a27977e 100644 --- a/web/main.py +++ b/web/main.py @@ -19,20 +19,14 @@ def main(page: ft.Page) -> None: view = MainView(page, client) page.add(view.build()) - # probe env-configured LLM and initial library load + # настройки живут на сервере (БД) — заполняем поля сохранёнными значениями try: - status = client.llm_status() - if status.get("configured"): - view._set_status(ok=True) - if status.get("base_url") and not view.base_url.value: - view.base_url.value = status["base_url"] - if status.get("model") and not view.selected_model: - view.selected_model = status["model"] - view.model_input.value = status["model"] + view.apply_settings(client.get_settings()) except Exception: pass view._refresh_library() + view._start_jobs_poll() page.update() diff --git a/web/views/main_view.py b/web/views/main_view.py index 2faabf5..1483714 100644 --- a/web/views/main_view.py +++ b/web/views/main_view.py @@ -1,7 +1,8 @@ -"""Main page of the LLM-infa Flet web app.""" +"""Main page of the LLM-infa Flet web app: вкладки «Главная» и «Настройки».""" from __future__ import annotations import threading +import time import flet as ft @@ -9,6 +10,16 @@ from web.api_client import ApiClient, ApiError import web.designer as d +_JOB_STATUS = { + "queued": ("в очереди", None), + "running": ("выполняется", None), + "done": ("готово", "ok"), + "error": ("ошибка", "danger"), + "cancelled": ("отменена", None), + "interrupted": ("прервана", "danger"), +} + + class MainView: def __init__(self, page: ft.Page, client: ApiClient): self.page = page @@ -20,14 +31,15 @@ class MainView: self.sections: list[dict] = [] self.current_section_id: int | None = None # None = все видео - # ── LLM config fields ────────────────────────────────────────────── + self._polling = False + + # ── Settings tab fields ──────────────────────────────────────────── self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True) self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True) - # Searchable model combobox self.model_input = d.text_field( "Модель", - hint="Сначала нажмите «Загрузить модели»", + hint="Нажмите «Загрузить модели» и выберите из списка", on_change=self._on_model_search, expand=True, ) @@ -44,7 +56,7 @@ class MainView: self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True) self.llm_status_chip = ft.Container( - content=ft.Text("LLM не подключена", size=12, color=d.MUTED), + content=ft.Text("LLM не настроена", size=12, color=d.MUTED), padding=ft.padding.symmetric(horizontal=10, vertical=4), border=ft.border.all(1, d.BORDER), border_radius=99, @@ -52,12 +64,6 @@ class MainView: self.btn_load = d.primary_button("Загрузить модели", self._load_models) self.btn_test = d.secondary_button("Проверить связь", self._test_conn) - # ── Pipeline fields ──────────────────────────────────────────────── - self.yt_url = d.text_field( - "Ссылка на YouTube", - hint="https://youtu.be/…", - expand=True, - ) self.vosk_path = d.text_field( "Vosk-модель (fallback, когда нет субтитров)", value="models/vosk-model-small-ru-0.22", @@ -76,19 +82,28 @@ class MainView: width=220, ) self.keep_video_sw = ft.Switch( - label="Сохранять видео на диске", + label="Сохранять видео на диске (иначе качается только аудио)", value=True, active_color=d.ACCENT, label_style=ft.TextStyle(color=d.TEXT, size=13), ) - self.pipeline_section_dd = self._dropdown("Раздел для нового видео", width=280) + self.settings_msg = ft.Text("", color=d.MUTED, size=13, expand=True) + self.btn_save = d.primary_button("💾 Сохранить настройки", self._save_settings) + + # ── Pipeline (main tab) ──────────────────────────────────────────── + self.yt_url = d.text_field( + "Ссылка на YouTube: видео или плейлист", + hint="https://youtu.be/… или https://www.youtube.com/playlist?list=…", + expand=True, + ) + self.pipeline_section_dd = self._dropdown("Раздел для новых видео", width=280) self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True) - self.btn_run = d.primary_button("▶ Запустить", self._run_pipeline) + self.btn_run = d.primary_button("▶ В очередь", self._run_pipeline) - # Result block (initially hidden) - self._result_col = ft.Column(spacing=6, visible=False) + # ── Jobs ─────────────────────────────────────────────────────────── + self._jobs_col = ft.Column(spacing=6) - # ── Library controls ─────────────────────────────────────────────── + # ── Library ──────────────────────────────────────────────────────── self.search_field = d.text_field( "Поиск по названию / URL", on_change=None, expand=True ) @@ -124,24 +139,47 @@ class MainView: # ─────────────────────── build ─────────────────────────────────────── def build(self) -> ft.Control: - return ft.Column( - controls=[ - self._build_topbar(), - ft.Container( - content=ft.Column( - [ - self._build_llm_card(), - self._build_pipeline_card(), - self._build_library_card(), - ], - spacing=20, - scroll=ft.ScrollMode.AUTO, - ), - padding=ft.padding.symmetric(horizontal=24, vertical=20), - expand=True, - ), + main_tab = ft.Container( + content=ft.Column( + [ + self._build_pipeline_card(), + self._build_jobs_card(), + self._build_library_card(), + ], + spacing=20, + scroll=ft.ScrollMode.AUTO, + ), + padding=ft.padding.symmetric(horizontal=24, vertical=20), + expand=True, + ) + settings_tab = ft.Container( + content=ft.Column( + [ + self._build_llm_card(), + self._build_defaults_card(), + ], + spacing=20, + scroll=ft.ScrollMode.AUTO, + ), + padding=ft.padding.symmetric(horizontal=24, vertical=20), + expand=True, + ) + tabs = ft.Tabs( + selected_index=0, + animation_duration=200, + label_color=d.ACCENT_2, + unselected_label_color=d.MUTED, + indicator_color=d.ACCENT, + divider_color=d.BORDER, + tabs=[ + ft.Tab(text="Главная", content=main_tab), + ft.Tab(text="Настройки", content=settings_tab), ], expand=True, + ) + return ft.Column( + controls=[self._build_topbar(), tabs], + expand=True, spacing=0, ) @@ -174,13 +212,14 @@ class MainView: border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)), ) + # ── Settings tab cards ──────────────────────────────────────────────── + def _build_llm_card(self) -> ft.Container: return d.card( d.section_title("Подключение к LLM"), d.hint( - "Укажите любой OpenAI-совместимый провайдер. " - "Нейронка внутри Docker не разворачивается — " - "указываем URL внешнего сервиса." + "Любой OpenAI-совместимый провайдер (OpenRouter, OpenAI, локальная Ollama…). " + "Ключ и модель хранятся на сервере в БД и переживают перезапуск." ), ft.Row([self.base_url, self.api_key], spacing=12), ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10), @@ -189,22 +228,43 @@ class MainView: self._model_list_wrap, ) + def _build_defaults_card(self) -> ft.Container: + return d.card( + d.section_title("Параметры обработки"), + d.hint("Применяются ко всем новым задачам."), + ft.Row([self.vosk_path, self.n_sentences], spacing=12), + self.keep_video_sw, + ft.Divider(color=d.BORDER, height=1), + ft.Row([self.btn_save, self.settings_msg], spacing=10), + ) + + # ── Main tab cards ──────────────────────────────────────────────────── + def _build_pipeline_card(self) -> ft.Container: return d.card( - d.section_title("Обработать видео"), + d.section_title("Обработать видео или плейлист"), d.hint( - "Pipeline: yt-dlp → субтитры (или Vosk fallback) " - "→ выжимка через выбранную LLM → запись в БД." + "Задача уходит в фон: качаем, расшифровываем (субтитры или Vosk), " + "сокращаем через LLM, пишем в БД. Плейлисты и лекции на 4–5 часов — ок." ), self.yt_url, - ft.Row([self.vosk_path, self.n_sentences], spacing=12), ft.Row( - [self.pipeline_section_dd, self.keep_video_sw], - spacing=18, + [self.btn_run, self.pipeline_section_dd, self.pipeline_msg], + spacing=12, vertical_alignment=ft.CrossAxisAlignment.CENTER, ), - ft.Row([self.btn_run, self.pipeline_msg], spacing=10), - self._result_col, + ) + + def _build_jobs_card(self) -> ft.Container: + return d.card( + ft.Row( + [ + d.section_title("Задачи"), + ft.Container(expand=True), + d.secondary_button("Обновить", lambda _: self._start_jobs_poll()), + ] + ), + self._jobs_col, ) def _build_library_card(self) -> ft.Container: @@ -256,6 +316,39 @@ class MainView: ), ) + # ─────────────────── settings load/save ────────────────────────────── + + def apply_settings(self, s: dict) -> None: + """Заполняет поля значениями с сервера (вызывается при старте сессии).""" + self.base_url.value = s.get("llm_base_url", "") + self.api_key.value = s.get("llm_api_key", "") + self.selected_model = s.get("llm_model", "") + self.model_input.value = self.selected_model + self.n_sentences.value = str(s.get("summary_max_sentences", 15)) + self.keep_video_sw.value = bool(s.get("keep_video", True)) + self.vosk_path.value = s.get("vosk_model_path", "models/vosk-model-small-ru-0.22") + self._set_status(ok=bool(self.base_url.value and self.selected_model)) + + def _save_settings(self, _): + model = self.selected_model or (self.model_input.value or "").strip() + payload = { + "llm_base_url": self.base_url.value.strip(), + "llm_api_key": self.api_key.value.strip(), + "llm_model": model, + "summary_max_sentences": int(self.n_sentences.value or 15), + "keep_video": bool(self.keep_video_sw.value), + "vosk_model_path": self.vosk_path.value.strip(), + } + try: + self.client.save_settings(payload) + self.settings_msg.value = "Сохранено ✓ (хранится в БД, переживёт перезапуск)" + self.settings_msg.color = d.OK + self._set_status(ok=bool(payload["llm_base_url"] and model)) + except ApiError as e: + self.settings_msg.value = str(e) + self.settings_msg.color = d.DANGER + self.page.update() + # ─────────────────── model combobox ────────────────────────────────── def _on_model_search(self, e): @@ -292,6 +385,8 @@ class MainView: self.selected_model = mid self.model_input.value = mid self._model_list_wrap.visible = False + self.llm_msg.value = "Не забудьте нажать «Сохранить настройки»" + self.llm_msg.color = d.MUTED self.page.update() # ─────────────────── LLM actions ───────────────────────────────────── @@ -311,10 +406,8 @@ class MainView: self.models = self.client.llm_models(base_url, self.api_key.value.strip()) self._render_model_list(self.models) self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True) - self._set_status(ok=True) except ApiError as e: self._set_llm_msg(str(e), error=True) - self._set_status(ok=False) finally: self.btn_load.disabled = False self.page.update() @@ -322,7 +415,8 @@ class MainView: threading.Thread(target=_task, daemon=True).start() def _test_conn(self, _): - if not self.selected_model: + model = self.selected_model or (self.model_input.value or "").strip() + if not model: self._set_llm_msg("Сначала выберите модель", error=True) return @@ -335,87 +429,164 @@ class MainView: r = self.client.llm_test( self.base_url.value.strip(), self.api_key.value.strip(), - self.selected_model, + model, ) preview = (r.get("reply") or "")[:80] self._set_llm_msg(f"OK · {preview}", ok=True) - self._set_status(ok=True) except ApiError as e: self._set_llm_msg(str(e), error=True) - self._set_status(ok=False) finally: self.btn_test.disabled = False self.page.update() threading.Thread(target=_task, daemon=True).start() - # ─────────────────── pipeline ───────────────────────────────────────── + # ─────────────────── pipeline → jobs ────────────────────────────────── def _run_pipeline(self, _): url = self.yt_url.value.strip() if not url: - self._set_pipeline_msg("Укажите ссылку на видео", error=True) + self._set_pipeline_msg("Укажите ссылку на видео или плейлист", error=True) return - payload: dict = { - "url": url, - "model_path": self.vosk_path.value.strip() or None, - "summary_max_sentences": int(self.n_sentences.value or 15), - "keep_video": bool(self.keep_video_sw.value), - } + payload: dict = {"url": url} if self.pipeline_section_dd.value: payload["section_id"] = int(self.pipeline_section_dd.value) - if self.base_url.value.strip() and self.selected_model: - payload["llm"] = { - "base_url": self.base_url.value.strip(), - "api_key": self.api_key.value.strip(), - "model": self.selected_model, - } - self._set_pipeline_msg("Обработка видео… (может занять несколько минут)") - self._result_col.visible = False - self.btn_run.disabled = True + try: + self.client.create_job(payload) + self.yt_url.value = "" + self._set_pipeline_msg("Задача поставлена в очередь ✓", ok=True) + except ApiError as e: + self._set_pipeline_msg(str(e), error=True) self.page.update() + self._start_jobs_poll() - def _task(): - try: - r = self.client.pipeline_process(payload) - self._render_result(r) - self._set_pipeline_msg("Готово ✓", ok=True) - self._refresh_library() - except ApiError as e: - self._set_pipeline_msg(str(e), error=True) - finally: - self.btn_run.disabled = False - self.page.update() + # ─────────────────── jobs ───────────────────────────────────────────── - threading.Thread(target=_task, daemon=True).start() + def _start_jobs_poll(self): + if self._polling: + return + self._polling = True + threading.Thread(target=self._poll_loop, daemon=True).start() - def _render_result(self, r: dict): - self._result_col.controls = [ - ft.Divider(color=d.BORDER, height=1), - ft.Text(r.get("title") or r.get("uuid", ""), color=d.ACCENT_2, size=15, weight=ft.FontWeight.W_600), - *[ - self._kv(k, v) - for k, v in [ - ("UUID", r.get("uuid", "")), - ("Источник", r.get("source_url", "")), - ("Метод", r.get("transcription_method", "")), - ("Видео", r.get("video_path") or "не сохранено"), - ("Полный текст", r.get("text_full_path", "")), - ("Выжимка", r.get("text_summary_path", "")), - ] - ], - ft.Text("Превью выжимки:", color=d.MUTED, size=12), - ft.Container( - content=ft.Text(r.get("summary_preview", ""), color=d.TEXT, size=13, selectable=True, no_wrap=False), - padding=12, - bgcolor=d.SURFACE_3, - border=ft.border.all(1, d.BORDER), - border_radius=8, + def _poll_loop(self): + prev: dict = {} + try: + while True: + try: + jobs = self.client.list_jobs() + except (ApiError, Exception): + break + self._render_jobs(jobs) + state = {j["id"]: (j["done_items"], j["status"]) for j in jobs} + if state != prev: + prev = state + self._refresh_library() # там же page.update() + else: + self.page.update() + if not any(j["status"] in ("queued", "running") for j in jobs): + break + time.sleep(3) + finally: + self._polling = False + + def _render_jobs(self, jobs: list[dict]): + if not jobs: + self._jobs_col.controls = [ + ft.Text("Задач ещё не было", color=d.MUTED, size=13) + ] + return + self._jobs_col.controls = [self._job_row(j) for j in jobs] + + def _job_row(self, j: dict) -> ft.Container: + label, tone = _JOB_STATUS.get(j["status"], (j["status"], None)) + color = d.OK if tone == "ok" else (d.DANGER if tone == "danger" else d.MUTED) + active = j["status"] in ("queued", "running") + + chip = ft.Container( + content=ft.Text(label, size=11, color=color), + padding=ft.padding.symmetric(horizontal=8, vertical=2), + border=ft.border.all(1, color), + border_radius=99, + ) + head = [ + chip, + ft.Text( + (j["title"] or j["url"])[:90], + color=d.TEXT, + size=13, + weight=ft.FontWeight.W_600, + expand=True, + no_wrap=True, ), ] - self._result_col.visible = True + if j["kind"] == "playlist": + head.append(ft.Text("плейлист", color=d.ACCENT_2, size=11)) + if j["total_items"]: + head.append( + ft.Text(f"{j['done_items']}/{j['total_items']}", color=d.MUTED, size=12) + ) + if active: + head.append( + ft.IconButton( + icon=ft.Icons.STOP_CIRCLE_OUTLINED, + icon_color=d.DANGER, + icon_size=17, + tooltip="Отменить (после текущего элемента)", + on_click=lambda e, jid=j["id"]: self._cancel_job(jid), + ) + ) + else: + head.append( + ft.IconButton( + icon=ft.Icons.CLOSE, + icon_color=d.MUTED, + icon_size=15, + tooltip="Убрать из списка", + on_click=lambda e, jid=j["id"]: self._delete_job(jid), + ) + ) + + rows: list[ft.Control] = [ft.Row(head, spacing=8)] + if active: + total = j["total_items"] or 0 + value = (j["done_items"] / total) if total else None + rows.append( + ft.ProgressBar(value=value, color=d.ACCENT, bgcolor=d.SURFACE_3, height=4) + ) + if j.get("live"): + rows.append(ft.Text(j["live"][:140], color=d.ACCENT_2, size=11)) + if j.get("error"): + rows.append( + ft.Text(j["error"][:300], color=d.DANGER, size=11, no_wrap=False) + ) + if j["failed_items"]: + rows.append( + ft.Text(f"с ошибкой: {j['failed_items']}", color=d.DANGER, size=11) + ) + + return ft.Container( + content=ft.Column(rows, spacing=5, tight=True), + padding=10, + bgcolor=d.SURFACE_2, + border=ft.border.all(1, d.BORDER), + border_radius=10, + ) + + def _cancel_job(self, job_id: str): + try: + self.client.cancel_job(job_id) + except ApiError as e: + self.lib_msg.value = str(e) + self._start_jobs_poll() + + def _delete_job(self, job_id: str): + try: + self.client.delete_job(job_id) + except ApiError as e: + self.lib_msg.value = str(e) + self._start_jobs_poll() # ─────────────────── sections ───────────────────────────────────────── @@ -688,6 +859,6 @@ class MainView: def _set_status(self, *, ok: bool): chip = self.llm_status_chip - chip.content.value = "LLM подключена" if ok else "Ошибка подключения" - chip.content.color = d.OK if ok else d.DANGER - chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else "rgba(227,90,114,.4)") + chip.content.value = "LLM настроена" if ok else "LLM не настроена" + chip.content.color = d.OK if ok else d.MUTED + chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else d.BORDER)