Фоновые задачи: плейлисты, длинные лекции, вкладка настроек
- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
плейлиста обрабатываются последовательно, есть отмена и живой прогресс
(скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
готовности элементов
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
27
README.md
27
README.md
@@ -48,19 +48,38 @@ OpenAI-совместимый эндпоинт. Два способа:
|
|||||||
Если LLM не настроена, работает встроенная экстрактивная суммаризация
|
Если LLM не настроена, работает встроенная экстрактивная суммаризация
|
||||||
(выбор ключевых предложений, без нейросети).
|
(выбор ключевых предложений, без нейросети).
|
||||||
|
|
||||||
|
## Плейлисты и длинные лекции
|
||||||
|
|
||||||
|
Обработка идёт через фоновые задачи: `POST /jobs` сразу возвращает id, элементы
|
||||||
|
плейлиста обрабатываются по очереди, прогресс виден в UI (скачивание %,
|
||||||
|
минуты распознанного аудио, номер видео в плейлисте). Если в настройках
|
||||||
|
выключено «Сохранять видео», для лекций качается только аудиодорожка —
|
||||||
|
в разы быстрее и легче.
|
||||||
|
|
||||||
|
Настройки (ключ LLM, модель, длина выжимки, хранение видео) сохраняются
|
||||||
|
в Postgres — переживают перезапуск контейнеров и перезагрузку страницы.
|
||||||
|
Вкладка «Настройки» в UI.
|
||||||
|
|
||||||
## API
|
## API
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# обработать видео целиком
|
# фоновая задача: видео или плейлист
|
||||||
|
curl -X POST localhost:8000/jobs/ \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{"url": "https://www.youtube.com/playlist?list=..."}'
|
||||||
|
curl localhost:8000/jobs/ # статусы и прогресс
|
||||||
|
|
||||||
|
# синхронно, одно видео (для коротких роликов)
|
||||||
curl -X POST localhost:8000/pipeline/process \
|
curl -X POST localhost:8000/pipeline/process \
|
||||||
-H 'Content-Type: application/json' \
|
-H 'Content-Type: application/json' \
|
||||||
-d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}'
|
-d '{"url": "https://youtu.be/...", "summary_max_sentences": 10}'
|
||||||
|
|
||||||
# каталог обработанных видео
|
# каталог обработанных видео (фильтры: ?q=, ?section_id=, ?method=)
|
||||||
curl localhost:8000/videos/
|
curl localhost:8000/videos/
|
||||||
|
|
||||||
# проверить связь с LLM
|
# настройки
|
||||||
curl -X POST localhost:8000/llm/test
|
curl -X PUT localhost:8000/settings/ -H 'Content-Type: application/json' \
|
||||||
|
-d '{"llm_base_url": "https://openrouter.ai/api/v1", "llm_api_key": "sk-...", "llm_model": "..."}'
|
||||||
```
|
```
|
||||||
|
|
||||||
## Заметки
|
## Заметки
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ from sqlalchemy.orm import sessionmaker
|
|||||||
|
|
||||||
from api.config import settings
|
from api.config import settings
|
||||||
from api.models.base import Base
|
from api.models.base import Base
|
||||||
|
import api.models.job # noqa: F401 - register model with Base.metadata
|
||||||
import api.models.section # noqa: F401 - register model with Base.metadata
|
import api.models.section # noqa: F401 - register model with Base.metadata
|
||||||
|
import api.models.setting # noqa: F401 - register model with Base.metadata
|
||||||
import api.models.video # noqa: F401 - register model with Base.metadata
|
import api.models.video # noqa: F401 - register model with Base.metadata
|
||||||
|
|
||||||
|
|
||||||
@@ -69,3 +71,6 @@ async def init_db() -> None:
|
|||||||
"REFERENCES sections(id) ON DELETE SET NULL"
|
"REFERENCES sections(id) ON DELETE SET NULL"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
await conn.execute(
|
||||||
|
text("ALTER TABLE videos ADD COLUMN IF NOT EXISTS job_id VARCHAR")
|
||||||
|
)
|
||||||
|
|||||||
53
api/db/job.py
Normal file
53
api/db/job.py
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
from api.db.connection import SessionLocal
|
||||||
|
from api.models.job import Job
|
||||||
|
|
||||||
|
|
||||||
|
async def create_job(job_id: str, url: str) -> Job:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
j = Job(id=job_id, url=url, status="queued")
|
||||||
|
session.add(j)
|
||||||
|
await session.commit()
|
||||||
|
await session.refresh(j)
|
||||||
|
return j
|
||||||
|
|
||||||
|
|
||||||
|
async def get_job(job_id: str) -> Job | None:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
return await session.get(Job, job_id)
|
||||||
|
|
||||||
|
|
||||||
|
async def list_jobs(limit: int = 50) -> list[Job]:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
res = await session.execute(
|
||||||
|
select(Job).order_by(Job.created_at.desc()).limit(limit)
|
||||||
|
)
|
||||||
|
return list(res.scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
async def update_job(job_id: str, **fields) -> None:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
await session.execute(update(Job).where(Job.id == job_id).values(**fields))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def delete_job(job_id: str) -> bool:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
j = await session.get(Job, job_id)
|
||||||
|
if j is None:
|
||||||
|
return False
|
||||||
|
await session.delete(j)
|
||||||
|
await session.commit()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def mark_interrupted() -> None:
|
||||||
|
"""На старте приложения: задачи, шедшие до перезапуска, отмечаем прерванными."""
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
await session.execute(
|
||||||
|
update(Job)
|
||||||
|
.where(Job.status.in_(("queued", "running")))
|
||||||
|
.values(status="interrupted", current_item="")
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
21
api/db/setting.py
Normal file
21
api/db/setting.py
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from api.db.connection import SessionLocal
|
||||||
|
from api.models.setting import Setting
|
||||||
|
|
||||||
|
|
||||||
|
async def get_all_settings() -> dict[str, str]:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
res = await session.execute(select(Setting))
|
||||||
|
return {s.key: s.value for s in res.scalars().all()}
|
||||||
|
|
||||||
|
|
||||||
|
async def set_settings(values: dict[str, str]) -> None:
|
||||||
|
async with SessionLocal() as session:
|
||||||
|
for key, value in values.items():
|
||||||
|
existing = await session.get(Setting, key)
|
||||||
|
if existing is None:
|
||||||
|
session.add(Setting(key=key, value=str(value)))
|
||||||
|
else:
|
||||||
|
existing.value = str(value)
|
||||||
|
await session.commit()
|
||||||
@@ -14,12 +14,14 @@ async def create_video(
|
|||||||
text_summary_path: str,
|
text_summary_path: str,
|
||||||
transcription_method: str,
|
transcription_method: str,
|
||||||
section_id: int | None = None,
|
section_id: int | None = None,
|
||||||
|
job_id: str | None = None,
|
||||||
) -> Video:
|
) -> Video:
|
||||||
async with SessionLocal() as session:
|
async with SessionLocal() as session:
|
||||||
v = Video(
|
v = Video(
|
||||||
uuid=uuid,
|
uuid=uuid,
|
||||||
source_url=source_url,
|
source_url=source_url,
|
||||||
section_id=section_id,
|
section_id=section_id,
|
||||||
|
job_id=job_id,
|
||||||
title=title or "",
|
title=title or "",
|
||||||
video_path=video_path or "",
|
video_path=video_path or "",
|
||||||
text_full_path=text_full_path or "",
|
text_full_path=text_full_path or "",
|
||||||
|
|||||||
119
api/lib/jobqueue.py
Normal file
119
api/lib/jobqueue.py
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
"""Очередь фоновых задач: одно видео или плейлист, элементы обрабатываются
|
||||||
|
последовательно (чтобы не долбить YouTube и LLM параллельными запросами).
|
||||||
|
|
||||||
|
Живой прогресс держится в памяти (_live) и подмешивается в GET /jobs;
|
||||||
|
устойчивое состояние (счётчики, статус) — в таблице jobs.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from api.db.job import get_job, update_job
|
||||||
|
from api.lib.processing import process_single_video
|
||||||
|
from api.lib.youtube import expand_url
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("jobs")
|
||||||
|
|
||||||
|
_queue: asyncio.Queue[str] = asyncio.Queue()
|
||||||
|
_live: dict[str, str] = {}
|
||||||
|
_cancel: set[str] = set()
|
||||||
|
_params: dict[str, dict] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def submit(job_id: str, params: dict) -> None:
|
||||||
|
_params[job_id] = params
|
||||||
|
_queue.put_nowait(job_id)
|
||||||
|
|
||||||
|
|
||||||
|
def live_status(job_id: str) -> str:
|
||||||
|
return _live.get(job_id, "")
|
||||||
|
|
||||||
|
|
||||||
|
def request_cancel(job_id: str) -> None:
|
||||||
|
_cancel.add(job_id)
|
||||||
|
|
||||||
|
|
||||||
|
def is_cancelled(job_id: str) -> bool:
|
||||||
|
return job_id in _cancel
|
||||||
|
|
||||||
|
|
||||||
|
async def worker() -> None:
|
||||||
|
"""Единственный воркер; запускается на старте приложения."""
|
||||||
|
logger.info("job worker started")
|
||||||
|
while True:
|
||||||
|
job_id = await _queue.get()
|
||||||
|
try:
|
||||||
|
await _run_job(job_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("job %s crashed", job_id)
|
||||||
|
try:
|
||||||
|
await update_job(job_id, status="error", current_item="")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
_live.pop(job_id, None)
|
||||||
|
_cancel.discard(job_id)
|
||||||
|
_params.pop(job_id, None)
|
||||||
|
_queue.task_done()
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_job(job_id: str) -> None:
|
||||||
|
job = await get_job(job_id)
|
||||||
|
if job is None or job.status == "cancelled":
|
||||||
|
return
|
||||||
|
params = _params.get(job_id, {})
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
await update_job(job_id, status="running", current_item="анализ ссылки…")
|
||||||
|
_live[job_id] = "анализ ссылки…"
|
||||||
|
|
||||||
|
plan = await loop.run_in_executor(None, expand_url, job.url)
|
||||||
|
entries = plan["entries"]
|
||||||
|
if not entries:
|
||||||
|
await update_job(job_id, status="error", error="Плейлист пуст", current_item="")
|
||||||
|
return
|
||||||
|
await update_job(
|
||||||
|
job_id, kind=plan["kind"], title=plan["title"], total_items=len(entries)
|
||||||
|
)
|
||||||
|
|
||||||
|
done = failed = 0
|
||||||
|
errors: list[str] = []
|
||||||
|
n = len(entries)
|
||||||
|
|
||||||
|
for i, entry in enumerate(entries, start=1):
|
||||||
|
if is_cancelled(job_id):
|
||||||
|
await update_job(job_id, status="cancelled", current_item="")
|
||||||
|
logger.info("job %s cancelled at item %d/%d", job_id, i, n)
|
||||||
|
return
|
||||||
|
|
||||||
|
label = f"{i}/{n} · {entry['title']}"
|
||||||
|
await update_job(job_id, current_item=label)
|
||||||
|
_live[job_id] = label
|
||||||
|
|
||||||
|
def _notify(msg: str, _label: str = label) -> None:
|
||||||
|
_live[job_id] = f"{_label} — {msg}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
await process_single_video(
|
||||||
|
entry["url"],
|
||||||
|
section_id=params.get("section_id"),
|
||||||
|
model_path=params.get("model_path"),
|
||||||
|
summary_max_sentences=params.get("summary_max_sentences"),
|
||||||
|
keep_video=params.get("keep_video"),
|
||||||
|
job_id=job_id,
|
||||||
|
progress=_notify,
|
||||||
|
)
|
||||||
|
done += 1
|
||||||
|
except Exception as e:
|
||||||
|
failed += 1
|
||||||
|
errors.append(f"{entry['title']}: {e}")
|
||||||
|
logger.warning("job %s item %d/%d failed: %s", job_id, i, n, e)
|
||||||
|
await update_job(
|
||||||
|
job_id, done_items=done, failed_items=failed, error="\n".join(errors)[:4000]
|
||||||
|
)
|
||||||
|
|
||||||
|
status = "error" if done == 0 and failed > 0 else "done"
|
||||||
|
await update_job(job_id, status=status, current_item="")
|
||||||
|
logger.info("job %s finished: %d ok, %d failed", job_id, done, failed)
|
||||||
229
api/lib/processing.py
Normal file
229
api/lib/processing.py
Normal file
@@ -0,0 +1,229 @@
|
|||||||
|
"""Ядро пайплайна: одно видео от URL до записи в БД.
|
||||||
|
|
||||||
|
Используется и синхронным роутом /pipeline/process, и фоновыми задачами /jobs.
|
||||||
|
Все дефолты (LLM, длина выжимки, хранение видео, Vosk-модель) берутся из
|
||||||
|
настроек в БД (таблица settings) с фолбэком на переменные окружения.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import uuid as _uuid
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable, Optional
|
||||||
|
|
||||||
|
from api.config import settings
|
||||||
|
from api.db.setting import get_all_settings
|
||||||
|
from api.db.video import create_video
|
||||||
|
from api.lib.llm import LLMConfig
|
||||||
|
from api.lib.summarize import summarize
|
||||||
|
from api.lib.transcribe import transcribe_via_ffmpeg
|
||||||
|
from api.lib.youtube import download_video_with_subs, vtt_to_text
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("processing")
|
||||||
|
|
||||||
|
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||||
|
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
|
||||||
|
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
|
||||||
|
_MODELS_DIR = _PROJECT_ROOT / "models"
|
||||||
|
|
||||||
|
|
||||||
|
class PipelineError(RuntimeError):
|
||||||
|
def __init__(self, message: str, status: int = 500):
|
||||||
|
super().__init__(message)
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
async def effective_defaults() -> dict:
|
||||||
|
"""Настройки из БД поверх переменных окружения."""
|
||||||
|
db = await get_all_settings()
|
||||||
|
base_url = db.get("llm_base_url") or settings.LLM_BASE_URL
|
||||||
|
model = db.get("llm_model") or settings.LLM_MODEL
|
||||||
|
llm_cfg = None
|
||||||
|
if base_url and model:
|
||||||
|
llm_cfg = LLMConfig(
|
||||||
|
base_url=base_url,
|
||||||
|
api_key=db.get("llm_api_key") or settings.LLM_API_KEY,
|
||||||
|
model=model,
|
||||||
|
timeout=settings.LLM_TIMEOUT,
|
||||||
|
max_tokens=settings.LLM_MAX_TOKENS,
|
||||||
|
temperature=settings.LLM_TEMPERATURE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
max_sentences = int(db.get("summary_max_sentences") or 15)
|
||||||
|
except ValueError:
|
||||||
|
max_sentences = 15
|
||||||
|
return {
|
||||||
|
"llm_cfg": llm_cfg,
|
||||||
|
"summary_max_sentences": max_sentences,
|
||||||
|
"keep_video": db.get("keep_video", "1") != "0",
|
||||||
|
"model_path": db.get("vosk_model_path") or settings.DEFAULT_VOSK_MODEL,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_model_path(raw: str) -> Path:
|
||||||
|
p = Path(raw)
|
||||||
|
if not p.is_absolute():
|
||||||
|
p = _PROJECT_ROOT / raw
|
||||||
|
p = p.resolve()
|
||||||
|
try:
|
||||||
|
p.relative_to(_MODELS_DIR.resolve())
|
||||||
|
except ValueError:
|
||||||
|
raise PipelineError(
|
||||||
|
"model_path must point inside the project's models directory", status=400
|
||||||
|
)
|
||||||
|
if not p.exists():
|
||||||
|
raise PipelineError(f"Model not found: {p}", status=400)
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
async def process_single_video(
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
model_path: str | None = None,
|
||||||
|
summary_max_sentences: int | None = None,
|
||||||
|
keep_video: bool | None = None,
|
||||||
|
section_id: int | None = None,
|
||||||
|
job_id: str | None = None,
|
||||||
|
llm_cfg: Optional[LLMConfig] = None,
|
||||||
|
system_prompt: str | None = None,
|
||||||
|
user_template: str | None = None,
|
||||||
|
progress: Optional[Callable[[str], None]] = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Скачивает, расшифровывает, сокращает и сохраняет одно видео.
|
||||||
|
|
||||||
|
Параметры со значением None подтягиваются из настроек (БД/env).
|
||||||
|
progress(msg) — живой статус для UI задач.
|
||||||
|
"""
|
||||||
|
notify = progress or (lambda msg: None)
|
||||||
|
defaults = await effective_defaults()
|
||||||
|
if llm_cfg is None:
|
||||||
|
llm_cfg = defaults["llm_cfg"]
|
||||||
|
if summary_max_sentences is None:
|
||||||
|
summary_max_sentences = defaults["summary_max_sentences"]
|
||||||
|
if keep_video is None:
|
||||||
|
keep_video = defaults["keep_video"]
|
||||||
|
raw_model_path = model_path or defaults["model_path"]
|
||||||
|
|
||||||
|
video_uuid = _uuid.uuid4().hex
|
||||||
|
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
logger.info("[%s] downloading %s (keep_video=%s)", video_uuid, url, keep_video)
|
||||||
|
notify("скачивание…")
|
||||||
|
try:
|
||||||
|
info = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: download_video_with_subs(
|
||||||
|
url,
|
||||||
|
video_uuid,
|
||||||
|
_VIDEO_DIR,
|
||||||
|
settings.subtitle_langs_list,
|
||||||
|
# видео не сохраняем — достаточно аудио (для лекций в разы быстрее)
|
||||||
|
audio_only=not keep_video,
|
||||||
|
on_progress=lambda pct: notify(f"скачивание {pct}%"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise PipelineError(f"Download failed: {e}")
|
||||||
|
|
||||||
|
title: str = info["title"]
|
||||||
|
video_path: Path = info["video_path"]
|
||||||
|
sub_path: Path | None = info["subtitle_path"]
|
||||||
|
|
||||||
|
if not video_path.exists():
|
||||||
|
raise PipelineError("Video file missing after download")
|
||||||
|
|
||||||
|
full_text = ""
|
||||||
|
method = ""
|
||||||
|
|
||||||
|
if sub_path is not None and sub_path.exists():
|
||||||
|
try:
|
||||||
|
content = sub_path.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
full_text = vtt_to_text(content)
|
||||||
|
if full_text.strip():
|
||||||
|
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
|
||||||
|
logger.info("[%s] using subtitles (%s)", video_uuid, method)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
|
||||||
|
full_text = ""
|
||||||
|
|
||||||
|
if not full_text.strip():
|
||||||
|
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
|
||||||
|
notify("распознавание речи (Vosk)…")
|
||||||
|
resolved_model = resolve_model_path(raw_model_path)
|
||||||
|
try:
|
||||||
|
full_text = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: transcribe_via_ffmpeg(
|
||||||
|
video_path,
|
||||||
|
resolved_model,
|
||||||
|
on_progress=lambda m: notify(f"распознано {m} мин аудио"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
method = f"vosk:{resolved_model.name}"
|
||||||
|
except Exception as e:
|
||||||
|
raise PipelineError(f"Transcription failed: {e}")
|
||||||
|
|
||||||
|
if not full_text.strip():
|
||||||
|
raise PipelineError("No subtitles and transcription returned empty text")
|
||||||
|
|
||||||
|
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
|
||||||
|
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
|
||||||
|
text_full_path.write_text(full_text, encoding="utf-8")
|
||||||
|
|
||||||
|
notify("выжимка (LLM)…" if llm_cfg else "выжимка…")
|
||||||
|
try:
|
||||||
|
summary = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: summarize(
|
||||||
|
full_text,
|
||||||
|
llm_cfg=llm_cfg,
|
||||||
|
system_prompt=system_prompt,
|
||||||
|
user_template=user_template,
|
||||||
|
max_sentences=summary_max_sentences,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise PipelineError(f"Summarization failed: {e}")
|
||||||
|
text_summary_path.write_text(summary, encoding="utf-8")
|
||||||
|
|
||||||
|
if keep_video:
|
||||||
|
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
|
||||||
|
else:
|
||||||
|
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
|
||||||
|
video_path.unlink(missing_ok=True)
|
||||||
|
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
|
||||||
|
p.unlink(missing_ok=True)
|
||||||
|
rel_video = ""
|
||||||
|
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
|
||||||
|
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
|
||||||
|
|
||||||
|
try:
|
||||||
|
await create_video(
|
||||||
|
uuid=video_uuid,
|
||||||
|
source_url=url,
|
||||||
|
title=title,
|
||||||
|
video_path=rel_video,
|
||||||
|
text_full_path=rel_full,
|
||||||
|
text_summary_path=rel_summary,
|
||||||
|
transcription_method=method,
|
||||||
|
section_id=section_id,
|
||||||
|
job_id=job_id,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise PipelineError(f"DB insert failed: {e}")
|
||||||
|
|
||||||
|
logger.info("[%s] done (method=%s)", video_uuid, method)
|
||||||
|
return {
|
||||||
|
"uuid": video_uuid,
|
||||||
|
"title": title,
|
||||||
|
"source_url": url,
|
||||||
|
"video_path": rel_video,
|
||||||
|
"text_full_path": rel_full,
|
||||||
|
"text_summary_path": rel_summary,
|
||||||
|
"transcription_method": method,
|
||||||
|
"summary_preview": summary[:300],
|
||||||
|
}
|
||||||
@@ -65,7 +65,7 @@ DEFAULT_SYSTEM_PROMPT = (
|
|||||||
|
|
||||||
DEFAULT_USER_TEMPLATE = (
|
DEFAULT_USER_TEMPLATE = (
|
||||||
"Сделай связную краткую выжимку следующего текста. "
|
"Сделай связную краткую выжимку следующего текста. "
|
||||||
"Выдели основные тезисы и логику изложения. Объём — 5–12 предложений.\n\n"
|
"Выдели основные тезисы и логику изложения. Объём — примерно {n} предложений.\n\n"
|
||||||
"---\n{text}\n---"
|
"---\n{text}\n---"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -154,6 +154,7 @@ def summarize_llm(
|
|||||||
user_template: Optional[str] = None,
|
user_template: Optional[str] = None,
|
||||||
reduce_template: Optional[str] = None,
|
reduce_template: Optional[str] = None,
|
||||||
chunk_chars: int = 10000,
|
chunk_chars: int = 10000,
|
||||||
|
max_sentences: int = 15,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
|
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
|
||||||
text = (text or "").strip()
|
text = (text or "").strip()
|
||||||
@@ -164,12 +165,19 @@ def summarize_llm(
|
|||||||
user_t = user_template or DEFAULT_USER_TEMPLATE
|
user_t = user_template or DEFAULT_USER_TEMPLATE
|
||||||
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
|
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
|
||||||
|
|
||||||
|
def _fmt(template: str, chunk: str) -> str:
|
||||||
|
try:
|
||||||
|
return template.format(text=chunk, n=max_sentences)
|
||||||
|
except (KeyError, IndexError):
|
||||||
|
# пользовательский шаблон без {n} / с лишними скобками
|
||||||
|
return template.replace("{text}", chunk)
|
||||||
|
|
||||||
def _one(chunk: str, template: str) -> str:
|
def _one(chunk: str, template: str) -> str:
|
||||||
return chat_complete(
|
return chat_complete(
|
||||||
cfg,
|
cfg,
|
||||||
[
|
[
|
||||||
{"role": "system", "content": sys_p},
|
{"role": "system", "content": sys_p},
|
||||||
{"role": "user", "content": template.format(text=chunk)},
|
{"role": "user", "content": _fmt(template, chunk)},
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -194,6 +202,7 @@ def summarize_llm(
|
|||||||
user_template=reduce_t,
|
user_template=reduce_t,
|
||||||
reduce_template=reduce_t,
|
reduce_template=reduce_t,
|
||||||
chunk_chars=chunk_chars,
|
chunk_chars=chunk_chars,
|
||||||
|
max_sentences=max_sentences,
|
||||||
)
|
)
|
||||||
logger.info("LLM reduce step over %d partials", len(partials))
|
logger.info("LLM reduce step over %d partials", len(partials))
|
||||||
return _one(combined, reduce_t)
|
return _one(combined, reduce_t)
|
||||||
@@ -215,6 +224,7 @@ def summarize(
|
|||||||
llm_cfg,
|
llm_cfg,
|
||||||
system_prompt=system_prompt,
|
system_prompt=system_prompt,
|
||||||
user_template=user_template,
|
user_template=user_template,
|
||||||
|
max_sentences=max_sentences,
|
||||||
)
|
)
|
||||||
except LLMError as e:
|
except LLMError as e:
|
||||||
logger.warning("LLM summarization failed, falling back to extractive: %s", e)
|
logger.warning("LLM summarization failed, falling back to extractive: %s", e)
|
||||||
|
|||||||
@@ -53,8 +53,16 @@ class VoskModelManager:
|
|||||||
t.join(timeout=2)
|
t.join(timeout=2)
|
||||||
|
|
||||||
|
|
||||||
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
|
def transcribe_via_ffmpeg(
|
||||||
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
|
media_path: Path,
|
||||||
|
model_path: Path,
|
||||||
|
sample_rate: int = 16000,
|
||||||
|
on_progress=None,
|
||||||
|
) -> str:
|
||||||
|
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks.
|
||||||
|
|
||||||
|
on_progress: callback(minutes: int) — вызывается каждые ~5 минут распознанного аудио.
|
||||||
|
"""
|
||||||
if Model is None or KaldiRecognizer is None:
|
if Model is None or KaldiRecognizer is None:
|
||||||
raise RuntimeError("vosk is not installed in runtime")
|
raise RuntimeError("vosk is not installed in runtime")
|
||||||
if shutil.which("ffmpeg") is None:
|
if shutil.which("ffmpeg") is None:
|
||||||
@@ -103,10 +111,16 @@ def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int =
|
|||||||
break
|
break
|
||||||
recognizer.AcceptWaveform(chunk)
|
recognizer.AcceptWaveform(chunk)
|
||||||
bytes_read += len(chunk)
|
bytes_read += len(chunk)
|
||||||
mb = bytes_read // (1024 * 1024)
|
# PCM s16le mono: sample_rate * 2 байта в секунду
|
||||||
if mb >= last_logged_mb + 5:
|
minutes = bytes_read // (sample_rate * 2 * 60)
|
||||||
last_logged_mb = mb
|
if minutes >= last_logged_mb + 5:
|
||||||
logger.info("Transcribed %d MB of audio so far", mb)
|
last_logged_mb = minutes
|
||||||
|
logger.info("Transcribed %d minutes of audio so far", minutes)
|
||||||
|
if on_progress is not None:
|
||||||
|
try:
|
||||||
|
on_progress(minutes)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
result = json.loads(recognizer.FinalResult())
|
result = json.loads(recognizer.FinalResult())
|
||||||
text = result.get("text", "")
|
text = result.get("text", "")
|
||||||
|
|||||||
@@ -12,7 +12,44 @@ import yt_dlp
|
|||||||
logger = logging.getLogger("youtube")
|
logger = logging.getLogger("youtube")
|
||||||
|
|
||||||
|
|
||||||
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"}
|
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov", ".opus", ".ogg", ".aac"}
|
||||||
|
|
||||||
|
|
||||||
|
def expand_url(url: str) -> dict:
|
||||||
|
"""Определяет, видео это или плейлист, и возвращает список элементов.
|
||||||
|
|
||||||
|
Returns: {"kind": "video"|"playlist", "title": str,
|
||||||
|
"entries": [{"url": str, "title": str}, ...]}
|
||||||
|
"""
|
||||||
|
opts = {
|
||||||
|
"extract_flat": "in_playlist",
|
||||||
|
"quiet": True,
|
||||||
|
"no_warnings": True,
|
||||||
|
"skip_download": True,
|
||||||
|
}
|
||||||
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||||
|
info = ydl.extract_info(url, download=False)
|
||||||
|
|
||||||
|
if info.get("_type") == "playlist":
|
||||||
|
entries = []
|
||||||
|
for e in info.get("entries") or []:
|
||||||
|
if not e:
|
||||||
|
continue
|
||||||
|
u = e.get("url") or ""
|
||||||
|
if u and not u.startswith("http"):
|
||||||
|
u = f"https://www.youtube.com/watch?v={u}"
|
||||||
|
if not u and e.get("id"):
|
||||||
|
u = f"https://www.youtube.com/watch?v={e['id']}"
|
||||||
|
if u:
|
||||||
|
entries.append({"url": u, "title": e.get("title") or u})
|
||||||
|
return {
|
||||||
|
"kind": "playlist",
|
||||||
|
"title": info.get("title") or url,
|
||||||
|
"entries": entries,
|
||||||
|
}
|
||||||
|
|
||||||
|
title = info.get("title") or url
|
||||||
|
return {"kind": "video", "title": title, "entries": [{"url": url, "title": title}]}
|
||||||
|
|
||||||
|
|
||||||
def download_video_with_subs(
|
def download_video_with_subs(
|
||||||
@@ -20,9 +57,15 @@ def download_video_with_subs(
|
|||||||
video_id: str,
|
video_id: str,
|
||||||
video_dir: Path,
|
video_dir: Path,
|
||||||
langs: Optional[list[str]] = None,
|
langs: Optional[list[str]] = None,
|
||||||
|
audio_only: bool = False,
|
||||||
|
on_progress=None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Download a single video, also requesting subtitles for the given languages.
|
"""Download a single video, also requesting subtitles for the given languages.
|
||||||
|
|
||||||
|
audio_only: качает только аудиодорожку (для длинных лекций, когда видео
|
||||||
|
не сохраняется — в разы быстрее и меньше по объёму).
|
||||||
|
on_progress: callback(percent: int), вызывается с шагом ~5%.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
{
|
{
|
||||||
"title": str,
|
"title": str,
|
||||||
@@ -40,11 +83,34 @@ def download_video_with_subs(
|
|||||||
|
|
||||||
base_opts = {
|
base_opts = {
|
||||||
"outtmpl": outtmpl,
|
"outtmpl": outtmpl,
|
||||||
"format": "bv*+ba/b",
|
|
||||||
"merge_output_format": "mp4",
|
|
||||||
"quiet": True,
|
"quiet": True,
|
||||||
"no_warnings": True,
|
"no_warnings": True,
|
||||||
|
"noplaylist": True, # ссылка с list= в задаче на одно видео не тянет весь плейлист
|
||||||
}
|
}
|
||||||
|
if audio_only:
|
||||||
|
base_opts["format"] = "ba/b"
|
||||||
|
else:
|
||||||
|
base_opts["format"] = "bv*+ba/b"
|
||||||
|
base_opts["merge_output_format"] = "mp4"
|
||||||
|
|
||||||
|
if on_progress is not None:
|
||||||
|
state = {"last": -5}
|
||||||
|
|
||||||
|
def _hook(dd: dict):
|
||||||
|
if dd.get("status") != "downloading":
|
||||||
|
return
|
||||||
|
total = dd.get("total_bytes") or dd.get("total_bytes_estimate")
|
||||||
|
if not total:
|
||||||
|
return
|
||||||
|
pct = int(dd.get("downloaded_bytes", 0) * 100 / total)
|
||||||
|
if pct >= state["last"] + 5:
|
||||||
|
state["last"] = pct
|
||||||
|
try:
|
||||||
|
on_progress(pct)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
base_opts["progress_hooks"] = [_hook]
|
||||||
sub_opts = {
|
sub_opts = {
|
||||||
**base_opts,
|
**base_opts,
|
||||||
"writesubtitles": True,
|
"writesubtitles": True,
|
||||||
|
|||||||
12
api/main.py
12
api/main.py
@@ -1,10 +1,13 @@
|
|||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
from api.db.connection import wait_for_postgres, init_db
|
from api.db.connection import wait_for_postgres, init_db
|
||||||
from api.route import default, llm, pipeline, sections, videos
|
from api.db.job import mark_interrupted
|
||||||
|
from api.lib import jobqueue
|
||||||
|
from api.route import app_settings, default, jobs, llm, pipeline, sections, videos
|
||||||
|
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -17,13 +20,18 @@ logging.basicConfig(
|
|||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
await wait_for_postgres()
|
await wait_for_postgres()
|
||||||
await init_db()
|
await init_db()
|
||||||
|
await mark_interrupted()
|
||||||
|
worker_task = asyncio.create_task(jobqueue.worker())
|
||||||
yield
|
yield
|
||||||
|
worker_task.cancel()
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan)
|
app = FastAPI(title="LLM-infa API", version="0.4.0", lifespan=lifespan)
|
||||||
|
|
||||||
app.include_router(default.router)
|
app.include_router(default.router)
|
||||||
app.include_router(pipeline.router)
|
app.include_router(pipeline.router)
|
||||||
|
app.include_router(jobs.router)
|
||||||
app.include_router(videos.router)
|
app.include_router(videos.router)
|
||||||
app.include_router(sections.router)
|
app.include_router(sections.router)
|
||||||
|
app.include_router(app_settings.router)
|
||||||
app.include_router(llm.router)
|
app.include_router(llm.router)
|
||||||
|
|||||||
59
api/route/app_settings.py
Normal file
59
api/route/app_settings.py
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres,
|
||||||
|
переживают перезапуск контейнеров и перезагрузку страницы.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from api.db.setting import get_all_settings, set_settings
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||||
|
|
||||||
|
_KEYS = (
|
||||||
|
"llm_base_url",
|
||||||
|
"llm_api_key",
|
||||||
|
"llm_model",
|
||||||
|
"summary_max_sentences",
|
||||||
|
"keep_video",
|
||||||
|
"vosk_model_path",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SettingsPayload(BaseModel):
|
||||||
|
llm_base_url: str | None = None
|
||||||
|
llm_api_key: str | None = None
|
||||||
|
llm_model: str | None = None
|
||||||
|
summary_max_sentences: int | None = None
|
||||||
|
keep_video: bool | None = None
|
||||||
|
vosk_model_path: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/")
|
||||||
|
async def get_settings():
|
||||||
|
db = await get_all_settings()
|
||||||
|
return {
|
||||||
|
"llm_base_url": db.get("llm_base_url", ""),
|
||||||
|
"llm_api_key": db.get("llm_api_key", ""),
|
||||||
|
"llm_model": db.get("llm_model", ""),
|
||||||
|
"summary_max_sentences": int(db.get("summary_max_sentences") or 15),
|
||||||
|
"keep_video": db.get("keep_video", "1") != "0",
|
||||||
|
"vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/")
|
||||||
|
async def put_settings(body: SettingsPayload):
|
||||||
|
values: dict[str, str] = {}
|
||||||
|
for key in _KEYS:
|
||||||
|
v = getattr(body, key)
|
||||||
|
if v is None:
|
||||||
|
continue
|
||||||
|
if key == "keep_video":
|
||||||
|
values[key] = "1" if v else "0"
|
||||||
|
else:
|
||||||
|
values[key] = str(v)
|
||||||
|
if values:
|
||||||
|
await set_settings(values)
|
||||||
|
return await get_settings()
|
||||||
112
api/route/jobs.py
Normal file
112
api/route/jobs.py
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
"""Фоновые задачи: видео или плейлист любой длины.
|
||||||
|
|
||||||
|
POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid as _uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from api.db.job import create_job, delete_job, get_job, list_jobs
|
||||||
|
from api.lib import jobqueue
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||||
|
|
||||||
|
|
||||||
|
class JobIn(BaseModel):
|
||||||
|
url: str
|
||||||
|
section_id: int | None = None
|
||||||
|
# None → берутся из сохранённых настроек
|
||||||
|
keep_video: bool | None = None
|
||||||
|
summary_max_sentences: int | None = None
|
||||||
|
model_path: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class JobOut(BaseModel):
|
||||||
|
id: str
|
||||||
|
url: str
|
||||||
|
kind: str
|
||||||
|
title: str
|
||||||
|
status: str
|
||||||
|
total_items: int
|
||||||
|
done_items: int
|
||||||
|
failed_items: int
|
||||||
|
current_item: str
|
||||||
|
live: str
|
||||||
|
error: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
def _to_out(j) -> JobOut:
|
||||||
|
return JobOut(
|
||||||
|
id=j.id,
|
||||||
|
url=j.url,
|
||||||
|
kind=j.kind,
|
||||||
|
title=j.title,
|
||||||
|
status=j.status,
|
||||||
|
total_items=j.total_items,
|
||||||
|
done_items=j.done_items,
|
||||||
|
failed_items=j.failed_items,
|
||||||
|
current_item=j.current_item,
|
||||||
|
live=jobqueue.live_status(j.id) or j.current_item,
|
||||||
|
error=j.error,
|
||||||
|
created_at=j.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/", response_model=JobOut)
|
||||||
|
async def create(body: JobIn):
|
||||||
|
url = body.url.strip()
|
||||||
|
if not url:
|
||||||
|
raise HTTPException(status_code=400, detail="URL is empty")
|
||||||
|
job_id = _uuid.uuid4().hex
|
||||||
|
j = await create_job(job_id, url)
|
||||||
|
jobqueue.submit(
|
||||||
|
job_id,
|
||||||
|
{
|
||||||
|
"section_id": body.section_id,
|
||||||
|
"keep_video": body.keep_video,
|
||||||
|
"summary_max_sentences": body.summary_max_sentences,
|
||||||
|
"model_path": body.model_path,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return _to_out(j)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/", response_model=list[JobOut])
|
||||||
|
async def list_all(limit: int = 50):
|
||||||
|
return [_to_out(j) for j in await list_jobs(limit)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}", response_model=JobOut)
|
||||||
|
async def get_one(job_id: str):
|
||||||
|
j = await get_job(job_id)
|
||||||
|
if j is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
return _to_out(j)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{job_id}/cancel")
|
||||||
|
async def cancel(job_id: str):
|
||||||
|
j = await get_job(job_id)
|
||||||
|
if j is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
if j.status not in ("queued", "running"):
|
||||||
|
raise HTTPException(status_code=400, detail=f"Job is {j.status}")
|
||||||
|
jobqueue.request_cancel(job_id)
|
||||||
|
return {"ok": True, "note": "остановится после текущего элемента"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{job_id}")
|
||||||
|
async def delete(job_id: str):
|
||||||
|
j = await get_job(job_id)
|
||||||
|
if j is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Job not found")
|
||||||
|
if j.status in ("queued", "running"):
|
||||||
|
raise HTTPException(status_code=400, detail="Сначала отмените задачу")
|
||||||
|
await delete_job(job_id)
|
||||||
|
return {"ok": True}
|
||||||
@@ -1,29 +1,16 @@
|
|||||||
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist."""
|
"""Синхронный запуск пайплайна для одного видео (для длинных видео и
|
||||||
|
плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым).
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
import uuid as _uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, HTTPException
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from api.config import settings
|
from api.lib.llm import LLMConfig
|
||||||
from api.db.video import create_video
|
from api.lib.processing import PipelineError, process_single_video
|
||||||
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
|
|
||||||
from api.lib.summarize import summarize
|
|
||||||
from api.lib.transcribe import transcribe_via_ffmpeg
|
|
||||||
from api.lib.youtube import download_video_with_subs, vtt_to_text
|
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
|
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
|
||||||
logger = logging.getLogger("pipeline")
|
|
||||||
|
|
||||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
||||||
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
|
|
||||||
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
|
|
||||||
_MODELS_DIR = _PROJECT_ROOT / "models"
|
|
||||||
|
|
||||||
|
|
||||||
class LLMOverride(BaseModel):
|
class LLMOverride(BaseModel):
|
||||||
@@ -40,12 +27,12 @@ class LLMOverride(BaseModel):
|
|||||||
|
|
||||||
class ProcessRequest(BaseModel):
|
class ProcessRequest(BaseModel):
|
||||||
url: str
|
url: str
|
||||||
model_path: str | None = None # path inside models/, used when subtitles are unavailable
|
model_path: str | None = None # None → из настроек
|
||||||
summary_max_sentences: int = 15
|
summary_max_sentences: int | None = None
|
||||||
keep_video: bool = True # False: файл видео удаляется после расшифровки
|
keep_video: bool | None = None
|
||||||
section_id: int | None = None
|
section_id: int | None = None
|
||||||
# If supplied, this LLM is used for the summary; otherwise env-configured LLM;
|
# Если задано — используется этот LLM; иначе сохранённые настройки/env;
|
||||||
# otherwise the built-in extractive fallback.
|
# иначе встроенная экстрактивная выжимка.
|
||||||
llm: LLMOverride | None = None
|
llm: LLMOverride | None = None
|
||||||
|
|
||||||
|
|
||||||
@@ -60,86 +47,10 @@ class ProcessResponse(BaseModel):
|
|||||||
summary_preview: str
|
summary_preview: str
|
||||||
|
|
||||||
|
|
||||||
def _resolve_model_path(req_model_path: str | None) -> Path:
|
|
||||||
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
|
|
||||||
p = Path(raw)
|
|
||||||
if not p.is_absolute():
|
|
||||||
p = _PROJECT_ROOT / raw
|
|
||||||
p = p.resolve()
|
|
||||||
try:
|
|
||||||
p.relative_to(_MODELS_DIR.resolve())
|
|
||||||
except ValueError:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=400,
|
|
||||||
detail="model_path must point inside the project's models directory",
|
|
||||||
)
|
|
||||||
if not p.exists():
|
|
||||||
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/process", response_model=ProcessResponse)
|
@router.post("/process", response_model=ProcessResponse)
|
||||||
async def process_video(req: ProcessRequest):
|
async def process_video(req: ProcessRequest):
|
||||||
video_uuid = _uuid.uuid4().hex
|
llm_cfg = None
|
||||||
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
|
sys_p = user_t = None
|
||||||
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
|
||||||
|
|
||||||
logger.info("[%s] downloading %s", video_uuid, req.url)
|
|
||||||
try:
|
|
||||||
info = await loop.run_in_executor(
|
|
||||||
None,
|
|
||||||
download_video_with_subs,
|
|
||||||
req.url,
|
|
||||||
video_uuid,
|
|
||||||
_VIDEO_DIR,
|
|
||||||
settings.subtitle_langs_list,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
|
|
||||||
|
|
||||||
title: str = info["title"]
|
|
||||||
video_path: Path = info["video_path"]
|
|
||||||
sub_path: Path | None = info["subtitle_path"]
|
|
||||||
|
|
||||||
if not video_path.exists():
|
|
||||||
raise HTTPException(status_code=500, detail="Video file missing after download")
|
|
||||||
|
|
||||||
full_text = ""
|
|
||||||
method = ""
|
|
||||||
|
|
||||||
if sub_path is not None and sub_path.exists():
|
|
||||||
try:
|
|
||||||
content = sub_path.read_text(encoding="utf-8", errors="ignore")
|
|
||||||
full_text = vtt_to_text(content)
|
|
||||||
if full_text.strip():
|
|
||||||
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
|
|
||||||
logger.info("[%s] using subtitles (%s)", video_uuid, method)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
|
|
||||||
full_text = ""
|
|
||||||
|
|
||||||
if not full_text.strip():
|
|
||||||
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
|
|
||||||
model_path = _resolve_model_path(req.model_path)
|
|
||||||
try:
|
|
||||||
full_text = await loop.run_in_executor(
|
|
||||||
None, transcribe_via_ffmpeg, video_path, model_path
|
|
||||||
)
|
|
||||||
method = f"vosk:{model_path.name}"
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
|
|
||||||
|
|
||||||
if not full_text.strip():
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=500, detail="No subtitles and transcription returned empty text"
|
|
||||||
)
|
|
||||||
|
|
||||||
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
|
|
||||||
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
|
|
||||||
text_full_path.write_text(full_text, encoding="utf-8")
|
|
||||||
|
|
||||||
if req.llm is not None:
|
if req.llm is not None:
|
||||||
llm_cfg = LLMConfig(
|
llm_cfg = LLMConfig(
|
||||||
base_url=req.llm.base_url,
|
base_url=req.llm.base_url,
|
||||||
@@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest):
|
|||||||
)
|
)
|
||||||
sys_p = req.llm.system_prompt
|
sys_p = req.llm.system_prompt
|
||||||
user_t = req.llm.user_template
|
user_t = req.llm.user_template
|
||||||
else:
|
|
||||||
llm_cfg = llm_from_settings()
|
|
||||||
sys_p = None
|
|
||||||
user_t = None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
summary = await loop.run_in_executor(
|
result = await process_single_video(
|
||||||
None,
|
req.url,
|
||||||
lambda: summarize(
|
model_path=req.model_path,
|
||||||
full_text,
|
summary_max_sentences=req.summary_max_sentences,
|
||||||
|
keep_video=req.keep_video,
|
||||||
|
section_id=req.section_id,
|
||||||
llm_cfg=llm_cfg,
|
llm_cfg=llm_cfg,
|
||||||
system_prompt=sys_p,
|
system_prompt=sys_p,
|
||||||
user_template=user_t,
|
user_template=user_t,
|
||||||
max_sentences=req.summary_max_sentences,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
|
|
||||||
text_summary_path.write_text(summary, encoding="utf-8")
|
|
||||||
|
|
||||||
if req.keep_video:
|
|
||||||
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
|
|
||||||
else:
|
|
||||||
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
|
|
||||||
video_path.unlink(missing_ok=True)
|
|
||||||
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
|
|
||||||
p.unlink(missing_ok=True)
|
|
||||||
rel_video = ""
|
|
||||||
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
|
|
||||||
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
|
|
||||||
|
|
||||||
try:
|
|
||||||
await create_video(
|
|
||||||
uuid=video_uuid,
|
|
||||||
source_url=req.url,
|
|
||||||
title=title,
|
|
||||||
video_path=rel_video,
|
|
||||||
text_full_path=rel_full,
|
|
||||||
text_summary_path=rel_summary,
|
|
||||||
transcription_method=method,
|
|
||||||
section_id=req.section_id,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")
|
|
||||||
|
|
||||||
logger.info("[%s] done (method=%s)", video_uuid, method)
|
|
||||||
|
|
||||||
return ProcessResponse(
|
|
||||||
uuid=video_uuid,
|
|
||||||
title=title,
|
|
||||||
source_url=req.url,
|
|
||||||
video_path=rel_video,
|
|
||||||
text_full_path=rel_full,
|
|
||||||
text_summary_path=rel_summary,
|
|
||||||
transcription_method=method,
|
|
||||||
summary_preview=summary[:300],
|
|
||||||
)
|
)
|
||||||
|
except PipelineError as e:
|
||||||
|
raise HTTPException(status_code=e.status, detail=str(e))
|
||||||
|
return ProcessResponse(**result)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "llm-infa"
|
name = "llm-infa"
|
||||||
version = "0.3.0"
|
version = "0.4.0"
|
||||||
description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres"
|
description = "YouTube -> транскрипция -> LLM-выжимка -> Postgres"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.14"
|
requires-python = ">=3.14"
|
||||||
|
|||||||
@@ -21,10 +21,15 @@ class ApiClient:
|
|||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
self._client.close()
|
self._client.close()
|
||||||
|
|
||||||
# --- LLM ---
|
# --- Settings (persisted server-side in Postgres) ---
|
||||||
|
|
||||||
def llm_status(self) -> dict:
|
def get_settings(self) -> dict:
|
||||||
return self._get("/llm/config")
|
return self._get("/settings/")
|
||||||
|
|
||||||
|
def save_settings(self, values: dict) -> dict:
|
||||||
|
return self._request("PUT", "/settings/", values)
|
||||||
|
|
||||||
|
# --- LLM ---
|
||||||
|
|
||||||
def llm_models(self, base_url: str, api_key: str) -> list[dict]:
|
def llm_models(self, base_url: str, api_key: str) -> list[dict]:
|
||||||
data = self._post(
|
data = self._post(
|
||||||
@@ -39,10 +44,21 @@ class ApiClient:
|
|||||||
{"base_url": base_url, "api_key": api_key, "model": model},
|
{"base_url": base_url, "api_key": api_key, "model": model},
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Pipeline / videos ---
|
# --- Jobs (видео и плейлисты, в фоне) ---
|
||||||
|
|
||||||
def pipeline_process(self, payload: dict) -> dict:
|
def create_job(self, payload: dict) -> dict:
|
||||||
return self._post("/pipeline/process", payload)
|
return self._post("/jobs/", payload)
|
||||||
|
|
||||||
|
def list_jobs(self) -> list[dict]:
|
||||||
|
return self._get("/jobs/")
|
||||||
|
|
||||||
|
def cancel_job(self, job_id: str) -> dict:
|
||||||
|
return self._post(f"/jobs/{job_id}/cancel", None)
|
||||||
|
|
||||||
|
def delete_job(self, job_id: str) -> dict:
|
||||||
|
return self._request("DELETE", f"/jobs/{job_id}")
|
||||||
|
|
||||||
|
# --- Videos ---
|
||||||
|
|
||||||
def list_videos(
|
def list_videos(
|
||||||
self,
|
self,
|
||||||
|
|||||||
12
web/main.py
12
web/main.py
@@ -19,20 +19,14 @@ def main(page: ft.Page) -> None:
|
|||||||
view = MainView(page, client)
|
view = MainView(page, client)
|
||||||
page.add(view.build())
|
page.add(view.build())
|
||||||
|
|
||||||
# probe env-configured LLM and initial library load
|
# настройки живут на сервере (БД) — заполняем поля сохранёнными значениями
|
||||||
try:
|
try:
|
||||||
status = client.llm_status()
|
view.apply_settings(client.get_settings())
|
||||||
if status.get("configured"):
|
|
||||||
view._set_status(ok=True)
|
|
||||||
if status.get("base_url") and not view.base_url.value:
|
|
||||||
view.base_url.value = status["base_url"]
|
|
||||||
if status.get("model") and not view.selected_model:
|
|
||||||
view.selected_model = status["model"]
|
|
||||||
view.model_input.value = status["model"]
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
view._refresh_library()
|
view._refresh_library()
|
||||||
|
view._start_jobs_poll()
|
||||||
page.update()
|
page.update()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
"""Main page of the LLM-infa Flet web app."""
|
"""Main page of the LLM-infa Flet web app: вкладки «Главная» и «Настройки»."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
import flet as ft
|
import flet as ft
|
||||||
|
|
||||||
@@ -9,6 +10,16 @@ from web.api_client import ApiClient, ApiError
|
|||||||
import web.designer as d
|
import web.designer as d
|
||||||
|
|
||||||
|
|
||||||
|
_JOB_STATUS = {
|
||||||
|
"queued": ("в очереди", None),
|
||||||
|
"running": ("выполняется", None),
|
||||||
|
"done": ("готово", "ok"),
|
||||||
|
"error": ("ошибка", "danger"),
|
||||||
|
"cancelled": ("отменена", None),
|
||||||
|
"interrupted": ("прервана", "danger"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class MainView:
|
class MainView:
|
||||||
def __init__(self, page: ft.Page, client: ApiClient):
|
def __init__(self, page: ft.Page, client: ApiClient):
|
||||||
self.page = page
|
self.page = page
|
||||||
@@ -20,14 +31,15 @@ class MainView:
|
|||||||
self.sections: list[dict] = []
|
self.sections: list[dict] = []
|
||||||
self.current_section_id: int | None = None # None = все видео
|
self.current_section_id: int | None = None # None = все видео
|
||||||
|
|
||||||
# ── LLM config fields ──────────────────────────────────────────────
|
self._polling = False
|
||||||
|
|
||||||
|
# ── Settings tab fields ────────────────────────────────────────────
|
||||||
self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True)
|
self.base_url = d.text_field("Base URL", hint="https://openrouter.ai/api/v1", expand=True)
|
||||||
self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True)
|
self.api_key = d.text_field("API Key", hint="sk-…", password=True, expand=True)
|
||||||
|
|
||||||
# Searchable model combobox
|
|
||||||
self.model_input = d.text_field(
|
self.model_input = d.text_field(
|
||||||
"Модель",
|
"Модель",
|
||||||
hint="Сначала нажмите «Загрузить модели»",
|
hint="Нажмите «Загрузить модели» и выберите из списка",
|
||||||
on_change=self._on_model_search,
|
on_change=self._on_model_search,
|
||||||
expand=True,
|
expand=True,
|
||||||
)
|
)
|
||||||
@@ -44,7 +56,7 @@ class MainView:
|
|||||||
|
|
||||||
self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
|
self.llm_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
|
||||||
self.llm_status_chip = ft.Container(
|
self.llm_status_chip = ft.Container(
|
||||||
content=ft.Text("LLM не подключена", size=12, color=d.MUTED),
|
content=ft.Text("LLM не настроена", size=12, color=d.MUTED),
|
||||||
padding=ft.padding.symmetric(horizontal=10, vertical=4),
|
padding=ft.padding.symmetric(horizontal=10, vertical=4),
|
||||||
border=ft.border.all(1, d.BORDER),
|
border=ft.border.all(1, d.BORDER),
|
||||||
border_radius=99,
|
border_radius=99,
|
||||||
@@ -52,12 +64,6 @@ class MainView:
|
|||||||
self.btn_load = d.primary_button("Загрузить модели", self._load_models)
|
self.btn_load = d.primary_button("Загрузить модели", self._load_models)
|
||||||
self.btn_test = d.secondary_button("Проверить связь", self._test_conn)
|
self.btn_test = d.secondary_button("Проверить связь", self._test_conn)
|
||||||
|
|
||||||
# ── Pipeline fields ────────────────────────────────────────────────
|
|
||||||
self.yt_url = d.text_field(
|
|
||||||
"Ссылка на YouTube",
|
|
||||||
hint="https://youtu.be/…",
|
|
||||||
expand=True,
|
|
||||||
)
|
|
||||||
self.vosk_path = d.text_field(
|
self.vosk_path = d.text_field(
|
||||||
"Vosk-модель (fallback, когда нет субтитров)",
|
"Vosk-модель (fallback, когда нет субтитров)",
|
||||||
value="models/vosk-model-small-ru-0.22",
|
value="models/vosk-model-small-ru-0.22",
|
||||||
@@ -76,19 +82,28 @@ class MainView:
|
|||||||
width=220,
|
width=220,
|
||||||
)
|
)
|
||||||
self.keep_video_sw = ft.Switch(
|
self.keep_video_sw = ft.Switch(
|
||||||
label="Сохранять видео на диске",
|
label="Сохранять видео на диске (иначе качается только аудио)",
|
||||||
value=True,
|
value=True,
|
||||||
active_color=d.ACCENT,
|
active_color=d.ACCENT,
|
||||||
label_style=ft.TextStyle(color=d.TEXT, size=13),
|
label_style=ft.TextStyle(color=d.TEXT, size=13),
|
||||||
)
|
)
|
||||||
self.pipeline_section_dd = self._dropdown("Раздел для нового видео", width=280)
|
self.settings_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
|
||||||
|
self.btn_save = d.primary_button("💾 Сохранить настройки", self._save_settings)
|
||||||
|
|
||||||
|
# ── Pipeline (main tab) ────────────────────────────────────────────
|
||||||
|
self.yt_url = d.text_field(
|
||||||
|
"Ссылка на YouTube: видео или плейлист",
|
||||||
|
hint="https://youtu.be/… или https://www.youtube.com/playlist?list=…",
|
||||||
|
expand=True,
|
||||||
|
)
|
||||||
|
self.pipeline_section_dd = self._dropdown("Раздел для новых видео", width=280)
|
||||||
self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
|
self.pipeline_msg = ft.Text("", color=d.MUTED, size=13, expand=True)
|
||||||
self.btn_run = d.primary_button("▶ Запустить", self._run_pipeline)
|
self.btn_run = d.primary_button("▶ В очередь", self._run_pipeline)
|
||||||
|
|
||||||
# Result block (initially hidden)
|
# ── Jobs ───────────────────────────────────────────────────────────
|
||||||
self._result_col = ft.Column(spacing=6, visible=False)
|
self._jobs_col = ft.Column(spacing=6)
|
||||||
|
|
||||||
# ── Library controls ───────────────────────────────────────────────
|
# ── Library ────────────────────────────────────────────────────────
|
||||||
self.search_field = d.text_field(
|
self.search_field = d.text_field(
|
||||||
"Поиск по названию / URL", on_change=None, expand=True
|
"Поиск по названию / URL", on_change=None, expand=True
|
||||||
)
|
)
|
||||||
@@ -124,14 +139,11 @@ class MainView:
|
|||||||
# ─────────────────────── build ───────────────────────────────────────
|
# ─────────────────────── build ───────────────────────────────────────
|
||||||
|
|
||||||
def build(self) -> ft.Control:
|
def build(self) -> ft.Control:
|
||||||
return ft.Column(
|
main_tab = ft.Container(
|
||||||
controls=[
|
|
||||||
self._build_topbar(),
|
|
||||||
ft.Container(
|
|
||||||
content=ft.Column(
|
content=ft.Column(
|
||||||
[
|
[
|
||||||
self._build_llm_card(),
|
|
||||||
self._build_pipeline_card(),
|
self._build_pipeline_card(),
|
||||||
|
self._build_jobs_card(),
|
||||||
self._build_library_card(),
|
self._build_library_card(),
|
||||||
],
|
],
|
||||||
spacing=20,
|
spacing=20,
|
||||||
@@ -139,8 +151,34 @@ class MainView:
|
|||||||
),
|
),
|
||||||
padding=ft.padding.symmetric(horizontal=24, vertical=20),
|
padding=ft.padding.symmetric(horizontal=24, vertical=20),
|
||||||
expand=True,
|
expand=True,
|
||||||
),
|
)
|
||||||
|
settings_tab = ft.Container(
|
||||||
|
content=ft.Column(
|
||||||
|
[
|
||||||
|
self._build_llm_card(),
|
||||||
|
self._build_defaults_card(),
|
||||||
],
|
],
|
||||||
|
spacing=20,
|
||||||
|
scroll=ft.ScrollMode.AUTO,
|
||||||
|
),
|
||||||
|
padding=ft.padding.symmetric(horizontal=24, vertical=20),
|
||||||
|
expand=True,
|
||||||
|
)
|
||||||
|
tabs = ft.Tabs(
|
||||||
|
selected_index=0,
|
||||||
|
animation_duration=200,
|
||||||
|
label_color=d.ACCENT_2,
|
||||||
|
unselected_label_color=d.MUTED,
|
||||||
|
indicator_color=d.ACCENT,
|
||||||
|
divider_color=d.BORDER,
|
||||||
|
tabs=[
|
||||||
|
ft.Tab(text="Главная", content=main_tab),
|
||||||
|
ft.Tab(text="Настройки", content=settings_tab),
|
||||||
|
],
|
||||||
|
expand=True,
|
||||||
|
)
|
||||||
|
return ft.Column(
|
||||||
|
controls=[self._build_topbar(), tabs],
|
||||||
expand=True,
|
expand=True,
|
||||||
spacing=0,
|
spacing=0,
|
||||||
)
|
)
|
||||||
@@ -174,13 +212,14 @@ class MainView:
|
|||||||
border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)),
|
border=ft.border.only(bottom=ft.BorderSide(1, d.BORDER)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ── Settings tab cards ────────────────────────────────────────────────
|
||||||
|
|
||||||
def _build_llm_card(self) -> ft.Container:
|
def _build_llm_card(self) -> ft.Container:
|
||||||
return d.card(
|
return d.card(
|
||||||
d.section_title("Подключение к LLM"),
|
d.section_title("Подключение к LLM"),
|
||||||
d.hint(
|
d.hint(
|
||||||
"Укажите любой OpenAI-совместимый провайдер. "
|
"Любой OpenAI-совместимый провайдер (OpenRouter, OpenAI, локальная Ollama…). "
|
||||||
"Нейронка внутри Docker не разворачивается — "
|
"Ключ и модель хранятся на сервере в БД и переживают перезапуск."
|
||||||
"указываем URL внешнего сервиса."
|
|
||||||
),
|
),
|
||||||
ft.Row([self.base_url, self.api_key], spacing=12),
|
ft.Row([self.base_url, self.api_key], spacing=12),
|
||||||
ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10),
|
ft.Row([self.btn_load, self.btn_test, self.llm_msg], spacing=10),
|
||||||
@@ -189,22 +228,43 @@ class MainView:
|
|||||||
self._model_list_wrap,
|
self._model_list_wrap,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _build_defaults_card(self) -> ft.Container:
|
||||||
|
return d.card(
|
||||||
|
d.section_title("Параметры обработки"),
|
||||||
|
d.hint("Применяются ко всем новым задачам."),
|
||||||
|
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
|
||||||
|
self.keep_video_sw,
|
||||||
|
ft.Divider(color=d.BORDER, height=1),
|
||||||
|
ft.Row([self.btn_save, self.settings_msg], spacing=10),
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Main tab cards ────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _build_pipeline_card(self) -> ft.Container:
|
def _build_pipeline_card(self) -> ft.Container:
|
||||||
return d.card(
|
return d.card(
|
||||||
d.section_title("Обработать видео"),
|
d.section_title("Обработать видео или плейлист"),
|
||||||
d.hint(
|
d.hint(
|
||||||
"Pipeline: yt-dlp → субтитры (или Vosk fallback) "
|
"Задача уходит в фон: качаем, расшифровываем (субтитры или Vosk), "
|
||||||
"→ выжимка через выбранную LLM → запись в БД."
|
"сокращаем через LLM, пишем в БД. Плейлисты и лекции на 4–5 часов — ок."
|
||||||
),
|
),
|
||||||
self.yt_url,
|
self.yt_url,
|
||||||
ft.Row([self.vosk_path, self.n_sentences], spacing=12),
|
|
||||||
ft.Row(
|
ft.Row(
|
||||||
[self.pipeline_section_dd, self.keep_video_sw],
|
[self.btn_run, self.pipeline_section_dd, self.pipeline_msg],
|
||||||
spacing=18,
|
spacing=12,
|
||||||
vertical_alignment=ft.CrossAxisAlignment.CENTER,
|
vertical_alignment=ft.CrossAxisAlignment.CENTER,
|
||||||
),
|
),
|
||||||
ft.Row([self.btn_run, self.pipeline_msg], spacing=10),
|
)
|
||||||
self._result_col,
|
|
||||||
|
def _build_jobs_card(self) -> ft.Container:
|
||||||
|
return d.card(
|
||||||
|
ft.Row(
|
||||||
|
[
|
||||||
|
d.section_title("Задачи"),
|
||||||
|
ft.Container(expand=True),
|
||||||
|
d.secondary_button("Обновить", lambda _: self._start_jobs_poll()),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
self._jobs_col,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _build_library_card(self) -> ft.Container:
|
def _build_library_card(self) -> ft.Container:
|
||||||
@@ -256,6 +316,39 @@ class MainView:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ─────────────────── settings load/save ──────────────────────────────
|
||||||
|
|
||||||
|
def apply_settings(self, s: dict) -> None:
|
||||||
|
"""Заполняет поля значениями с сервера (вызывается при старте сессии)."""
|
||||||
|
self.base_url.value = s.get("llm_base_url", "")
|
||||||
|
self.api_key.value = s.get("llm_api_key", "")
|
||||||
|
self.selected_model = s.get("llm_model", "")
|
||||||
|
self.model_input.value = self.selected_model
|
||||||
|
self.n_sentences.value = str(s.get("summary_max_sentences", 15))
|
||||||
|
self.keep_video_sw.value = bool(s.get("keep_video", True))
|
||||||
|
self.vosk_path.value = s.get("vosk_model_path", "models/vosk-model-small-ru-0.22")
|
||||||
|
self._set_status(ok=bool(self.base_url.value and self.selected_model))
|
||||||
|
|
||||||
|
def _save_settings(self, _):
|
||||||
|
model = self.selected_model or (self.model_input.value or "").strip()
|
||||||
|
payload = {
|
||||||
|
"llm_base_url": self.base_url.value.strip(),
|
||||||
|
"llm_api_key": self.api_key.value.strip(),
|
||||||
|
"llm_model": model,
|
||||||
|
"summary_max_sentences": int(self.n_sentences.value or 15),
|
||||||
|
"keep_video": bool(self.keep_video_sw.value),
|
||||||
|
"vosk_model_path": self.vosk_path.value.strip(),
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
self.client.save_settings(payload)
|
||||||
|
self.settings_msg.value = "Сохранено ✓ (хранится в БД, переживёт перезапуск)"
|
||||||
|
self.settings_msg.color = d.OK
|
||||||
|
self._set_status(ok=bool(payload["llm_base_url"] and model))
|
||||||
|
except ApiError as e:
|
||||||
|
self.settings_msg.value = str(e)
|
||||||
|
self.settings_msg.color = d.DANGER
|
||||||
|
self.page.update()
|
||||||
|
|
||||||
# ─────────────────── model combobox ──────────────────────────────────
|
# ─────────────────── model combobox ──────────────────────────────────
|
||||||
|
|
||||||
def _on_model_search(self, e):
|
def _on_model_search(self, e):
|
||||||
@@ -292,6 +385,8 @@ class MainView:
|
|||||||
self.selected_model = mid
|
self.selected_model = mid
|
||||||
self.model_input.value = mid
|
self.model_input.value = mid
|
||||||
self._model_list_wrap.visible = False
|
self._model_list_wrap.visible = False
|
||||||
|
self.llm_msg.value = "Не забудьте нажать «Сохранить настройки»"
|
||||||
|
self.llm_msg.color = d.MUTED
|
||||||
self.page.update()
|
self.page.update()
|
||||||
|
|
||||||
# ─────────────────── LLM actions ─────────────────────────────────────
|
# ─────────────────── LLM actions ─────────────────────────────────────
|
||||||
@@ -311,10 +406,8 @@ class MainView:
|
|||||||
self.models = self.client.llm_models(base_url, self.api_key.value.strip())
|
self.models = self.client.llm_models(base_url, self.api_key.value.strip())
|
||||||
self._render_model_list(self.models)
|
self._render_model_list(self.models)
|
||||||
self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True)
|
self._set_llm_msg(f"Загружено: {len(self.models)} моделей", ok=True)
|
||||||
self._set_status(ok=True)
|
|
||||||
except ApiError as e:
|
except ApiError as e:
|
||||||
self._set_llm_msg(str(e), error=True)
|
self._set_llm_msg(str(e), error=True)
|
||||||
self._set_status(ok=False)
|
|
||||||
finally:
|
finally:
|
||||||
self.btn_load.disabled = False
|
self.btn_load.disabled = False
|
||||||
self.page.update()
|
self.page.update()
|
||||||
@@ -322,7 +415,8 @@ class MainView:
|
|||||||
threading.Thread(target=_task, daemon=True).start()
|
threading.Thread(target=_task, daemon=True).start()
|
||||||
|
|
||||||
def _test_conn(self, _):
|
def _test_conn(self, _):
|
||||||
if not self.selected_model:
|
model = self.selected_model or (self.model_input.value or "").strip()
|
||||||
|
if not model:
|
||||||
self._set_llm_msg("Сначала выберите модель", error=True)
|
self._set_llm_msg("Сначала выберите модель", error=True)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -335,87 +429,164 @@ class MainView:
|
|||||||
r = self.client.llm_test(
|
r = self.client.llm_test(
|
||||||
self.base_url.value.strip(),
|
self.base_url.value.strip(),
|
||||||
self.api_key.value.strip(),
|
self.api_key.value.strip(),
|
||||||
self.selected_model,
|
model,
|
||||||
)
|
)
|
||||||
preview = (r.get("reply") or "")[:80]
|
preview = (r.get("reply") or "")[:80]
|
||||||
self._set_llm_msg(f"OK · {preview}", ok=True)
|
self._set_llm_msg(f"OK · {preview}", ok=True)
|
||||||
self._set_status(ok=True)
|
|
||||||
except ApiError as e:
|
except ApiError as e:
|
||||||
self._set_llm_msg(str(e), error=True)
|
self._set_llm_msg(str(e), error=True)
|
||||||
self._set_status(ok=False)
|
|
||||||
finally:
|
finally:
|
||||||
self.btn_test.disabled = False
|
self.btn_test.disabled = False
|
||||||
self.page.update()
|
self.page.update()
|
||||||
|
|
||||||
threading.Thread(target=_task, daemon=True).start()
|
threading.Thread(target=_task, daemon=True).start()
|
||||||
|
|
||||||
# ─────────────────── pipeline ─────────────────────────────────────────
|
# ─────────────────── pipeline → jobs ──────────────────────────────────
|
||||||
|
|
||||||
def _run_pipeline(self, _):
|
def _run_pipeline(self, _):
|
||||||
url = self.yt_url.value.strip()
|
url = self.yt_url.value.strip()
|
||||||
if not url:
|
if not url:
|
||||||
self._set_pipeline_msg("Укажите ссылку на видео", error=True)
|
self._set_pipeline_msg("Укажите ссылку на видео или плейлист", error=True)
|
||||||
return
|
return
|
||||||
|
|
||||||
payload: dict = {
|
payload: dict = {"url": url}
|
||||||
"url": url,
|
|
||||||
"model_path": self.vosk_path.value.strip() or None,
|
|
||||||
"summary_max_sentences": int(self.n_sentences.value or 15),
|
|
||||||
"keep_video": bool(self.keep_video_sw.value),
|
|
||||||
}
|
|
||||||
if self.pipeline_section_dd.value:
|
if self.pipeline_section_dd.value:
|
||||||
payload["section_id"] = int(self.pipeline_section_dd.value)
|
payload["section_id"] = int(self.pipeline_section_dd.value)
|
||||||
if self.base_url.value.strip() and self.selected_model:
|
|
||||||
payload["llm"] = {
|
|
||||||
"base_url": self.base_url.value.strip(),
|
|
||||||
"api_key": self.api_key.value.strip(),
|
|
||||||
"model": self.selected_model,
|
|
||||||
}
|
|
||||||
|
|
||||||
self._set_pipeline_msg("Обработка видео… (может занять несколько минут)")
|
|
||||||
self._result_col.visible = False
|
|
||||||
self.btn_run.disabled = True
|
|
||||||
self.page.update()
|
|
||||||
|
|
||||||
def _task():
|
|
||||||
try:
|
try:
|
||||||
r = self.client.pipeline_process(payload)
|
self.client.create_job(payload)
|
||||||
self._render_result(r)
|
self.yt_url.value = ""
|
||||||
self._set_pipeline_msg("Готово ✓", ok=True)
|
self._set_pipeline_msg("Задача поставлена в очередь ✓", ok=True)
|
||||||
self._refresh_library()
|
|
||||||
except ApiError as e:
|
except ApiError as e:
|
||||||
self._set_pipeline_msg(str(e), error=True)
|
self._set_pipeline_msg(str(e), error=True)
|
||||||
finally:
|
|
||||||
self.btn_run.disabled = False
|
|
||||||
self.page.update()
|
self.page.update()
|
||||||
|
self._start_jobs_poll()
|
||||||
|
|
||||||
threading.Thread(target=_task, daemon=True).start()
|
# ─────────────────── jobs ─────────────────────────────────────────────
|
||||||
|
|
||||||
def _render_result(self, r: dict):
|
def _start_jobs_poll(self):
|
||||||
self._result_col.controls = [
|
if self._polling:
|
||||||
ft.Divider(color=d.BORDER, height=1),
|
return
|
||||||
ft.Text(r.get("title") or r.get("uuid", ""), color=d.ACCENT_2, size=15, weight=ft.FontWeight.W_600),
|
self._polling = True
|
||||||
*[
|
threading.Thread(target=self._poll_loop, daemon=True).start()
|
||||||
self._kv(k, v)
|
|
||||||
for k, v in [
|
def _poll_loop(self):
|
||||||
("UUID", r.get("uuid", "")),
|
prev: dict = {}
|
||||||
("Источник", r.get("source_url", "")),
|
try:
|
||||||
("Метод", r.get("transcription_method", "")),
|
while True:
|
||||||
("Видео", r.get("video_path") or "не сохранено"),
|
try:
|
||||||
("Полный текст", r.get("text_full_path", "")),
|
jobs = self.client.list_jobs()
|
||||||
("Выжимка", r.get("text_summary_path", "")),
|
except (ApiError, Exception):
|
||||||
|
break
|
||||||
|
self._render_jobs(jobs)
|
||||||
|
state = {j["id"]: (j["done_items"], j["status"]) for j in jobs}
|
||||||
|
if state != prev:
|
||||||
|
prev = state
|
||||||
|
self._refresh_library() # там же page.update()
|
||||||
|
else:
|
||||||
|
self.page.update()
|
||||||
|
if not any(j["status"] in ("queued", "running") for j in jobs):
|
||||||
|
break
|
||||||
|
time.sleep(3)
|
||||||
|
finally:
|
||||||
|
self._polling = False
|
||||||
|
|
||||||
|
def _render_jobs(self, jobs: list[dict]):
|
||||||
|
if not jobs:
|
||||||
|
self._jobs_col.controls = [
|
||||||
|
ft.Text("Задач ещё не было", color=d.MUTED, size=13)
|
||||||
]
|
]
|
||||||
],
|
return
|
||||||
ft.Text("Превью выжимки:", color=d.MUTED, size=12),
|
self._jobs_col.controls = [self._job_row(j) for j in jobs]
|
||||||
ft.Container(
|
|
||||||
content=ft.Text(r.get("summary_preview", ""), color=d.TEXT, size=13, selectable=True, no_wrap=False),
|
def _job_row(self, j: dict) -> ft.Container:
|
||||||
padding=12,
|
label, tone = _JOB_STATUS.get(j["status"], (j["status"], None))
|
||||||
bgcolor=d.SURFACE_3,
|
color = d.OK if tone == "ok" else (d.DANGER if tone == "danger" else d.MUTED)
|
||||||
border=ft.border.all(1, d.BORDER),
|
active = j["status"] in ("queued", "running")
|
||||||
border_radius=8,
|
|
||||||
|
chip = ft.Container(
|
||||||
|
content=ft.Text(label, size=11, color=color),
|
||||||
|
padding=ft.padding.symmetric(horizontal=8, vertical=2),
|
||||||
|
border=ft.border.all(1, color),
|
||||||
|
border_radius=99,
|
||||||
|
)
|
||||||
|
head = [
|
||||||
|
chip,
|
||||||
|
ft.Text(
|
||||||
|
(j["title"] or j["url"])[:90],
|
||||||
|
color=d.TEXT,
|
||||||
|
size=13,
|
||||||
|
weight=ft.FontWeight.W_600,
|
||||||
|
expand=True,
|
||||||
|
no_wrap=True,
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
self._result_col.visible = True
|
if j["kind"] == "playlist":
|
||||||
|
head.append(ft.Text("плейлист", color=d.ACCENT_2, size=11))
|
||||||
|
if j["total_items"]:
|
||||||
|
head.append(
|
||||||
|
ft.Text(f"{j['done_items']}/{j['total_items']}", color=d.MUTED, size=12)
|
||||||
|
)
|
||||||
|
if active:
|
||||||
|
head.append(
|
||||||
|
ft.IconButton(
|
||||||
|
icon=ft.Icons.STOP_CIRCLE_OUTLINED,
|
||||||
|
icon_color=d.DANGER,
|
||||||
|
icon_size=17,
|
||||||
|
tooltip="Отменить (после текущего элемента)",
|
||||||
|
on_click=lambda e, jid=j["id"]: self._cancel_job(jid),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
head.append(
|
||||||
|
ft.IconButton(
|
||||||
|
icon=ft.Icons.CLOSE,
|
||||||
|
icon_color=d.MUTED,
|
||||||
|
icon_size=15,
|
||||||
|
tooltip="Убрать из списка",
|
||||||
|
on_click=lambda e, jid=j["id"]: self._delete_job(jid),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
rows: list[ft.Control] = [ft.Row(head, spacing=8)]
|
||||||
|
if active:
|
||||||
|
total = j["total_items"] or 0
|
||||||
|
value = (j["done_items"] / total) if total else None
|
||||||
|
rows.append(
|
||||||
|
ft.ProgressBar(value=value, color=d.ACCENT, bgcolor=d.SURFACE_3, height=4)
|
||||||
|
)
|
||||||
|
if j.get("live"):
|
||||||
|
rows.append(ft.Text(j["live"][:140], color=d.ACCENT_2, size=11))
|
||||||
|
if j.get("error"):
|
||||||
|
rows.append(
|
||||||
|
ft.Text(j["error"][:300], color=d.DANGER, size=11, no_wrap=False)
|
||||||
|
)
|
||||||
|
if j["failed_items"]:
|
||||||
|
rows.append(
|
||||||
|
ft.Text(f"с ошибкой: {j['failed_items']}", color=d.DANGER, size=11)
|
||||||
|
)
|
||||||
|
|
||||||
|
return ft.Container(
|
||||||
|
content=ft.Column(rows, spacing=5, tight=True),
|
||||||
|
padding=10,
|
||||||
|
bgcolor=d.SURFACE_2,
|
||||||
|
border=ft.border.all(1, d.BORDER),
|
||||||
|
border_radius=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _cancel_job(self, job_id: str):
|
||||||
|
try:
|
||||||
|
self.client.cancel_job(job_id)
|
||||||
|
except ApiError as e:
|
||||||
|
self.lib_msg.value = str(e)
|
||||||
|
self._start_jobs_poll()
|
||||||
|
|
||||||
|
def _delete_job(self, job_id: str):
|
||||||
|
try:
|
||||||
|
self.client.delete_job(job_id)
|
||||||
|
except ApiError as e:
|
||||||
|
self.lib_msg.value = str(e)
|
||||||
|
self._start_jobs_poll()
|
||||||
|
|
||||||
# ─────────────────── sections ─────────────────────────────────────────
|
# ─────────────────── sections ─────────────────────────────────────────
|
||||||
|
|
||||||
@@ -688,6 +859,6 @@ class MainView:
|
|||||||
|
|
||||||
def _set_status(self, *, ok: bool):
|
def _set_status(self, *, ok: bool):
|
||||||
chip = self.llm_status_chip
|
chip = self.llm_status_chip
|
||||||
chip.content.value = "LLM подключена" if ok else "Ошибка подключения"
|
chip.content.value = "LLM настроена" if ok else "LLM не настроена"
|
||||||
chip.content.color = d.OK if ok else d.DANGER
|
chip.content.color = d.OK if ok else d.MUTED
|
||||||
chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else "rgba(227,90,114,.4)")
|
chip.border = ft.border.all(1, "rgba(109,213,140,.4)" if ok else d.BORDER)
|
||||||
|
|||||||
Reference in New Issue
Block a user