Фоновые задачи: плейлисты, длинные лекции, вкладка настроек
- таблица jobs + очередь с воркером: POST /jobs сразу отвечает, элементы
плейлиста обрабатываются последовательно, есть отмена и живой прогресс
(скачивание %, минуты распознанного Vosk-аудио)
- expand_url: плейлист раскрывается в список видео
- keep_video=false теперь качает только аудио (для 4-5ч лекций)
- настройки (LLM-ключ, модель, длина выжимки, хранение видео, путь Vosk)
хранятся в Postgres: переживают перезапуск и перезагрузку страницы
- длина выжимки прокидывается в LLM-промпт ({n} предложений)
- UI: вкладки «Главная»/«Настройки», карточка задач с автообновлением
каждые 3 сек, прогресс-бар, отмена, библиотека обновляется по мере
готовности элементов
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
59
api/route/app_settings.py
Normal file
59
api/route/app_settings.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Настройки приложения (LLM-ключи, дефолты пайплайна) — живут в Postgres,
|
||||
переживают перезапуск контейнеров и перезагрузку страницы.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
|
||||
from api.db.setting import get_all_settings, set_settings
|
||||
|
||||
|
||||
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||
|
||||
_KEYS = (
|
||||
"llm_base_url",
|
||||
"llm_api_key",
|
||||
"llm_model",
|
||||
"summary_max_sentences",
|
||||
"keep_video",
|
||||
"vosk_model_path",
|
||||
)
|
||||
|
||||
|
||||
class SettingsPayload(BaseModel):
|
||||
llm_base_url: str | None = None
|
||||
llm_api_key: str | None = None
|
||||
llm_model: str | None = None
|
||||
summary_max_sentences: int | None = None
|
||||
keep_video: bool | None = None
|
||||
vosk_model_path: str | None = None
|
||||
|
||||
|
||||
@router.get("/")
|
||||
async def get_settings():
|
||||
db = await get_all_settings()
|
||||
return {
|
||||
"llm_base_url": db.get("llm_base_url", ""),
|
||||
"llm_api_key": db.get("llm_api_key", ""),
|
||||
"llm_model": db.get("llm_model", ""),
|
||||
"summary_max_sentences": int(db.get("summary_max_sentences") or 15),
|
||||
"keep_video": db.get("keep_video", "1") != "0",
|
||||
"vosk_model_path": db.get("vosk_model_path", "models/vosk-model-small-ru-0.22"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/")
|
||||
async def put_settings(body: SettingsPayload):
|
||||
values: dict[str, str] = {}
|
||||
for key in _KEYS:
|
||||
v = getattr(body, key)
|
||||
if v is None:
|
||||
continue
|
||||
if key == "keep_video":
|
||||
values[key] = "1" if v else "0"
|
||||
else:
|
||||
values[key] = str(v)
|
||||
if values:
|
||||
await set_settings(values)
|
||||
return await get_settings()
|
||||
112
api/route/jobs.py
Normal file
112
api/route/jobs.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Фоновые задачи: видео или плейлист любой длины.
|
||||
|
||||
POST ставит задачу в очередь и сразу возвращает id; прогресс — в GET /jobs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid as _uuid
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from api.db.job import create_job, delete_job, get_job, list_jobs
|
||||
from api.lib import jobqueue
|
||||
|
||||
|
||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
class JobIn(BaseModel):
|
||||
url: str
|
||||
section_id: int | None = None
|
||||
# None → берутся из сохранённых настроек
|
||||
keep_video: bool | None = None
|
||||
summary_max_sentences: int | None = None
|
||||
model_path: str | None = None
|
||||
|
||||
|
||||
class JobOut(BaseModel):
|
||||
id: str
|
||||
url: str
|
||||
kind: str
|
||||
title: str
|
||||
status: str
|
||||
total_items: int
|
||||
done_items: int
|
||||
failed_items: int
|
||||
current_item: str
|
||||
live: str
|
||||
error: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
def _to_out(j) -> JobOut:
|
||||
return JobOut(
|
||||
id=j.id,
|
||||
url=j.url,
|
||||
kind=j.kind,
|
||||
title=j.title,
|
||||
status=j.status,
|
||||
total_items=j.total_items,
|
||||
done_items=j.done_items,
|
||||
failed_items=j.failed_items,
|
||||
current_item=j.current_item,
|
||||
live=jobqueue.live_status(j.id) or j.current_item,
|
||||
error=j.error,
|
||||
created_at=j.created_at,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/", response_model=JobOut)
|
||||
async def create(body: JobIn):
|
||||
url = body.url.strip()
|
||||
if not url:
|
||||
raise HTTPException(status_code=400, detail="URL is empty")
|
||||
job_id = _uuid.uuid4().hex
|
||||
j = await create_job(job_id, url)
|
||||
jobqueue.submit(
|
||||
job_id,
|
||||
{
|
||||
"section_id": body.section_id,
|
||||
"keep_video": body.keep_video,
|
||||
"summary_max_sentences": body.summary_max_sentences,
|
||||
"model_path": body.model_path,
|
||||
},
|
||||
)
|
||||
return _to_out(j)
|
||||
|
||||
|
||||
@router.get("/", response_model=list[JobOut])
|
||||
async def list_all(limit: int = 50):
|
||||
return [_to_out(j) for j in await list_jobs(limit)]
|
||||
|
||||
|
||||
@router.get("/{job_id}", response_model=JobOut)
|
||||
async def get_one(job_id: str):
|
||||
j = await get_job(job_id)
|
||||
if j is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return _to_out(j)
|
||||
|
||||
|
||||
@router.post("/{job_id}/cancel")
|
||||
async def cancel(job_id: str):
|
||||
j = await get_job(job_id)
|
||||
if j is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
if j.status not in ("queued", "running"):
|
||||
raise HTTPException(status_code=400, detail=f"Job is {j.status}")
|
||||
jobqueue.request_cancel(job_id)
|
||||
return {"ok": True, "note": "остановится после текущего элемента"}
|
||||
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
async def delete(job_id: str):
|
||||
j = await get_job(job_id)
|
||||
if j is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
if j.status in ("queued", "running"):
|
||||
raise HTTPException(status_code=400, detail="Сначала отмените задачу")
|
||||
await delete_job(job_id)
|
||||
return {"ok": True}
|
||||
@@ -1,29 +1,16 @@
|
||||
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist."""
|
||||
"""Синхронный запуск пайплайна для одного видео (для длинных видео и
|
||||
плейлистов используйте /jobs — этот эндпоинт держит HTTP-соединение открытым).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid as _uuid
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from api.config import settings
|
||||
from api.db.video import create_video
|
||||
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
|
||||
from api.lib.summarize import summarize
|
||||
from api.lib.transcribe import transcribe_via_ffmpeg
|
||||
from api.lib.youtube import download_video_with_subs, vtt_to_text
|
||||
from api.lib.llm import LLMConfig
|
||||
from api.lib.processing import PipelineError, process_single_video
|
||||
|
||||
|
||||
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
|
||||
logger = logging.getLogger("pipeline")
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
|
||||
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
|
||||
_MODELS_DIR = _PROJECT_ROOT / "models"
|
||||
|
||||
|
||||
class LLMOverride(BaseModel):
|
||||
@@ -40,12 +27,12 @@ class LLMOverride(BaseModel):
|
||||
|
||||
class ProcessRequest(BaseModel):
|
||||
url: str
|
||||
model_path: str | None = None # path inside models/, used when subtitles are unavailable
|
||||
summary_max_sentences: int = 15
|
||||
keep_video: bool = True # False: файл видео удаляется после расшифровки
|
||||
model_path: str | None = None # None → из настроек
|
||||
summary_max_sentences: int | None = None
|
||||
keep_video: bool | None = None
|
||||
section_id: int | None = None
|
||||
# If supplied, this LLM is used for the summary; otherwise env-configured LLM;
|
||||
# otherwise the built-in extractive fallback.
|
||||
# Если задано — используется этот LLM; иначе сохранённые настройки/env;
|
||||
# иначе встроенная экстрактивная выжимка.
|
||||
llm: LLMOverride | None = None
|
||||
|
||||
|
||||
@@ -60,86 +47,10 @@ class ProcessResponse(BaseModel):
|
||||
summary_preview: str
|
||||
|
||||
|
||||
def _resolve_model_path(req_model_path: str | None) -> Path:
|
||||
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
|
||||
p = Path(raw)
|
||||
if not p.is_absolute():
|
||||
p = _PROJECT_ROOT / raw
|
||||
p = p.resolve()
|
||||
try:
|
||||
p.relative_to(_MODELS_DIR.resolve())
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="model_path must point inside the project's models directory",
|
||||
)
|
||||
if not p.exists():
|
||||
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
|
||||
return p
|
||||
|
||||
|
||||
@router.post("/process", response_model=ProcessResponse)
|
||||
async def process_video(req: ProcessRequest):
|
||||
video_uuid = _uuid.uuid4().hex
|
||||
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
logger.info("[%s] downloading %s", video_uuid, req.url)
|
||||
try:
|
||||
info = await loop.run_in_executor(
|
||||
None,
|
||||
download_video_with_subs,
|
||||
req.url,
|
||||
video_uuid,
|
||||
_VIDEO_DIR,
|
||||
settings.subtitle_langs_list,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
|
||||
|
||||
title: str = info["title"]
|
||||
video_path: Path = info["video_path"]
|
||||
sub_path: Path | None = info["subtitle_path"]
|
||||
|
||||
if not video_path.exists():
|
||||
raise HTTPException(status_code=500, detail="Video file missing after download")
|
||||
|
||||
full_text = ""
|
||||
method = ""
|
||||
|
||||
if sub_path is not None and sub_path.exists():
|
||||
try:
|
||||
content = sub_path.read_text(encoding="utf-8", errors="ignore")
|
||||
full_text = vtt_to_text(content)
|
||||
if full_text.strip():
|
||||
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
|
||||
logger.info("[%s] using subtitles (%s)", video_uuid, method)
|
||||
except Exception as e:
|
||||
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
|
||||
full_text = ""
|
||||
|
||||
if not full_text.strip():
|
||||
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
|
||||
model_path = _resolve_model_path(req.model_path)
|
||||
try:
|
||||
full_text = await loop.run_in_executor(
|
||||
None, transcribe_via_ffmpeg, video_path, model_path
|
||||
)
|
||||
method = f"vosk:{model_path.name}"
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
|
||||
|
||||
if not full_text.strip():
|
||||
raise HTTPException(
|
||||
status_code=500, detail="No subtitles and transcription returned empty text"
|
||||
)
|
||||
|
||||
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
|
||||
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
|
||||
text_full_path.write_text(full_text, encoding="utf-8")
|
||||
|
||||
llm_cfg = None
|
||||
sys_p = user_t = None
|
||||
if req.llm is not None:
|
||||
llm_cfg = LLMConfig(
|
||||
base_url=req.llm.base_url,
|
||||
@@ -152,60 +63,18 @@ async def process_video(req: ProcessRequest):
|
||||
)
|
||||
sys_p = req.llm.system_prompt
|
||||
user_t = req.llm.user_template
|
||||
else:
|
||||
llm_cfg = llm_from_settings()
|
||||
sys_p = None
|
||||
user_t = None
|
||||
|
||||
try:
|
||||
summary = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: summarize(
|
||||
full_text,
|
||||
llm_cfg=llm_cfg,
|
||||
system_prompt=sys_p,
|
||||
user_template=user_t,
|
||||
max_sentences=req.summary_max_sentences,
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
|
||||
text_summary_path.write_text(summary, encoding="utf-8")
|
||||
|
||||
if req.keep_video:
|
||||
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
|
||||
else:
|
||||
# пользователь просил не хранить видео: текст уже извлечён, файлы не нужны
|
||||
video_path.unlink(missing_ok=True)
|
||||
for p in _VIDEO_DIR.glob(f"{video_uuid}*.vtt"):
|
||||
p.unlink(missing_ok=True)
|
||||
rel_video = ""
|
||||
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
|
||||
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
|
||||
|
||||
try:
|
||||
await create_video(
|
||||
uuid=video_uuid,
|
||||
source_url=req.url,
|
||||
title=title,
|
||||
video_path=rel_video,
|
||||
text_full_path=rel_full,
|
||||
text_summary_path=rel_summary,
|
||||
transcription_method=method,
|
||||
result = await process_single_video(
|
||||
req.url,
|
||||
model_path=req.model_path,
|
||||
summary_max_sentences=req.summary_max_sentences,
|
||||
keep_video=req.keep_video,
|
||||
section_id=req.section_id,
|
||||
llm_cfg=llm_cfg,
|
||||
system_prompt=sys_p,
|
||||
user_template=user_t,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")
|
||||
|
||||
logger.info("[%s] done (method=%s)", video_uuid, method)
|
||||
|
||||
return ProcessResponse(
|
||||
uuid=video_uuid,
|
||||
title=title,
|
||||
source_url=req.url,
|
||||
video_path=rel_video,
|
||||
text_full_path=rel_full,
|
||||
text_summary_path=rel_summary,
|
||||
transcription_method=method,
|
||||
summary_preview=summary[:300],
|
||||
)
|
||||
except PipelineError as e:
|
||||
raise HTTPException(status_code=e.status, detail=str(e))
|
||||
return ProcessResponse(**result)
|
||||
|
||||
Reference in New Issue
Block a user