Рабочий пайплайн: YouTube -> транскрипция -> выжимка -> Postgres

- новый api/: /pipeline/process (yt-dlp, субтитры или Vosk, LLM/экстрактивная
  суммаризация), /videos, /llm; старый код перенесён в api_legacy/
- web/: Flet UI (flet 0.28.3 + flet-web, порт 8550)
- Dockerfile.api: ffmpeg слоем из mwader/static-ffmpeg, pip с кэш-маунтом
- requirements/pyproject: убраны moviepy, imageio-ffmpeg, битый asyncio
- README с инструкцией запуска и загрузкой Vosk-модели

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 11:12:46 +05:00
parent 564c9c2d2f
commit 2697e01714
51 changed files with 2313 additions and 284 deletions

View File

@@ -1,2 +0,0 @@
# Для запуска API отдельно:
# uvicorn api.main:app --reload

0
api/__init__.py Normal file
View File

View File

@@ -1,28 +1,38 @@
from pydantic_settings import BaseSettings
import os
from pydantic_settings import BaseSettings
def _env(*names, default=None):
for n in names:
v = os.getenv(n)
if v:
return v
return default
class Settings(BaseSettings):
# support both custom DB_* names and common POSTGRES_* names from postgres images
DB_USER: str = _env("DB_USER", "POSTGRES_USER", default="postgres")
DB_PASS: str = _env("DB_PASS", "POSTGRES_PASSWORD", default="postgres")
DB_NAME: str = _env("DB_NAME", "POSTGRES_DB", default="test_db")
DB_HOST: str = _env("DB_HOST", "POSTGRES_HOST", "DB_HOST", default="db")
DB_PORT: str = _env("DB_PORT", "POSTGRES_PORT", default="5432")
DB_USER: str = os.getenv("DB_USER", "postgres")
DB_PASS: str = os.getenv("DB_PASS", "postgres")
DB_NAME: str = os.getenv("DB_NAME", "test_db")
DB_HOST: str = os.getenv("DB_HOST", "db")
DB_PORT: str = os.getenv("DB_PORT", "5432")
# If explicit DB_* variables are provided, build DATABASE_URL from them (priority).
# Otherwise fall back to explicit DATABASE_URL or DATABASE_URL_ASYNC env.
_built_db_url: str | None = None
if DB_USER and DB_PASS and DB_NAME and DB_HOST and DB_PORT:
_built_db_url = f"postgresql+asyncpg://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
DEFAULT_VOSK_MODEL: str = os.getenv(
"DEFAULT_VOSK_MODEL", "models/vosk-model-small-ru-0.22"
)
SUBTITLE_LANGS: str = os.getenv("SUBTITLE_LANGS", "ru,en")
DATABASE_URL: str = _built_db_url or _env("DATABASE_URL", "DATABASE_URL_ASYNC", default="")
# External LLM (OpenAI-compatible). The LLM itself is NOT hosted in this container —
# point at any provider/proxy/local server that speaks /v1/chat/completions.
LLM_BASE_URL: str = os.getenv("LLM_BASE_URL", "")
LLM_API_KEY: str = os.getenv("LLM_API_KEY", "")
LLM_MODEL: str = os.getenv("LLM_MODEL", "")
LLM_TIMEOUT: int = int(os.getenv("LLM_TIMEOUT", "120"))
LLM_MAX_TOKENS: int = int(os.getenv("LLM_MAX_TOKENS", "1000"))
LLM_TEMPERATURE: float = float(os.getenv("LLM_TEMPERATURE", "0.3"))
@property
def database_url(self) -> str:
return (
f"postgresql+asyncpg://{self.DB_USER}:{self.DB_PASS}"
f"@{self.DB_HOST}:{self.DB_PORT}/{self.DB_NAME}"
)
@property
def subtitle_langs_list(self) -> list[str]:
return [x.strip() for x in self.SUBTITLE_LANGS.split(",") if x.strip()]
settings = Settings()

0
api/db/__init__.py Normal file
View File

View File

@@ -1,27 +1,63 @@
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy import text
import asyncio
from api.config import settings
import logging
DATABASE_URL = settings.DATABASE_URL
engine = create_async_engine(DATABASE_URL, echo=False, future=True)
import asyncpg
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker
from api.config import settings
from api.models.base import Base
import api.models.video # noqa: F401 - register model with Base.metadata
logger = logging.getLogger("db")
engine = create_async_engine(settings.database_url, echo=False, future=True)
SessionLocal = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
async def wait_for_db(retries: int = 30, delay: float = 1.0) -> None:
"""Wait until the database is available. Retries with exponential backoff.
async def wait_for_postgres(retries: int = 30, delay: float = 1.0) -> None:
"""Wait for the postgres server to accept connections (using the default 'postgres' db)."""
last_exc: Exception | None = None
for attempt in range(1, retries + 1):
try:
conn = await asyncpg.connect(
user=settings.DB_USER,
password=settings.DB_PASS,
database="postgres",
host=settings.DB_HOST,
port=int(settings.DB_PORT),
)
await conn.close()
return
except Exception as e:
last_exc = e
wait = min(delay * (2 ** (attempt - 1)), 5)
logger.info("Waiting for postgres (attempt %s/%s): %s", attempt, retries, e)
await asyncio.sleep(wait)
raise RuntimeError(f"Could not connect to postgres after {retries} attempts") from last_exc
Raises RuntimeError if DB is still unavailable after retries.
"""
last_exc = None
for attempt in range(1, retries + 1):
try:
async with engine.connect() as conn:
await conn.execute(text("SELECT 1"))
return
except Exception as e:
last_exc = e
wait = min(delay * (2 ** (attempt - 1)), 5)
await asyncio.sleep(wait)
raise RuntimeError(f"Could not connect to DB after {retries} attempts") from last_exc
async def _create_database_if_missing() -> None:
admin = await asyncpg.connect(
user=settings.DB_USER,
password=settings.DB_PASS,
database="postgres",
host=settings.DB_HOST,
port=int(settings.DB_PORT),
)
try:
exists = await admin.fetchval(
"SELECT 1 FROM pg_database WHERE datname=$1", settings.DB_NAME
)
if not exists:
await admin.execute(f'CREATE DATABASE "{settings.DB_NAME}"')
finally:
await admin.close()
async def init_db() -> None:
await _create_database_if_missing()
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)

View File

@@ -1,121 +0,0 @@
from api.db.connection import engine, SessionLocal
from api.models.subscription import Subscription
from api.models.base import Base
from api.config import settings
from datetime import datetime, timedelta
from sqlalchemy import select
import asyncpg
async def init_db():
try:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
return
except Exception as exc: # try to create database if it doesn't exist
msg = str(exc).lower()
if "does not exist" in msg or "invalidcatalogname" in msg or "database" in msg:
try:
admin_conn = await asyncpg.connect(
user=settings.DB_USER,
password=settings.DB_PASS,
database="postgres",
host=settings.DB_HOST,
port=int(settings.DB_PORT),
)
try:
await admin_conn.execute(f'CREATE DATABASE "{settings.DB_NAME}"')
except Exception:
pass
await admin_conn.close()
except Exception:
raise
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
return
raise
async def create_subscription(user_id: str, uuid: str, until: datetime, subscription_link: str | None = None, connect_link: str | None = None, active: bool = True):
async with SessionLocal() as session:
# if user already has a subscription, remove it (replace)
q = select(Subscription).where(Subscription.user_id == user_id)
res = await session.execute(q)
existing = res.scalars().first()
if existing:
await session.delete(existing)
# avoid NULL in DB: replace None links with empty strings
safe_subscription_link = subscription_link if subscription_link is not None else ""
safe_connect_link = connect_link if connect_link is not None else ""
sub = Subscription(user_id=user_id, uuid=uuid, until=until, active=active, subscription_link=safe_subscription_link, connect_link=safe_connect_link)
session.add(sub)
await session.commit()
await session.refresh(sub)
return sub
async def get_subscription_by_user(user_id: str):
async with SessionLocal() as session:
q = select(Subscription).where(Subscription.user_id == user_id)
res = await session.execute(q)
sub = res.scalars().first()
return sub
async def get_subscription_by_uuid(sub_uuid: str):
async with SessionLocal() as session:
q = select(Subscription).where(Subscription.uuid == sub_uuid)
res = await session.execute(q)
return res.scalars().first()
async def update_subscription(sub_uuid: str, until: datetime = None, user_id: str | None = None, active: bool | None = None, subscription_link: str | None = None, connect_link: str | None = None):
async with SessionLocal() as session:
q = select(Subscription).where(Subscription.uuid == sub_uuid)
res = await session.execute(q)
sub = res.scalars().first()
if not sub:
return None
if until is not None:
sub.until = until
if user_id is not None:
sub.user_id = user_id
if active is not None:
sub.active = active
if subscription_link is not None:
sub.subscription_link = subscription_link
if connect_link is not None:
sub.connect_link = connect_link
await session.commit()
return sub
async def delete_subscription(sub_uuid: str):
async with SessionLocal() as session:
q = select(Subscription).where(Subscription.uuid == sub_uuid)
res = await session.execute(q)
sub = res.scalars().first()
if not sub:
return False
await session.delete(sub)
await session.commit()
return True
async def list_subscriptions():
async with SessionLocal() as session:
q = select(Subscription)
res = await session.execute(q)
return res.scalars().all()
async def extend_subscription_by_user(user_id: str, months: int):
sub = await get_subscription_by_user(user_id)
if sub:
new_until = max(sub.until, datetime.now()) + timedelta(days=30 * months)
return await update_subscription(sub.uuid, until=new_until, active=True)
return None
async def deactivate_subscription(sub_uuid: str):
return await update_subscription(sub_uuid, active=False)

View File

@@ -1,64 +0,0 @@
from api.db.connection import SessionLocal
from api.models.user import User
from sqlalchemy import select
import uuid
async def create_user(telegram_id: str, name: str | None = None):
async with SessionLocal() as session:
# ensure no NULL in DB: replace None with empty string
safe_name = name if name is not None else ""
user = User(id=str(uuid.uuid4()), telegram_id=telegram_id, name=safe_name)
session.add(user)
await session.commit()
await session.refresh(user)
return user
async def get_user(user_id: str):
async with SessionLocal() as session:
q = select(User).where(User.id == user_id)
res = await session.execute(q)
return res.scalars().first()
async def get_user_by_telegram(telegram_id: str):
async with SessionLocal() as session:
q = select(User).where(User.telegram_id == telegram_id)
res = await session.execute(q)
return res.scalars().first()
async def update_user(user_id: str, name: str | None = None):
async with SessionLocal() as session:
q = select(User).where(User.id == user_id)
res = await session.execute(q)
user = res.scalars().first()
if not user:
return None
if name is not None:
# replace None handled by caller; here update only when provided
user.name = name
await session.commit()
return user
async def delete_user(user_id: str):
async with SessionLocal() as session:
q = select(User).where(User.id == user_id)
res = await session.execute(q)
user = res.scalars().first()
if not user:
return False
await session.delete(user)
await session.commit()
return True
async def list_users():
async with SessionLocal() as session:
q = select(User)
res = await session.execute(q)
return res.scalars().all()

53
api/db/video.py Normal file
View File

@@ -0,0 +1,53 @@
from sqlalchemy import select
from api.db.connection import SessionLocal
from api.models.video import Video
async def create_video(
*,
uuid: str,
source_url: str,
title: str,
video_path: str,
text_full_path: str,
text_summary_path: str,
transcription_method: str,
) -> Video:
async with SessionLocal() as session:
v = Video(
uuid=uuid,
source_url=source_url,
title=title or "",
video_path=video_path or "",
text_full_path=text_full_path or "",
text_summary_path=text_summary_path or "",
transcription_method=transcription_method or "",
)
session.add(v)
await session.commit()
await session.refresh(v)
return v
async def get_video(video_uuid: str) -> Video | None:
async with SessionLocal() as session:
res = await session.execute(select(Video).where(Video.uuid == video_uuid))
return res.scalars().first()
async def list_videos() -> list[Video]:
async with SessionLocal() as session:
res = await session.execute(select(Video).order_by(Video.created_at.desc()))
return list(res.scalars().all())
async def delete_video(video_uuid: str) -> bool:
async with SessionLocal() as session:
res = await session.execute(select(Video).where(Video.uuid == video_uuid))
v = res.scalars().first()
if not v:
return False
await session.delete(v)
await session.commit()
return True

0
api/lib/__init__.py Normal file
View File

View File

@@ -1,134 +0,0 @@
from pathlib import Path
import os
import sys
import threading
import logging
import wave
import json
import subprocess
import shutil
try:
from vosk import Model, KaldiRecognizer
except Exception:
Model = None
KaldiRecognizer = None
class FFmpegConverter:
"""Utility to convert audio files using ffmpeg."""
@staticmethod
def convert_mp3_to_wav(src: Path, dest: Path) -> None:
cmd = [
"ffmpeg",
"-y",
"-i",
str(src),
"-ac",
"1",
"-ar",
"16000",
"-vn",
"-f",
"wav",
str(dest),
]
subprocess.run(cmd, check=True)
class VoskModelManager:
"""Loads a Vosk model and provides transcription methods.
Loading captures model stderr into Python logs to show progress.
"""
def __init__(self, model_path: Path):
self.logger = logging.getLogger("vosk.model_manager")
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
self.model = self._load_model_with_progress(model_path)
def _load_model_with_progress(self, path: Path):
logger = logging.getLogger("vosk.loader")
r_fd, w_fd = os.pipe()
saved_stderr_fd = os.dup(sys.stderr.fileno())
os.dup2(w_fd, sys.stderr.fileno())
def _reader(fd):
with os.fdopen(fd, "rb") as fh:
for raw in iter(fh.readline, b""):
try:
logger.info(raw.decode(errors="ignore").rstrip())
except Exception:
pass
reader_thread = threading.Thread(target=_reader, args=(r_fd,), daemon=True)
reader_thread.start()
try:
model = Model(str(path))
finally:
try:
os.dup2(saved_stderr_fd, sys.stderr.fileno())
except Exception:
pass
try:
os.close(saved_stderr_fd)
except Exception:
pass
reader_thread.join(timeout=2)
return model
def transcribe_wav(self, wav_path: Path) -> str:
if not wav_path.exists():
raise FileNotFoundError(wav_path)
try:
with wave.open(str(wav_path), "rb") as wf:
if wf.getnchannels() != 1:
raise ValueError("Audio must be mono (1 channel)")
recognizer = KaldiRecognizer(self.model, wf.getframerate())
try:
recognizer.SetWords(True)
except Exception:
# older vosk bindings may not expose SetWords; ignore if absent
pass
while True:
data = wf.readframes(4000)
if not data:
break
recognizer.AcceptWaveform(data)
result = json.loads(recognizer.FinalResult())
return result.get("text", "")
except wave.Error as e:
raise ValueError(f"Invalid WAV file: {e}")
def transcribe_wav_with_words(self, wav_path: Path):
"""Transcribe WAV and return dict with 'text' and 'words' (word-level timestamps).
Returns: { 'text': str, 'words': [{'word': str, 'start': float, 'end': float}, ...] }
"""
if not wav_path.exists():
raise FileNotFoundError(wav_path)
try:
with wave.open(str(wav_path), "rb") as wf:
if wf.getnchannels() != 1:
raise ValueError("Audio must be mono (1 channel)")
recognizer = KaldiRecognizer(self.model, wf.getframerate())
try:
recognizer.SetWords(True)
except Exception:
pass
while True:
data = wf.readframes(4000)
if not data:
break
recognizer.AcceptWaveform(data)
res = json.loads(recognizer.FinalResult())
words = res.get("result", [])
# words are dictionaries with word, start, end
text = res.get("text", "")
return {"text": text, "words": words}
except wave.Error as e:
raise ValueError(f"Invalid WAV file: {e}")

View File

@@ -1,69 +0,0 @@
from pathlib import Path
from typing import List, Dict
def diarize_with_resemblyzer(wav_path: Path, window_s: float = 1.5, hop_s: float = 0.75, distance_threshold: float = 0.6) -> List[Dict]:
"""Lightweight embedding-based diarization using resemblyzer + sklearn.
Returns list of segments: {'start': float, 'end': float, 'speaker': 'spk_N'}
"""
try:
import numpy as np
import librosa
from resemblyzer import VoiceEncoder
from sklearn.cluster import AgglomerativeClustering
except Exception as e:
raise RuntimeError(f"Missing dependency for embedding diarization: {e}")
sr = 16000
wav, sr_loaded = librosa.load(str(wav_path), sr=sr)
n = wav.shape[0]
win = int(window_s * sr)
hop = int(hop_s * sr)
if n <= win:
# short file: embed whole
encoder = VoiceEncoder()
emb = encoder.embed_utterance(wav)
labels = [0]
centers = [ (0.0 + n / sr) / 2.0 ]
windows = [(0.0, n / sr)]
else:
encoder = VoiceEncoder()
embeddings = []
centers = []
windows = []
for start in range(0, n - win + 1, hop):
chunk = wav[start:start+win]
try:
e = encoder.embed_utterance(chunk)
except Exception:
# fallback: mean pooling
e = np.mean(chunk)
embeddings.append(e)
t0 = start / sr
t1 = (start + win) / sr
centers.append((t0 + t1) / 2.0)
windows.append((t0, t1))
X = np.vstack(embeddings)
# Agglomerative clustering with distance threshold
model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold, affinity='euclidean', linkage='average')
labels = model.fit_predict(X)
# merge consecutive windows with same label into segments
segs = []
if len(labels) == 0:
return segs
cur_label = labels[0]
cur_start = windows[0][0]
cur_end = windows[0][1]
for i, lab in enumerate(labels[1:], start=1):
if lab == cur_label:
cur_end = windows[i][1]
else:
segs.append({"start": cur_start, "end": cur_end, "speaker": f"spk_{int(cur_label)+1}"})
cur_label = lab
cur_start = windows[i][0]
cur_end = windows[i][1]
segs.append({"start": cur_start, "end": cur_end, "speaker": f"spk_{int(cur_label)+1}"})
return segs

164
api/lib/llm.py Normal file
View File

@@ -0,0 +1,164 @@
"""Generic OpenAI-compatible chat-completions client.
The LLM is *not* deployed in this container — point this client at any external
endpoint that speaks the OpenAI Chat Completions wire format:
- https://api.openai.com/v1
- https://openrouter.ai/api/v1
- https://api.anthropic.com (via OpenAI-compat proxies)
- http://host.docker.internal:11434/v1 (local Ollama)
- http://vllm-host:8000/v1 (vLLM / llama.cpp server / LM Studio)
Configure once via environment (LLM_BASE_URL, LLM_API_KEY, LLM_MODEL),
or override per-request through /pipeline/process by passing an `llm` block.
"""
from __future__ import annotations
import json
import logging
import urllib.error
import urllib.request
from typing import Optional
from pydantic import BaseModel, Field
from api.config import settings
logger = logging.getLogger("llm")
class LLMConfig(BaseModel):
"""Wire-level config for an OpenAI-compatible chat-completions endpoint."""
base_url: str = Field(..., description="Base URL ending in /v1 (or equivalent)")
api_key: str = ""
# `model` is optional so the same config can be used for /v1/models discovery
# before the user has picked one.
model: str = ""
timeout: int = 120
max_tokens: int = 1000
temperature: float = 0.3
extra_headers: dict[str, str] = Field(default_factory=dict)
class LLMError(RuntimeError):
pass
def from_settings() -> Optional[LLMConfig]:
"""Build LLMConfig from environment variables, or return None if not configured."""
if not settings.LLM_BASE_URL or not settings.LLM_MODEL:
return None
return LLMConfig(
base_url=settings.LLM_BASE_URL,
api_key=settings.LLM_API_KEY,
model=settings.LLM_MODEL,
timeout=settings.LLM_TIMEOUT,
max_tokens=settings.LLM_MAX_TOKENS,
temperature=settings.LLM_TEMPERATURE,
)
def chat_complete(cfg: LLMConfig, messages: list[dict]) -> str:
"""Synchronous Chat Completions call. Returns assistant message content."""
url = cfg.base_url.rstrip("/") + "/chat/completions"
payload: dict = {
"model": cfg.model,
"messages": messages,
"max_tokens": cfg.max_tokens,
"temperature": cfg.temperature,
}
headers = {"Content-Type": "application/json", "Accept": "application/json"}
if cfg.api_key:
headers["Authorization"] = f"Bearer {cfg.api_key}"
headers.update(cfg.extra_headers or {})
body = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
try:
with urllib.request.urlopen(req, timeout=cfg.timeout) as resp:
raw = resp.read()
except urllib.error.HTTPError as e:
try:
err_body = e.read().decode("utf-8", errors="ignore")
except Exception:
err_body = ""
raise LLMError(f"LLM HTTP {e.code} {e.reason}: {err_body[:500]}")
except Exception as e:
raise LLMError(f"LLM request failed: {e}")
try:
data = json.loads(raw.decode("utf-8"))
choices = data.get("choices") or []
if not choices:
raise LLMError(f"LLM returned no choices: {data}")
msg = choices[0].get("message") or {}
content = msg.get("content")
if content is None:
raise LLMError(f"LLM choice has no content: {choices[0]}")
return content.strip()
except LLMError:
raise
except Exception as e:
raise LLMError(f"Failed to parse LLM response: {e}")
def ping(cfg: LLMConfig) -> str:
"""Tiny round-trip to verify connectivity. Returns the assistant reply text."""
return chat_complete(
cfg,
[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Reply with the single word OK."},
],
)
def list_models(cfg: LLMConfig) -> list[dict]:
"""GET {base_url}/models. Returns a normalized list of {id, name, owned_by?}.
Compatible with OpenAI, OpenRouter, Ollama OpenAI-compat, vLLM, LM Studio.
"""
url = cfg.base_url.rstrip("/") + "/models"
headers = {"Accept": "application/json"}
if cfg.api_key:
headers["Authorization"] = f"Bearer {cfg.api_key}"
headers.update(cfg.extra_headers or {})
req = urllib.request.Request(url, headers=headers, method="GET")
try:
with urllib.request.urlopen(req, timeout=cfg.timeout) as resp:
raw = resp.read()
except urllib.error.HTTPError as e:
try:
err_body = e.read().decode("utf-8", errors="ignore")
except Exception:
err_body = ""
raise LLMError(f"LLM HTTP {e.code} {e.reason}: {err_body[:500]}")
except Exception as e:
raise LLMError(f"List models failed: {e}")
try:
data = json.loads(raw.decode("utf-8"))
except Exception as e:
raise LLMError(f"Failed to parse models response: {e}")
items = data.get("data") or data.get("models") or data.get("results") or []
out: list[dict] = []
for item in items:
if isinstance(item, str):
out.append({"id": item, "name": item})
continue
if not isinstance(item, dict):
continue
mid = item.get("id") or item.get("name") or item.get("model")
if not mid:
continue
entry = {"id": mid, "name": item.get("name") or mid}
owned = item.get("owned_by") or item.get("organization")
if owned:
entry["owned_by"] = owned
out.append(entry)
out.sort(key=lambda x: x["id"].lower())
return out

221
api/lib/summarize.py Normal file
View File

@@ -0,0 +1,221 @@
"""Summarization with two backends:
* `summarize_llm` — uses any external OpenAI-compatible API (configured via
`LLMConfig`). Long transcripts are map-reduced: per-chunk summaries are
concatenated and re-summarized.
* `summarize_extractive` — pure-Python TF-IDF-ish fallback (RU + EN aware).
`summarize(text, llm_cfg=...)` picks the LLM path when a config is supplied,
otherwise falls back to extractive. The pipeline calls this entry point.
"""
from __future__ import annotations
import logging
import re
from collections import Counter
from typing import Optional
from api.lib.llm import LLMConfig, LLMError, chat_complete
logger = logging.getLogger("summarize")
_RU_STOPWORDS = {
"и", "в", "во", "не", "на", "я", "что", "тот", "быть", "с", "со", "а", "весь",
"как", "это", "но", "он", "она", "оно", "мы", "вы", "они", "к", "у", "из", "за",
"по", "до", "от", "для", "же", "или", "бы", "если", "так", "там", "тут", "когда",
"где", "кто", "какой", "этот", "эта", "эти", "мой", "твой", "наш", "ваш",
"свой", "его", "её", "их", "был", "была", "было", "были", "есть", "нет", "да",
"ну", "вот", "ещё", "еще", "уже", "только", "очень", "может", "можно", "надо",
"будет", "будут", "всё", "все", "этого", "этом", "этой", "тоже", "также", "чтобы",
"кого", "чего", "чем", "тем", "том", "нем", "ней", "себя", "себе", "мне", "тебе",
"нам", "вам", "нас", "вас", "про", "без", "над", "под", "через", "при", "об",
"о", "обо", "ли", "вон", "сюда", "туда", "оттуда", "потом", "теперь", "тогда",
"сейчас", "иногда", "всегда", "никогда", "просто", "именно", "ведь", "значит",
"хотя", "что-то", "кто-то",
}
_EN_STOPWORDS = {
"the", "a", "an", "and", "or", "but", "is", "are", "was", "were", "be", "been",
"being", "have", "has", "had", "do", "does", "did", "will", "would", "should",
"could", "may", "might", "must", "shall", "can", "i", "you", "he", "she", "it",
"we", "they", "them", "my", "your", "his", "her", "its", "our", "their", "this",
"that", "these", "those", "in", "on", "at", "by", "for", "with", "about", "as",
"of", "to", "from", "into", "so", "not", "no", "yes", "if", "then", "than",
"when", "where", "why", "how", "what", "which", "who", "whom", "whose", "me",
"us", "him", "just", "only", "also", "too", "very", "there", "here", "out",
"up", "down", "over", "under", "between", "through", "again", "more", "most",
"some", "any", "all", "each", "every", "such", "while", "because", "until",
"during", "above", "below", "now", "ever", "never",
}
_STOPWORDS = _RU_STOPWORDS | _EN_STOPWORDS
_WORD_RE = re.compile(r"[\w']+", re.UNICODE)
_SENT_SPLIT_RE = re.compile(r"(?<=[.!?…])\s+(?=[А-ЯA-Z\"«„])")
DEFAULT_SYSTEM_PROMPT = (
"Ты ассистент, который делает краткую и точную выжимку текста. "
"Сохраняй ключевые факты, имена, цифры, причинно-следственные связи. "
"Если оригинал на русском — пиши по-русски, иначе — на языке оригинала. "
"Не добавляй информацию, которой нет в исходном тексте."
)
DEFAULT_USER_TEMPLATE = (
"Сделай связную краткую выжимку следующего текста. "
"Выдели основные тезисы и логику изложения. Объём — 512 предложений.\n\n"
"---\n{text}\n---"
)
DEFAULT_REDUCE_TEMPLATE = (
"Ниже — последовательность кратких выжимок частей одного длинного текста. "
"Объедини их в единую связную выжимку, сохранив главные факты и логику.\n\n"
"---\n{text}\n---"
)
def _split_sentences(text: str) -> list[str]:
text = re.sub(r"\s+", " ", text).strip()
if not text:
return []
parts = _SENT_SPLIT_RE.split(text)
if len(parts) <= 1:
parts = re.split(r"(?<=[.!?…])\s+", text)
return [p.strip() for p in parts if p.strip()]
def _chunk_by_chars(text: str, max_chars: int) -> list[str]:
if len(text) <= max_chars:
return [text]
chunks: list[str] = []
cur: list[str] = []
cur_len = 0
for s in _split_sentences(text):
if cur_len + len(s) + 1 > max_chars and cur:
chunks.append(" ".join(cur))
cur, cur_len = [], 0
cur.append(s)
cur_len += len(s) + 1
if cur:
chunks.append(" ".join(cur))
return chunks
def summarize_extractive(
text: str,
*,
max_sentences: int = 15,
min_sentences: int = 3,
ratio: float = 0.15,
) -> str:
"""Frequency-based extractive summary. Picks top sentences by mean word weight."""
text = (text or "").strip()
if not text:
return ""
sentences = _split_sentences(text)
if not sentences:
return ""
target = max(min_sentences, min(max_sentences, int(len(sentences) * ratio)))
target = max(1, min(target, len(sentences)))
if len(sentences) <= target:
return " ".join(sentences)
words = _WORD_RE.findall(text.lower())
freq = Counter(w for w in words if w not in _STOPWORDS and len(w) > 2)
if not freq:
return " ".join(sentences[:target])
max_f = max(freq.values())
norm = {w: c / max_f for w, c in freq.items()}
scored: list[tuple[int, float]] = []
for i, s in enumerate(sentences):
sw = _WORD_RE.findall(s.lower())
if not sw:
continue
score = sum(norm.get(w, 0.0) for w in sw) / len(sw)
scored.append((i, score))
if not scored:
return " ".join(sentences[:target])
top = sorted(scored, key=lambda x: -x[1])[:target]
top.sort(key=lambda x: x[0])
return " ".join(sentences[i] for i, _ in top)
def summarize_llm(
text: str,
cfg: LLMConfig,
*,
system_prompt: Optional[str] = None,
user_template: Optional[str] = None,
reduce_template: Optional[str] = None,
chunk_chars: int = 10000,
) -> str:
"""Summarize via an external OpenAI-compatible API. Map-reduce for long texts."""
text = (text or "").strip()
if not text:
return ""
sys_p = system_prompt or DEFAULT_SYSTEM_PROMPT
user_t = user_template or DEFAULT_USER_TEMPLATE
reduce_t = reduce_template or DEFAULT_REDUCE_TEMPLATE
def _one(chunk: str, template: str) -> str:
return chat_complete(
cfg,
[
{"role": "system", "content": sys_p},
{"role": "user", "content": template.format(text=chunk)},
],
)
chunks = _chunk_by_chars(text, chunk_chars)
logger.info("LLM summarize: %d chunk(s), total chars=%d", len(chunks), len(text))
if len(chunks) == 1:
return _one(chunks[0], user_t)
partials: list[str] = []
for i, ch in enumerate(chunks, start=1):
logger.info("LLM summarize chunk %d/%d", i, len(chunks))
partials.append(_one(ch, user_t))
combined = "\n\n".join(partials)
if len(combined) > chunk_chars:
# If even concatenated partials are too long, recurse: summarize partials in batches.
return summarize_llm(
combined,
cfg,
system_prompt=sys_p,
user_template=reduce_t,
reduce_template=reduce_t,
chunk_chars=chunk_chars,
)
logger.info("LLM reduce step over %d partials", len(partials))
return _one(combined, reduce_t)
def summarize(
text: str,
*,
llm_cfg: Optional[LLMConfig] = None,
system_prompt: Optional[str] = None,
user_template: Optional[str] = None,
max_sentences: int = 15,
) -> str:
"""Primary entry point. Uses LLM if configured, else extractive fallback."""
if llm_cfg is not None:
try:
return summarize_llm(
text,
llm_cfg,
system_prompt=system_prompt,
user_template=user_template,
)
except LLMError as e:
logger.warning("LLM summarization failed, falling back to extractive: %s", e)
return summarize_extractive(text, max_sentences=max_sentences)

119
api/lib/transcribe.py Normal file
View File

@@ -0,0 +1,119 @@
"""Vosk-based fallback transcription: stream PCM from ffmpeg into a Vosk recognizer."""
from __future__ import annotations
import json
import logging
import os
import shutil
import subprocess
import sys
import threading
from pathlib import Path
try:
from vosk import KaldiRecognizer, Model
except Exception:
Model = None
KaldiRecognizer = None
logger = logging.getLogger("transcribe")
class VoskModelManager:
"""Loads a Vosk model. Captures the model's stderr to Python logs while loading."""
def __init__(self, model_path: Path):
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
self.model = self._load_with_stderr_capture(model_path)
@staticmethod
def _load_with_stderr_capture(path: Path):
log = logging.getLogger("vosk.loader")
r_fd, w_fd = os.pipe()
saved_stderr = os.dup(sys.stderr.fileno())
os.dup2(w_fd, sys.stderr.fileno())
os.close(w_fd)
def reader(fd: int) -> None:
with os.fdopen(fd, "rb") as fh:
for raw in iter(fh.readline, b""):
log.info(raw.decode(errors="ignore").rstrip())
t = threading.Thread(target=reader, args=(r_fd,), daemon=True)
t.start()
try:
return Model(str(path))
finally:
try:
os.dup2(saved_stderr, sys.stderr.fileno())
finally:
os.close(saved_stderr)
t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None:
raise RuntimeError("ffmpeg is not available in PATH")
if not media_path.exists():
raise FileNotFoundError(media_path)
mgr = VoskModelManager(model_path)
recognizer = KaldiRecognizer(mgr.model, sample_rate)
try:
recognizer.SetWords(True)
except Exception:
pass
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(media_path),
"-f",
"s16le",
"-acodec",
"pcm_s16le",
"-ac",
"1",
"-ar",
str(sample_rate),
"-vn",
"-",
]
logger.info("Spawning ffmpeg streaming decode for %s", media_path.name)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if proc.stdout is None:
proc.kill()
raise RuntimeError("ffmpeg did not provide stdout")
try:
bytes_read = 0
last_logged_mb = 0
while True:
chunk = proc.stdout.read(4000)
if not chunk:
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024)
if mb >= last_logged_mb + 5:
last_logged_mb = mb
logger.info("Transcribed %d MB of audio so far", mb)
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")
logger.info("Transcription finished, length=%d", len(text))
return text
finally:
try:
proc.kill()
except Exception:
pass

View File

@@ -1,236 +0,0 @@
from pathlib import Path
import wave
import json
import math
from typing import List, Dict
def _rms(frames: bytes, sample_width: int) -> float:
# compute RMS of raw frames (little-endian signed samples)
if sample_width == 2:
import struct
count = len(frames) // 2
if count == 0:
return 0.0
fmt = "<%dh" % count
vals = struct.unpack(fmt, frames)
s = 0
for v in vals:
s += v * v
return math.sqrt(s / count)
else:
return 0.0
def diarize_by_silence(wav_path: Path, win_ms: int = 30, silence_thresh: float = 500.0, min_silence_ms: int = 400) -> List[Dict]:
"""Naive diarization by splitting on long silence.
Returns list of segments: {'start': float, 'end': float, 'speaker': str}
Speakers are assigned alternately: Speaker 1, Speaker 2, ...
"""
segs = []
with wave.open(str(wav_path), "rb") as wf:
sr = wf.getframerate()
sw = wf.getsampwidth()
n_channels = wf.getnchannels()
# we expect mono
frames_per_win = int(sr * win_ms / 1000)
min_silence_frames = int(sr * min_silence_ms / 1000)
total_frames = wf.getnframes()
pos = 0
silent_acc = 0
current_start = 0
speaker_idx = 1
while pos < total_frames:
to_read = min(frames_per_win, total_frames - pos)
raw = wf.readframes(to_read)
pos += to_read
level = _rms(raw, sw)
if level < silence_thresh:
silent_acc += to_read
else:
# if there was long silence before speech, start a new segment
if silent_acc >= min_silence_frames and pos / sr > current_start:
end_time = (pos - silent_acc) / sr
segs.append({"start": current_start, "end": end_time, "speaker": f"Speaker {speaker_idx}"})
speaker_idx = (speaker_idx % 2) + 1
current_start = end_time
silent_acc = 0
# finish last segment
if current_start < total_frames / sr:
segs.append({"start": current_start, "end": total_frames / sr, "speaker": f"Speaker {speaker_idx}"})
return segs
def assign_words_to_speakers(words: List[Dict], segments: List[Dict]) -> List[Dict]:
"""Assign word dicts (with start/end) to speaker segments. Returns list of speaker blocks with text."""
result = []
seg_idx = 0
current_block = {"speaker": segments[0]["speaker"] if segments else "Speaker 1", "start": None, "end": None, "words": []}
for w in words:
t = w.get("start", 0)
# move seg_idx until this word falls into segment
while seg_idx + 1 < len(segments) and t >= segments[seg_idx]["end"]:
# flush current
if current_block["words"]:
current_block["end"] = segments[seg_idx]["end"]
result.append(current_block)
seg_idx += 1
current_block = {"speaker": segments[seg_idx]["speaker"], "start": None, "end": None, "words": []}
# append word
if current_block["start"] is None:
current_block["start"] = w.get("start")
current_block["words"].append(w)
if current_block["words"]:
# set end
current_block["end"] = current_block["words"][-1].get("end")
result.append(current_block)
return result
def chapter_by_wordcount(words: List[Dict], speaker_blocks: List[Dict] = None, words_per_chapter: int = 100, min_words_on_speaker_change: int = 30) -> List[Dict]:
"""Chaptering that prefers boundaries at speaker changes.
- If `speaker_blocks` provided, will try to split a chapter when speaker changes
and the current chapter has at least `min_words_on_speaker_change` words.
- Also split when `words_per_chapter` is reached.
Returns list of chapters: {'start': float, 'end': float, 'words': [...]}.
"""
chapters = []
chapter = {"start": None, "end": None, "words": []}
count = 0
def speaker_for_time(t: float):
if not speaker_blocks or t is None:
return None
for b in speaker_blocks:
bstart = b.get("start")
bend = b.get("end")
if bstart is None or bend is None:
continue
try:
if bstart <= t <= bend:
return b.get("speaker")
except Exception:
continue
return None
prev_speaker = None
for w in words:
wstart = w.get("start")
if chapter["start"] is None:
chapter["start"] = wstart
chapter["words"].append(w)
chapter["end"] = w.get("end")
count += 1
# check speaker change boundary
cur_speaker = speaker_for_time(wstart)
if prev_speaker is None:
prev_speaker = cur_speaker
if cur_speaker is not None and prev_speaker is not None and cur_speaker != prev_speaker:
if count >= min_words_on_speaker_change:
# close chapter at previous word
chapters.append(chapter)
chapter = {"start": None, "end": None, "words": []}
count = 0
prev_speaker = cur_speaker
continue
else:
# don't split if too short to make a chapter
prev_speaker = cur_speaker
# check fixed-size boundary
if count >= words_per_chapter:
chapters.append(chapter)
chapter = {"start": None, "end": None, "words": []}
count = 0
prev_speaker = None
if chapter["words"]:
chapters.append(chapter)
return chapters
def save_transcript(text_dir: Path, base_name: str, speaker_blocks: List[Dict], chapters: List[Dict]):
text_dir.mkdir(parents=True, exist_ok=True)
out_json = text_dir / f"{base_name}.json"
out_txt = text_dir / f"{base_name}.txt"
# Automatic host labeling: speaker with most words -> 'Ведущий', others -> 'Участник N'
try:
# Prefer selecting host by total spoken duration (better for long monologues).
durations = []
for b in speaker_blocks:
dur = 0.0
for w in b.get("words", []):
s = w.get("start")
e = w.get("end")
if s is None or e is None:
continue
try:
dur += float(e) - float(s)
except Exception:
continue
durations.append(dur)
if any(durations):
host_idx = int(max(range(len(durations)), key=lambda i: durations[i]))
else:
# fallback to word counts if timestamps missing
counts = [sum(1 for w in b.get("words", [])) for b in speaker_blocks]
host_idx = int(max(range(len(counts)), key=lambda i: counts[i])) if counts else 0
new_blocks = []
participant_idx = 1
for i, b in enumerate(speaker_blocks):
b_copy = dict(b)
if i == host_idx:
b_copy["speaker"] = "Ведущий"
else:
b_copy["speaker"] = f"Участник {participant_idx}"
participant_idx += 1
new_blocks.append(b_copy)
speaker_blocks = new_blocks
except Exception:
# fall back to provided labels on error
pass
data = {"speakers": speaker_blocks, "chapters": chapters}
with open(out_json, "w", encoding="utf-8") as fh:
json.dump(data, fh, ensure_ascii=False, indent=2)
# plain text with speaker labels and chapter headings
with open(out_txt, "w", encoding="utf-8") as fh:
for i, ch in enumerate(chapters, start=1):
fh.write(f"=== Глава {i} ===\n")
# collect speaker text overlapping this chapter by word membership
# build a simple view: iterate speaker blocks and print their words that fall into chapter
for b in speaker_blocks:
# collect words in this chapter that belong to this speaker block
b_words = []
for w in ch["words"]:
# a word belongs to speaker block if its start is within block start/end (if available)
wstart = w.get("start")
if wstart is None:
continue
bstart = b.get("start")
bend = b.get("end")
if bstart is None or bend is None:
continue
if bstart <= wstart <= bend:
b_words.append(w["word"])
if b_words:
fh.write(f"{b['speaker']}: ")
fh.write(" ".join(b_words) + "\n")
fh.write("\n")
return out_json, out_txt

172
api/lib/youtube.py Normal file
View File

@@ -0,0 +1,172 @@
"""yt-dlp wrapper: download a YouTube video together with available subtitles."""
from __future__ import annotations
import logging
import re
from pathlib import Path
from typing import Optional
import yt_dlp
logger = logging.getLogger("youtube")
_VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".m4a", ".mp3", ".mov"}
def download_video_with_subs(
url: str,
video_id: str,
video_dir: Path,
langs: Optional[list[str]] = None,
) -> dict:
"""Download a single video, also requesting subtitles for the given languages.
Returns:
{
"title": str,
"video_path": Path,
"subtitle_path": Path | None,
"subtitle_lang": str | None,
"subtitle_kind": "manual" | "auto" | None,
}
"""
if langs is None:
langs = ["ru", "en"]
video_dir.mkdir(parents=True, exist_ok=True)
outtmpl = str(video_dir / f"{video_id}.%(ext)s")
ydl_opts = {
"outtmpl": outtmpl,
"format": "bv*+ba/b",
"merge_output_format": "mp4",
"writesubtitles": True,
"writeautomaticsub": True,
"subtitleslangs": langs,
"subtitlesformat": "vtt",
"quiet": True,
"no_warnings": True,
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url, download=True)
title = info.get("title") or video_id
# Locate the merged video file
video_path = video_dir / f"{video_id}.mp4"
if not video_path.exists():
for p in video_dir.glob(f"{video_id}.*"):
if p.suffix.lower() in _VIDEO_EXTS:
video_path = p
break
sub_path, sub_lang, sub_kind = _find_subtitle(video_dir, video_id, langs, info)
return {
"title": title,
"video_path": video_path,
"subtitle_path": sub_path,
"subtitle_lang": sub_lang,
"subtitle_kind": sub_kind,
}
def _find_subtitle(
video_dir: Path,
video_id: str,
langs: list[str],
info: dict,
) -> tuple[Optional[Path], Optional[str], Optional[str]]:
"""Pick the best subtitle file among downloaded ones, preferring manual subs."""
requested = info.get("requested_subtitles") or {}
manual_keys = set(info.get("subtitles", {}).keys())
# Build a list of (lang, kind, path) candidates from yt-dlp's reported requested_subtitles
candidates: list[tuple[str, str, Path]] = []
for lang_code, sub_info in requested.items():
filepath = sub_info.get("filepath")
if not filepath:
continue
p = Path(filepath)
if not p.exists():
continue
kind = "manual" if lang_code in manual_keys else "auto"
candidates.append((lang_code, kind, p))
# Fallback: glob the directory
if not candidates:
for p in video_dir.glob(f"{video_id}*.vtt"):
m = re.match(rf"^{re.escape(video_id)}\.([A-Za-z0-9_\-]+)\.vtt$", p.name)
lang_code = m.group(1) if m else "unknown"
candidates.append((lang_code, "auto", p))
if not candidates:
return None, None, None
def score(c: tuple[str, str, Path]) -> tuple[int, int]:
lang, kind, _ = c
try:
lang_rank = next(
i for i, l in enumerate(langs) if lang == l or lang.startswith(l + "-")
)
except StopIteration:
lang_rank = len(langs)
kind_rank = 0 if kind == "manual" else 1
return (lang_rank, kind_rank)
candidates.sort(key=score)
lang, kind, path = candidates[0]
return path, lang, kind
def vtt_to_text(vtt_content: str) -> str:
"""Convert a VTT subtitle file to a deduplicated plain-text string.
YouTube auto-captions ship as rolling cues — each new cue extends the previous.
Strategy: per cue block, keep only the last text line, then drop consecutive
duplicates and identical sentence fragments.
"""
blocks = re.split(r"\r?\n\s*\r?\n", vtt_content)
out: list[str] = []
last = ""
for block in blocks:
block = block.strip()
if not block:
continue
text_lines: list[str] = []
for raw in block.splitlines():
line = raw.strip()
if not line:
continue
if line.startswith("WEBVTT"):
continue
if line.startswith(("NOTE", "STYLE", "Kind:", "Language:", "Region:")):
continue
if "-->" in line:
continue
if re.match(r"^\d+$", line):
continue
line = re.sub(r"<[^>]+>", "", line)
line = re.sub(r"\s+", " ", line).strip()
if line:
text_lines.append(line)
if not text_lines:
continue
candidate = text_lines[-1]
if candidate == last:
continue
# Avoid the rolling-caption case where a cue strictly extends the previous one
if last and candidate.startswith(last):
out[-1] = candidate
last = candidate
continue
if last and last.startswith(candidate):
continue
out.append(candidate)
last = candidate
return " ".join(out)

View File

@@ -1,22 +1,28 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from api.route import default, subscription
# renamed route modules for clarity
from api.route import users, installing, media_convert, mp3_ffmpeg_stream, moviepy
#from api.db.subscription import init_db
#from api.db.connection import wait_for_db
app = FastAPI(title="LLM-infa API")
from api.db.connection import wait_for_postgres, init_db
from api.route import default, llm, pipeline, videos
#@app.on_event("startup")
#async def on_startup():
# await wait_for_db()
# await init_db()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
@asynccontextmanager
async def lifespan(app: FastAPI):
await wait_for_postgres()
await init_db()
yield
app = FastAPI(title="LLM-infa API", version="0.3.0", lifespan=lifespan)
app.include_router(default.router)
#app.include_router(subscription.router)
#app.include_router(users.router)
app.include_router(installing.router)
app.include_router(media_convert.router)
app.include_router(mp3_ffmpeg_stream.router)
app.include_router(moviepy.router)
app.include_router(pipeline.router)
app.include_router(videos.router)
app.include_router(llm.router)

View File

@@ -1,3 +0,0 @@
from sqlalchemy.orm import declarative_base
Base = declarative_base()

View File

@@ -1,14 +0,0 @@
from sqlalchemy import Column, String, DateTime, ForeignKey, Boolean, Text
from api.models.base import Base
class Subscription(Base):
__tablename__ = 'subscriptions'
uuid = Column(String, primary_key=True, index=True)
# one subscription per user (unique)
user_id = Column(String, ForeignKey('users.id'), nullable=False, index=True, unique=True)
until = Column(DateTime, nullable=False)
active = Column(Boolean, nullable=False, default=True)
# avoid NULL: store empty string when no link provided
subscription_link = Column(Text, nullable=False, default="")
connect_link = Column(Text, nullable=False, default="")

View File

@@ -1,12 +0,0 @@
from sqlalchemy import Column, String, DateTime
from sqlalchemy.sql import func
from api.models.base import Base
import uuid
class User(Base):
__tablename__ = 'users'
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
telegram_id = Column(String, unique=True, index=True, nullable=False)
name = Column(String, nullable=False, default="")
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)

0
api/route/__init__.py Normal file
View File

View File

@@ -1,16 +1,19 @@
from fastapi import APIRouter
import toml
from pathlib import Path
import toml
from fastapi import APIRouter
router = APIRouter()
@router.get("/")
async def root():
version = "dev"
pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
if pyproject_path.exists():
pyproject = Path(__file__).resolve().parent.parent.parent / "pyproject.toml"
if pyproject.exists():
try:
data = toml.load(pyproject_path)
data = toml.load(pyproject)
version = data.get("project", {}).get("version", "dev")
except Exception:
pass

View File

@@ -1,48 +0,0 @@
# Импортируем необходимые модули
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
import yt_dlp
import tempfile
from fastapi.responses import FileResponse
import os
router = APIRouter(prefix="/yt-dlp", tags=["yt-dlp"])
class DownloadRequest(BaseModel):
url: str
format: str = "best"
audio_only: bool = False
@router.post("/download")
async def download_video(data: DownloadRequest):
"""
Скачать видео по ссылке через yt-dlp и вернуть файл пользователю.
"""
save_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data'))
os.makedirs(save_dir, exist_ok=True)
# choose output dir and format
if data.audio_only:
out_dir = os.path.join(save_dir, 'audio')
os.makedirs(out_dir, exist_ok=True)
outtmpl = os.path.join(out_dir, '%(title)s.%(ext)s')
ydl_format = data.format if data.format != "best" else "bestaudio[ext=m4a]/bestaudio"
else:
outtmpl = os.path.join(save_dir, '%(title)s.%(ext)s')
ydl_format = data.format
ydl_opts = {
'outtmpl': outtmpl,
'format': ydl_format,
# Use aria2c for parallel segmented downloading to speed up large files
'external_downloader': 'aria2c',
'external_downloader_args': ['-x', '16', '-s', '16', '-k', '1M'],
}
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(data.url, download=True)
filename = ydl.prepare_filename(info)
except Exception as e:
raise HTTPException(status_code=400, detail=f"Ошибка загрузки: {e}")
if not os.path.exists(filename):
raise HTTPException(status_code=404, detail="Файл не найден после загрузки")
return FileResponse(filename, filename=os.path.basename(filename), media_type='application/octet-stream')

95
api/route/llm.py Normal file
View File

@@ -0,0 +1,95 @@
"""Endpoints for inspecting and testing the external LLM hookup.
The LLM is not hosted here — the user supplies a base URL + model + key.
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.lib.llm import (
LLMConfig,
LLMError,
chat_complete,
from_settings,
list_models,
ping,
)
router = APIRouter(prefix="/llm", tags=["llm"])
class LLMConfigIn(BaseModel):
base_url: str
api_key: str = ""
# model is optional here so /llm/models can be called before a model is picked
model: str = ""
timeout: int = 120
max_tokens: int = 1000
temperature: float = 0.3
extra_headers: dict[str, str] = {}
class LLMStatus(BaseModel):
configured: bool
base_url: str | None = None
model: str | None = None
class ChatIn(BaseModel):
config: LLMConfigIn | None = None
messages: list[dict]
@router.get("/config", response_model=LLMStatus)
async def get_status():
cfg = from_settings()
if cfg is None:
return LLMStatus(configured=False)
return LLMStatus(configured=True, base_url=cfg.base_url, model=cfg.model)
@router.post("/test")
async def test_connection(cfg: LLMConfigIn | None = None):
"""Send a tiny ping to verify the endpoint accepts our requests."""
config = LLMConfig(**cfg.model_dump()) if cfg else from_settings()
if config is None:
raise HTTPException(
status_code=400,
detail="No LLM config in body and none configured in environment",
)
try:
reply = ping(config)
except LLMError as e:
raise HTTPException(status_code=502, detail=str(e))
return {"ok": True, "model": config.model, "reply": reply}
@router.post("/models")
async def models(cfg: LLMConfigIn | None = None):
"""List available models from the configured (or supplied) endpoint."""
config = LLMConfig(**cfg.model_dump()) if cfg else from_settings()
if config is None:
raise HTTPException(
status_code=400,
detail="No LLM config in body and none configured in environment",
)
try:
items = list_models(config)
except LLMError as e:
raise HTTPException(status_code=502, detail=str(e))
return {"models": items, "count": len(items)}
@router.post("/chat")
async def chat(body: ChatIn):
"""Pass-through chat endpoint — useful for sanity-testing prompts."""
config = LLMConfig(**body.config.model_dump()) if body.config else from_settings()
if config is None:
raise HTTPException(status_code=400, detail="No LLM config supplied")
try:
reply = chat_complete(config, body.messages)
except LLMError as e:
raise HTTPException(status_code=502, detail=str(e))
return {"reply": reply, "model": config.model}

View File

@@ -1,344 +0,0 @@
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from fastapi.responses import FileResponse
import os
from pathlib import Path
import uuid
from typing import List
import asyncio
import subprocess
import shutil
import uuid
import wave
router = APIRouter(prefix="/convert", tags=["convert"])
@router.get("/list-audio", response_model=List[str])
async def list_audio_files():
base_dir = Path(__file__).resolve().parent.parent.parent / "data" / "audio"
if not base_dir.exists():
return []
files = [f.name for f in base_dir.iterdir() if f.is_file()]
return files
class DownloadAudioRequest(BaseModel):
filename: str
@router.post("/download-audio")
async def download_audio_file(data: DownloadAudioRequest):
base_dir = Path(__file__).resolve().parent.parent.parent / "data" / "audio"
src = (base_dir / data.filename).resolve()
try:
src.relative_to(base_dir.parent)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists():
raise HTTPException(status_code=404, detail="File not found")
return FileResponse(str(src), filename=src.name, media_type="audio/mpeg")
class VideoToMp3Request(BaseModel):
filename: str
def _run_moviepy(src: Path, dest: Path):
from moviepy import VideoFileClip
clip = VideoFileClip(str(src))
try:
if clip.audio is None:
raise RuntimeError("Source file has no audio stream")
clip.audio.write_audiofile(str(dest), codec="libmp3lame", bitrate="192k")
finally:
clip.close()
@router.post("/video-to-mp3")
async def video_to_mp3(req: VideoToMp3Request):
try:
import moviepy # noqa: F401
except Exception:
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Rebuild image or install moviepy/imageio-ffmpeg")
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
src = (data_dir / req.filename).resolve()
try:
src.relative_to(data_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
out_dir = data_dir / "audio"
out_dir.mkdir(parents=True, exist_ok=True)
dest = out_dir / f"{src.stem}.mp3"
try:
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy, src, dest)
except Exception as e:
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
return FileResponse(str(dest), filename=dest.name, media_type="audio/mpeg")
class VideoToWavRequest(BaseModel):
filename: str
def _run_moviepy_to_wav(src: Path, dest: Path):
from moviepy import VideoFileClip
clip = VideoFileClip(str(src))
try:
if clip.audio is None:
raise RuntimeError("Source file has no audio stream")
clip.audio.write_audiofile(str(dest), fps=16000, nbytes=2)
finally:
clip.close()
@router.post("/video-to-wav")
async def video_to_wav(req: VideoToWavRequest):
try:
import moviepy # noqa: F401
except Exception:
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Rebuild image or install moviepy/imageio-ffmpeg")
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
src = (data_dir / req.filename).resolve()
try:
src.relative_to(data_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
out_dir = data_dir / "audio"
out_dir.mkdir(parents=True, exist_ok=True)
dest = out_dir / f"{src.stem}.wav"
try:
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy_to_wav, src, dest)
except Exception as e:
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
return FileResponse(str(dest), filename=dest.name, media_type="audio/wav")
# MoviePy video-to-audio endpoints moved to api/route/moviepy.py
class Mp3ToTextRequest(BaseModel):
filename: str
model_path: str
def _run_ffmpeg(src: Path, dest: Path):
cmd = [
"ffmpeg",
"-y",
"-i",
str(src),
"-ac",
"1",
"-ar",
"16000",
"-vn",
"-f",
"wav",
str(dest),
]
subprocess.run(cmd, check=True)
@router.post("/mp3-to-text")
async def mp3_to_text(req: Mp3ToTextRequest):
logger = __import__("logging").getLogger("convert.mp3-to-text")
try:
if shutil.which("ffmpeg") is None:
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg in the environment/container.")
except Exception:
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg in the environment/container.")
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
audio_dir = data_dir / "audio"
src = (audio_dir / req.filename).resolve()
try:
src.relative_to(audio_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
audio_dir.mkdir(parents=True, exist_ok=True)
dest = audio_dir / f"{src.stem}_{uuid.uuid4().hex}.wav"
try:
logger.info("Starting ffmpeg conversion %s -> %s", src.name, dest.name)
await asyncio.get_running_loop().run_in_executor(None, _run_ffmpeg, src, dest)
logger.info("ffmpeg conversion finished")
except subprocess.CalledProcessError as e:
raise HTTPException(status_code=500, detail=f"ffmpeg conversion failed: {e}")
except Exception as e:
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
try:
from api.lib.audio_processing import VoskModelManager
except Exception as e:
try:
dest.unlink()
except Exception:
pass
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
try:
logger.info("Starting transcription stage for %s", dest.name)
project_root = Path(__file__).resolve().parent.parent.parent
models_dir = project_root / "models"
model_path = Path(req.model_path)
if not model_path.is_absolute():
model_path = project_root / req.model_path
model_path = model_path.resolve()
try:
model_path.relative_to(models_dir)
except Exception:
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
if not model_path.exists():
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
mgr = VoskModelManager(model_path)
res = mgr.transcribe_wav_with_words(dest)
text = res.get("text", "")
# Save plain transcript (no speaker split, no chapters)
text_dir = project_root / "data" / "text"
text_dir.mkdir(parents=True, exist_ok=True)
base_name = dest.stem
json_path = text_dir / f"{base_name}.json"
txt_path = text_dir / f"{base_name}.txt"
with open(json_path, "w", encoding="utf-8") as fh:
import json as _json
_json.dump({"text": text}, fh, ensure_ascii=False, indent=2)
with open(txt_path, "w", encoding="utf-8") as fh:
fh.write(text)
logger.info("Transcription stage finished for %s", dest.name)
finally:
try:
dest.unlink()
except Exception:
pass
return {"text": text}
class WavToTextRequest(BaseModel):
filename: str
model_path: str
use_embedding: bool = False
@router.post("/wav-to-text")
async def wav_to_text(req: WavToTextRequest):
logger = __import__("logging").getLogger("convert.wav-to-text")
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
audio_dir = data_dir / "audio"
src = (audio_dir / req.filename).resolve()
try:
src.relative_to(audio_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
try:
from api.lib.audio_processing import VoskModelManager
except Exception as e:
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
project_root = Path(__file__).resolve().parent.parent.parent
models_dir = project_root / "models"
model_path = Path(req.model_path)
if not model_path.is_absolute():
model_path = project_root / req.model_path
model_path = model_path.resolve()
try:
model_path.relative_to(models_dir)
except Exception:
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
if not model_path.exists():
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
temp_dest = None
to_cleanup = False
try:
if shutil.which("ffmpeg"):
temp_dest = audio_dir / f"{src.stem}_mono_{uuid.uuid4().hex}.wav"
cmd = [
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(src),
"-ac",
"1",
"-ar",
"16000",
"-sample_fmt",
"s16",
str(temp_dest),
]
try:
logger.info("Normalizing WAV via ffmpeg: %s -> %s", src.name, temp_dest.name)
subprocess.run(cmd, check=True)
src_for_transcription = temp_dest
to_cleanup = True
except subprocess.CalledProcessError as e:
raise HTTPException(status_code=500, detail=f"ffmpeg normalization failed: {e}")
else:
try:
with wave.open(str(src), "rb") as wf:
channels = wf.getnchannels()
rate = wf.getframerate()
sampwidth = wf.getsampwidth()
if channels != 1 or rate != 16000 or sampwidth != 2:
raise HTTPException(status_code=415, detail=("WAV must be mono PCM16 16kHz. "
"Install ffmpeg in the runtime or convert the file before calling this endpoint."))
src_for_transcription = src
except wave.Error as e:
raise HTTPException(status_code=400, detail=f"Invalid WAV file: {e}")
mgr = VoskModelManager(model_path)
res = mgr.transcribe_wav_with_words(src_for_transcription)
text = res.get("text", "")
words = res.get("words", [])
# Save plain transcript (no speaker split, no chapters)
text_dir = project_root / "data" / "text"
text_dir.mkdir(parents=True, exist_ok=True)
base_name = src.stem
json_path = text_dir / f"{base_name}.json"
txt_path = text_dir / f"{base_name}.txt"
import json as _json
with open(json_path, "w", encoding="utf-8") as fh:
_json.dump({"text": text}, fh, ensure_ascii=False, indent=2)
with open(txt_path, "w", encoding="utf-8") as fh:
fh.write(text)
logger.info("WAV transcription finished: %s", src.name)
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
finally:
if to_cleanup and temp_dest is not None:
try:
temp_dest.unlink()
except Exception:
pass
return {"text": text, "json": str(json_path.name), "txt": str(txt_path.name)}

View File

@@ -1,97 +0,0 @@
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from fastapi.responses import FileResponse
from pathlib import Path
import asyncio
router = APIRouter(prefix="/moviepy", tags=["moviepy"])
class VideoToMp3Request(BaseModel):
filename: str
class VideoToWavRequest(BaseModel):
filename: str
def _run_moviepy(src: Path, dest: Path):
from moviepy import VideoFileClip
clip = VideoFileClip(str(src))
try:
if clip.audio is None:
raise RuntimeError("Source file has no audio stream")
clip.audio.write_audiofile(str(dest), codec="libmp3lame", bitrate="192k")
finally:
clip.close()
def _run_moviepy_to_wav(src: Path, dest: Path):
from moviepy import VideoFileClip
clip = VideoFileClip(str(src))
try:
if clip.audio is None:
raise RuntimeError("Source file has no audio stream")
clip.audio.write_audiofile(str(dest), fps=16000, nbytes=2)
finally:
clip.close()
@router.post("/video-to-mp3")
async def video_to_mp3(req: VideoToMp3Request):
try:
import moviepy # noqa: F401
except Exception:
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Install moviepy/imageio-ffmpeg")
project_root = Path(__file__).resolve().parent.parent.parent
data_dir = project_root / "data"
src = (data_dir / req.filename).resolve()
try:
src.relative_to(data_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
out_dir = data_dir / "audio"
out_dir.mkdir(parents=True, exist_ok=True)
dest = out_dir / f"{src.stem}.mp3"
try:
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy, src, dest)
except Exception as e:
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
return FileResponse(str(dest), filename=dest.name, media_type="audio/mpeg")
@router.post("/video-to-wav")
async def video_to_wav(req: VideoToWavRequest):
try:
import moviepy # noqa: F401
except Exception:
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Install moviepy/imageio-ffmpeg")
project_root = Path(__file__).resolve().parent.parent.parent
data_dir = project_root / "data"
src = (data_dir / req.filename).resolve()
try:
src.relative_to(data_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
out_dir = data_dir / "audio"
out_dir.mkdir(parents=True, exist_ok=True)
dest = out_dir / f"{src.stem}.wav"
try:
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy_to_wav, src, dest)
except Exception as e:
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
return FileResponse(str(dest), filename=dest.name, media_type="audio/wav")

View File

@@ -1,138 +0,0 @@
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from pathlib import Path
import subprocess
import shutil
import json
import logging
try:
from vosk import Model, KaldiRecognizer
except Exception:
Model = None
KaldiRecognizer = None
router = APIRouter(prefix="/convert", tags=["convert"])
class Mp3ToTextRequest(BaseModel):
filename: str
model_path: str
@router.post("/mp3-to-text-ffmpeg")
async def mp3_to_text_ffmpeg(req: Mp3ToTextRequest):
"""Convert MP3 -> PCM via ffmpeg (stdout) and transcribe with Vosk.
This endpoint requires `ffmpeg` available in PATH. It decodes the input
MP3 to raw PCM s16le 16k mono and streams it into Vosk recognizer.
"""
logger = logging.getLogger("convert.mp3")
if Model is None or KaldiRecognizer is None:
raise HTTPException(status_code=503, detail="vosk is not installed in runtime")
project_root = Path(__file__).resolve().parent.parent.parent
audio_dir = project_root / "data" / "audio"
models_dir = project_root / "models"
src = (audio_dir / req.filename).resolve()
try:
src.relative_to(audio_dir)
except Exception:
raise HTTPException(status_code=400, detail="Invalid filename")
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Source file not found")
model_path = Path(req.model_path)
if not model_path.is_absolute():
model_path = project_root / req.model_path
model_path = model_path.resolve()
try:
model_path.relative_to(models_dir)
except Exception:
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
if not model_path.exists():
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
if shutil.which("ffmpeg") is None:
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg to use this endpoint.")
try:
from api.lib.audio_processing import FFmpegConverter, VoskModelManager
except Exception as e:
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
if shutil.which("ffmpeg") is None:
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg to use this endpoint.")
# Stream decode via ffmpeg to temporary process and feed recognizer
try:
mgr = VoskModelManager(model_path)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to load Vosk model: {e}")
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(src),
"-f",
"s16le",
"-acodec",
"pcm_s16le",
"-ac",
"1",
"-ar",
"16000",
"-",
]
logger.info("Starting ffmpeg streaming decode")
try:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to start ffmpeg: {e}")
if proc.stdout is None:
proc.kill()
raise HTTPException(status_code=500, detail="ffmpeg did not provide stdout")
recognizer = None
try:
recognizer = KaldiRecognizer(mgr.model, 16000)
try:
recognizer.SetWords(True)
except Exception:
pass
bytes_read = 0
total = None
try:
total = src.stat().st_size
except Exception:
total = None
last_pct = 0
while True:
chunk = proc.stdout.read(4000)
if not chunk:
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
if total:
pct = int(bytes_read * 100 / total)
if pct - last_pct >= 5:
last_pct = pct
logger.info("Decoding+transcription progress: %s%%", pct)
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")
logger.info("Transcription finished; result length=%d", len(text))
return {"text": text}
finally:
try:
proc.kill()
except Exception:
pass

201
api/route/pipeline.py Normal file
View File

@@ -0,0 +1,201 @@
"""End-to-end pipeline: download → subtitles-or-transcribe → summarize → persist."""
from __future__ import annotations
import asyncio
import logging
import uuid as _uuid
from pathlib import Path
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.config import settings
from api.db.video import create_video
from api.lib.llm import LLMConfig, from_settings as llm_from_settings
from api.lib.summarize import summarize
from api.lib.transcribe import transcribe_via_ffmpeg
from api.lib.youtube import download_video_with_subs, vtt_to_text
router = APIRouter(prefix="/pipeline", tags=["pipeline"])
logger = logging.getLogger("pipeline")
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
_VIDEO_DIR = _PROJECT_ROOT / "data" / "videos"
_TEXT_DIR = _PROJECT_ROOT / "data" / "text"
_MODELS_DIR = _PROJECT_ROOT / "models"
class LLMOverride(BaseModel):
base_url: str
api_key: str = ""
model: str
timeout: int = 120
max_tokens: int = 1000
temperature: float = 0.3
extra_headers: dict[str, str] = {}
system_prompt: str | None = None
user_template: str | None = None
class ProcessRequest(BaseModel):
url: str
model_path: str | None = None # path inside models/, used when subtitles are unavailable
summary_max_sentences: int = 15
# If supplied, this LLM is used for the summary; otherwise env-configured LLM;
# otherwise the built-in extractive fallback.
llm: LLMOverride | None = None
class ProcessResponse(BaseModel):
uuid: str
title: str
source_url: str
video_path: str
text_full_path: str
text_summary_path: str
transcription_method: str
summary_preview: str
def _resolve_model_path(req_model_path: str | None) -> Path:
raw = req_model_path or settings.DEFAULT_VOSK_MODEL
p = Path(raw)
if not p.is_absolute():
p = _PROJECT_ROOT / raw
p = p.resolve()
try:
p.relative_to(_MODELS_DIR.resolve())
except ValueError:
raise HTTPException(
status_code=400,
detail="model_path must point inside the project's models directory",
)
if not p.exists():
raise HTTPException(status_code=400, detail=f"Model not found: {p}")
return p
@router.post("/process", response_model=ProcessResponse)
async def process_video(req: ProcessRequest):
video_uuid = _uuid.uuid4().hex
_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
_TEXT_DIR.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
logger.info("[%s] downloading %s", video_uuid, req.url)
try:
info = await loop.run_in_executor(
None,
download_video_with_subs,
req.url,
video_uuid,
_VIDEO_DIR,
settings.subtitle_langs_list,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Download failed: {e}")
title: str = info["title"]
video_path: Path = info["video_path"]
sub_path: Path | None = info["subtitle_path"]
if not video_path.exists():
raise HTTPException(status_code=500, detail="Video file missing after download")
full_text = ""
method = ""
if sub_path is not None and sub_path.exists():
try:
content = sub_path.read_text(encoding="utf-8", errors="ignore")
full_text = vtt_to_text(content)
if full_text.strip():
method = f"subtitles:{info.get('subtitle_kind') or 'auto'}:{info.get('subtitle_lang') or 'unknown'}"
logger.info("[%s] using subtitles (%s)", video_uuid, method)
except Exception as e:
logger.warning("[%s] subtitle parse failed: %s", video_uuid, e)
full_text = ""
if not full_text.strip():
logger.info("[%s] no usable subtitles, falling back to Vosk", video_uuid)
model_path = _resolve_model_path(req.model_path)
try:
full_text = await loop.run_in_executor(
None, transcribe_via_ffmpeg, video_path, model_path
)
method = f"vosk:{model_path.name}"
except Exception as e:
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
if not full_text.strip():
raise HTTPException(
status_code=500, detail="No subtitles and transcription returned empty text"
)
text_full_path = _TEXT_DIR / f"{video_uuid}.txt"
text_summary_path = _TEXT_DIR / f"{video_uuid}_summary.txt"
text_full_path.write_text(full_text, encoding="utf-8")
if req.llm is not None:
llm_cfg = LLMConfig(
base_url=req.llm.base_url,
api_key=req.llm.api_key,
model=req.llm.model,
timeout=req.llm.timeout,
max_tokens=req.llm.max_tokens,
temperature=req.llm.temperature,
extra_headers=req.llm.extra_headers,
)
sys_p = req.llm.system_prompt
user_t = req.llm.user_template
else:
llm_cfg = llm_from_settings()
sys_p = None
user_t = None
try:
summary = await loop.run_in_executor(
None,
lambda: summarize(
full_text,
llm_cfg=llm_cfg,
system_prompt=sys_p,
user_template=user_t,
max_sentences=req.summary_max_sentences,
),
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Summarization failed: {e}")
text_summary_path.write_text(summary, encoding="utf-8")
rel_video = str(video_path.relative_to(_PROJECT_ROOT))
rel_full = str(text_full_path.relative_to(_PROJECT_ROOT))
rel_summary = str(text_summary_path.relative_to(_PROJECT_ROOT))
try:
await create_video(
uuid=video_uuid,
source_url=req.url,
title=title,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"DB insert failed: {e}")
logger.info("[%s] done (method=%s)", video_uuid, method)
return ProcessResponse(
uuid=video_uuid,
title=title,
source_url=req.url,
video_path=rel_video,
text_full_path=rel_full,
text_summary_path=rel_summary,
transcription_method=method,
summary_preview=summary[:300],
)

View File

@@ -1,94 +0,0 @@
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import Optional, List
from api.db.subscription import (
create_subscription,
get_subscription_by_user,
get_subscription_by_uuid,
update_subscription,
delete_subscription,
list_subscriptions,
extend_subscription_by_user,
)
import uuid
from datetime import datetime, timedelta
router = APIRouter(prefix="/subscription", tags=["subscription"])
class SubscriptionInfo(BaseModel):
user_id: str
uuid: str
until: datetime
active: bool
subscription_link: Optional[str] = None
connect_link: Optional[str] = None
class SubscriptionCreate(BaseModel):
user_id: str
months: int
subscription_link: Optional[str] = None
connect_link: Optional[str] = None
class SubscriptionUpdate(BaseModel):
until: Optional[datetime]
user_id: Optional[str]
active: Optional[bool] = None
subscription_link: Optional[str] = None
connect_link: Optional[str] = None
@router.get("/{user_id}", response_model=Optional[SubscriptionInfo])
async def get_user_subscription(user_id: str):
sub = await get_subscription_by_user(user_id)
if not sub:
return None
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until)
@router.post("/new", response_model=SubscriptionInfo)
async def create_subscription_endpoint(data: SubscriptionCreate):
sub_id = str(uuid.uuid4())
until = datetime.now() + timedelta(days=30 * data.months)
sub = await create_subscription(data.user_id, sub_id, until, subscription_link=data.subscription_link, connect_link=data.connect_link, active=True)
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until)
@router.post("/extend", response_model=SubscriptionInfo)
async def extend_user_subscription(data: SubscriptionCreate):
sub = await extend_subscription_by_user(data.user_id, data.months)
if not sub:
raise HTTPException(status_code=404, detail="Subscription not found")
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
@router.get("/by-uuid/{sub_uuid}", response_model=Optional[SubscriptionInfo])
async def get_subscription_uuid(sub_uuid: str):
sub = await get_subscription_by_uuid(sub_uuid)
if not sub:
raise HTTPException(status_code=404, detail="Subscription not found")
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
@router.patch("/{sub_uuid}", response_model=SubscriptionInfo)
async def update_subscription_endpoint(sub_uuid: str, data: SubscriptionUpdate):
sub = await update_subscription(sub_uuid, until=data.until, user_id=data.user_id, active=data.active, subscription_link=data.subscription_link, connect_link=data.connect_link)
if not sub:
raise HTTPException(status_code=404, detail="Subscription not found")
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
@router.delete("/{sub_uuid}")
async def delete_subscription_endpoint(sub_uuid: str):
ok = await delete_subscription(sub_uuid)
if not ok:
raise HTTPException(status_code=404, detail="Subscription not found")
return {"ok": True}
@router.get("/", response_model=List[SubscriptionInfo])
async def list_all_subscriptions():
subs = await list_subscriptions()
return [SubscriptionInfo(user_id=s.user_id, uuid=s.uuid, until=s.until) for s in subs]

View File

@@ -1,107 +0,0 @@
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import Optional, List
from api.db.user import create_user, get_user, get_user_by_telegram, update_user, delete_user, list_users
from api.db.subscription import (
create_subscription,
get_subscription_by_user,
update_subscription,
delete_subscription,
)
from api.route.subscription import SubscriptionInfo
router = APIRouter(prefix="/users", tags=["users"])
class UserOut(BaseModel):
id: str
telegram_id: str
name: Optional[str]
subscription: Optional[SubscriptionInfo] = None
class UserCreate(BaseModel):
telegram_id: str
name: Optional[str]
class UserUpdate(BaseModel):
name: Optional[str]
@router.post("/", response_model=UserOut)
async def api_create_user(data: UserCreate):
existing = await get_user_by_telegram(data.telegram_id)
if existing:
raise HTTPException(status_code=400, detail="User already exists")
u = await create_user(data.telegram_id, data.name)
return UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
@router.get("/{user_id}", response_model=UserOut)
async def api_get_user(user_id: str, expand: Optional[str] = None):
u = await get_user(user_id)
if not u:
raise HTTPException(status_code=404, detail="User not found")
out = UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
if expand == "subscription":
sub = await get_subscription_by_user(u.id)
if sub:
out.subscription = SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
return out
@router.post("/{user_id}/subscription", response_model=SubscriptionInfo)
async def api_create_user_subscription(user_id: str, data: dict):
# expected keys: months, subscription_link (opt), connect_link (opt)
months = int(data.get("months", 1))
subscription_link = data.get("subscription_link")
connect_link = data.get("connect_link")
import uuid as _uuid
sub_id = str(_uuid.uuid4())
from datetime import datetime, timedelta
until = datetime.now() + timedelta(days=30 * months)
sub = await create_subscription(user_id, sub_id, until, subscription_link=subscription_link, connect_link=connect_link, active=True)
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
@router.patch("/{user_id}/subscription", response_model=SubscriptionInfo)
async def api_update_user_subscription(user_id: str, data: dict):
sub = await get_subscription_by_user(user_id)
if not sub:
raise HTTPException(status_code=404, detail="Subscription not found for user")
updated = await update_subscription(sub.uuid, until=data.get("until"), user_id=data.get("user_id"), active=data.get("active"), subscription_link=data.get("subscription_link"), connect_link=data.get("connect_link"))
return SubscriptionInfo(user_id=updated.user_id, uuid=updated.uuid, until=updated.until, active=updated.active, subscription_link=updated.subscription_link, connect_link=updated.connect_link)
@router.delete("/{user_id}/subscription")
async def api_delete_user_subscription(user_id: str):
sub = await get_subscription_by_user(user_id)
if not sub:
raise HTTPException(status_code=404, detail="Subscription not found for user")
ok = await delete_subscription(sub.uuid)
if not ok:
raise HTTPException(status_code=500, detail="Failed to delete subscription")
return {"ok": True}
@router.get("/", response_model=List[UserOut])
async def api_list_users():
users = await list_users()
return [UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name) for u in users]
@router.patch("/{user_id}", response_model=UserOut)
async def api_update_user(user_id: str, data: UserUpdate):
u = await update_user(user_id, name=data.name)
if not u:
raise HTTPException(status_code=404, detail="User not found")
return UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
@router.delete("/{user_id}")
async def api_delete_user(user_id: str):
ok = await delete_user(user_id)
if not ok:
raise HTTPException(status_code=404, detail="User not found")
return {"ok": True}

55
api/route/videos.py Normal file
View File

@@ -0,0 +1,55 @@
from datetime import datetime
from typing import List
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from api.db.video import delete_video, get_video, list_videos
router = APIRouter(prefix="/videos", tags=["videos"])
class VideoOut(BaseModel):
uuid: str
source_url: str
title: str
video_path: str
text_full_path: str
text_summary_path: str
transcription_method: str
created_at: datetime
def _to_out(v) -> VideoOut:
return VideoOut(
uuid=v.uuid,
source_url=v.source_url,
title=v.title,
video_path=v.video_path,
text_full_path=v.text_full_path,
text_summary_path=v.text_summary_path,
transcription_method=v.transcription_method,
created_at=v.created_at,
)
@router.get("/", response_model=List[VideoOut])
async def list_all():
return [_to_out(v) for v in await list_videos()]
@router.get("/{video_uuid}", response_model=VideoOut)
async def get_one(video_uuid: str):
v = await get_video(video_uuid)
if not v:
raise HTTPException(status_code=404, detail="Video not found")
return _to_out(v)
@router.delete("/{video_uuid}")
async def delete_one(video_uuid: str):
ok = await delete_video(video_uuid)
if not ok:
raise HTTPException(status_code=404, detail="Video not found")
return {"ok": True}