Рабочий пайплайн: YouTube -> транскрипция -> выжимка -> Postgres

- новый api/: /pipeline/process (yt-dlp, субтитры или Vosk, LLM/экстрактивная
  суммаризация), /videos, /llm; старый код перенесён в api_legacy/
- web/: Flet UI (flet 0.28.3 + flet-web, порт 8550)
- Dockerfile.api: ffmpeg слоем из mwader/static-ffmpeg, pip с кэш-маунтом
- requirements/pyproject: убраны moviepy, imageio-ffmpeg, битый asyncio
- README с инструкцией запуска и загрузкой Vosk-модели

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-07-14 11:12:46 +05:00
parent 564c9c2d2f
commit 2697e01714
51 changed files with 2313 additions and 284 deletions

119
api/lib/transcribe.py Normal file
View File

@@ -0,0 +1,119 @@
"""Vosk-based fallback transcription: stream PCM from ffmpeg into a Vosk recognizer."""
from __future__ import annotations
import json
import logging
import os
import shutil
import subprocess
import sys
import threading
from pathlib import Path
try:
from vosk import KaldiRecognizer, Model
except Exception:
Model = None
KaldiRecognizer = None
logger = logging.getLogger("transcribe")
class VoskModelManager:
"""Loads a Vosk model. Captures the model's stderr to Python logs while loading."""
def __init__(self, model_path: Path):
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
self.model = self._load_with_stderr_capture(model_path)
@staticmethod
def _load_with_stderr_capture(path: Path):
log = logging.getLogger("vosk.loader")
r_fd, w_fd = os.pipe()
saved_stderr = os.dup(sys.stderr.fileno())
os.dup2(w_fd, sys.stderr.fileno())
os.close(w_fd)
def reader(fd: int) -> None:
with os.fdopen(fd, "rb") as fh:
for raw in iter(fh.readline, b""):
log.info(raw.decode(errors="ignore").rstrip())
t = threading.Thread(target=reader, args=(r_fd,), daemon=True)
t.start()
try:
return Model(str(path))
finally:
try:
os.dup2(saved_stderr, sys.stderr.fileno())
finally:
os.close(saved_stderr)
t.join(timeout=2)
def transcribe_via_ffmpeg(media_path: Path, model_path: Path, sample_rate: int = 16000) -> str:
"""Decode the source media to PCM s16le mono via ffmpeg and feed it to Vosk in chunks."""
if Model is None or KaldiRecognizer is None:
raise RuntimeError("vosk is not installed in runtime")
if shutil.which("ffmpeg") is None:
raise RuntimeError("ffmpeg is not available in PATH")
if not media_path.exists():
raise FileNotFoundError(media_path)
mgr = VoskModelManager(model_path)
recognizer = KaldiRecognizer(mgr.model, sample_rate)
try:
recognizer.SetWords(True)
except Exception:
pass
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(media_path),
"-f",
"s16le",
"-acodec",
"pcm_s16le",
"-ac",
"1",
"-ar",
str(sample_rate),
"-vn",
"-",
]
logger.info("Spawning ffmpeg streaming decode for %s", media_path.name)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if proc.stdout is None:
proc.kill()
raise RuntimeError("ffmpeg did not provide stdout")
try:
bytes_read = 0
last_logged_mb = 0
while True:
chunk = proc.stdout.read(4000)
if not chunk:
break
recognizer.AcceptWaveform(chunk)
bytes_read += len(chunk)
mb = bytes_read // (1024 * 1024)
if mb >= last_logged_mb + 5:
last_logged_mb = mb
logger.info("Transcribed %d MB of audio so far", mb)
result = json.loads(recognizer.FinalResult())
text = result.get("text", "")
logger.info("Transcription finished, length=%d", len(text))
return text
finally:
try:
proc.kill()
except Exception:
pass