Рабочий пайплайн: YouTube -> транскрипция -> выжимка -> Postgres
- новый api/: /pipeline/process (yt-dlp, субтитры или Vosk, LLM/экстрактивная суммаризация), /videos, /llm; старый код перенесён в api_legacy/ - web/: Flet UI (flet 0.28.3 + flet-web, порт 8550) - Dockerfile.api: ffmpeg слоем из mwader/static-ffmpeg, pip с кэш-маунтом - requirements/pyproject: убраны moviepy, imageio-ffmpeg, битый asyncio - README с инструкцией запуска и загрузкой Vosk-модели Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
2
api_legacy/README.md
Normal file
2
api_legacy/README.md
Normal file
@@ -0,0 +1,2 @@
|
||||
# Для запуска API отдельно:
|
||||
# uvicorn api.main:app --reload
|
||||
28
api_legacy/config.py
Normal file
28
api_legacy/config.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from pydantic_settings import BaseSettings
|
||||
import os
|
||||
|
||||
def _env(*names, default=None):
|
||||
for n in names:
|
||||
v = os.getenv(n)
|
||||
if v:
|
||||
return v
|
||||
return default
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# support both custom DB_* names and common POSTGRES_* names from postgres images
|
||||
DB_USER: str = _env("DB_USER", "POSTGRES_USER", default="postgres")
|
||||
DB_PASS: str = _env("DB_PASS", "POSTGRES_PASSWORD", default="postgres")
|
||||
DB_NAME: str = _env("DB_NAME", "POSTGRES_DB", default="test_db")
|
||||
DB_HOST: str = _env("DB_HOST", "POSTGRES_HOST", "DB_HOST", default="db")
|
||||
DB_PORT: str = _env("DB_PORT", "POSTGRES_PORT", default="5432")
|
||||
|
||||
# If explicit DB_* variables are provided, build DATABASE_URL from them (priority).
|
||||
# Otherwise fall back to explicit DATABASE_URL or DATABASE_URL_ASYNC env.
|
||||
_built_db_url: str | None = None
|
||||
if DB_USER and DB_PASS and DB_NAME and DB_HOST and DB_PORT:
|
||||
_built_db_url = f"postgresql+asyncpg://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
|
||||
|
||||
DATABASE_URL: str = _built_db_url or _env("DATABASE_URL", "DATABASE_URL_ASYNC", default="")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
27
api_legacy/db/connection.py
Normal file
27
api_legacy/db/connection.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import text
|
||||
import asyncio
|
||||
from api.config import settings
|
||||
|
||||
DATABASE_URL = settings.DATABASE_URL
|
||||
engine = create_async_engine(DATABASE_URL, echo=False, future=True)
|
||||
SessionLocal = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
||||
|
||||
|
||||
async def wait_for_db(retries: int = 30, delay: float = 1.0) -> None:
|
||||
"""Wait until the database is available. Retries with exponential backoff.
|
||||
|
||||
Raises RuntimeError if DB is still unavailable after retries.
|
||||
"""
|
||||
last_exc = None
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
return
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
wait = min(delay * (2 ** (attempt - 1)), 5)
|
||||
await asyncio.sleep(wait)
|
||||
raise RuntimeError(f"Could not connect to DB after {retries} attempts") from last_exc
|
||||
121
api_legacy/db/subscription.py
Normal file
121
api_legacy/db/subscription.py
Normal file
@@ -0,0 +1,121 @@
|
||||
from api.db.connection import engine, SessionLocal
|
||||
from api.models.subscription import Subscription
|
||||
from api.models.base import Base
|
||||
from api.config import settings
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy import select
|
||||
import asyncpg
|
||||
|
||||
|
||||
async def init_db():
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
return
|
||||
except Exception as exc: # try to create database if it doesn't exist
|
||||
msg = str(exc).lower()
|
||||
if "does not exist" in msg or "invalidcatalogname" in msg or "database" in msg:
|
||||
try:
|
||||
admin_conn = await asyncpg.connect(
|
||||
user=settings.DB_USER,
|
||||
password=settings.DB_PASS,
|
||||
database="postgres",
|
||||
host=settings.DB_HOST,
|
||||
port=int(settings.DB_PORT),
|
||||
)
|
||||
try:
|
||||
await admin_conn.execute(f'CREATE DATABASE "{settings.DB_NAME}"')
|
||||
except Exception:
|
||||
pass
|
||||
await admin_conn.close()
|
||||
except Exception:
|
||||
raise
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
async def create_subscription(user_id: str, uuid: str, until: datetime, subscription_link: str | None = None, connect_link: str | None = None, active: bool = True):
|
||||
async with SessionLocal() as session:
|
||||
# if user already has a subscription, remove it (replace)
|
||||
q = select(Subscription).where(Subscription.user_id == user_id)
|
||||
res = await session.execute(q)
|
||||
existing = res.scalars().first()
|
||||
if existing:
|
||||
await session.delete(existing)
|
||||
# avoid NULL in DB: replace None links with empty strings
|
||||
safe_subscription_link = subscription_link if subscription_link is not None else ""
|
||||
safe_connect_link = connect_link if connect_link is not None else ""
|
||||
sub = Subscription(user_id=user_id, uuid=uuid, until=until, active=active, subscription_link=safe_subscription_link, connect_link=safe_connect_link)
|
||||
session.add(sub)
|
||||
await session.commit()
|
||||
await session.refresh(sub)
|
||||
return sub
|
||||
|
||||
|
||||
async def get_subscription_by_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.user_id == user_id)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
return sub
|
||||
|
||||
|
||||
async def get_subscription_by_uuid(sub_uuid: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def update_subscription(sub_uuid: str, until: datetime = None, user_id: str | None = None, active: bool | None = None, subscription_link: str | None = None, connect_link: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
if not sub:
|
||||
return None
|
||||
if until is not None:
|
||||
sub.until = until
|
||||
if user_id is not None:
|
||||
sub.user_id = user_id
|
||||
if active is not None:
|
||||
sub.active = active
|
||||
if subscription_link is not None:
|
||||
sub.subscription_link = subscription_link
|
||||
if connect_link is not None:
|
||||
sub.connect_link = connect_link
|
||||
await session.commit()
|
||||
return sub
|
||||
|
||||
|
||||
async def delete_subscription(sub_uuid: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
if not sub:
|
||||
return False
|
||||
await session.delete(sub)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_subscriptions():
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().all()
|
||||
|
||||
|
||||
async def extend_subscription_by_user(user_id: str, months: int):
|
||||
sub = await get_subscription_by_user(user_id)
|
||||
if sub:
|
||||
new_until = max(sub.until, datetime.now()) + timedelta(days=30 * months)
|
||||
return await update_subscription(sub.uuid, until=new_until, active=True)
|
||||
return None
|
||||
|
||||
|
||||
async def deactivate_subscription(sub_uuid: str):
|
||||
return await update_subscription(sub_uuid, active=False)
|
||||
64
api_legacy/db/user.py
Normal file
64
api_legacy/db/user.py
Normal file
@@ -0,0 +1,64 @@
|
||||
from api.db.connection import SessionLocal
|
||||
from api.models.user import User
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
import uuid
|
||||
|
||||
|
||||
async def create_user(telegram_id: str, name: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
# ensure no NULL in DB: replace None with empty string
|
||||
safe_name = name if name is not None else ""
|
||||
user = User(id=str(uuid.uuid4()), telegram_id=telegram_id, name=safe_name)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def get_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def get_user_by_telegram(telegram_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.telegram_id == telegram_id)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def update_user(user_id: str, name: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
user = res.scalars().first()
|
||||
if not user:
|
||||
return None
|
||||
if name is not None:
|
||||
# replace None handled by caller; here update only when provided
|
||||
user.name = name
|
||||
await session.commit()
|
||||
return user
|
||||
|
||||
|
||||
async def delete_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
user = res.scalars().first()
|
||||
if not user:
|
||||
return False
|
||||
await session.delete(user)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_users():
|
||||
async with SessionLocal() as session:
|
||||
q = select(User)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().all()
|
||||
134
api_legacy/lib/audio_processing.py
Normal file
134
api_legacy/lib/audio_processing.py
Normal file
@@ -0,0 +1,134 @@
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import logging
|
||||
import wave
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
try:
|
||||
from vosk import Model, KaldiRecognizer
|
||||
except Exception:
|
||||
Model = None
|
||||
KaldiRecognizer = None
|
||||
|
||||
|
||||
class FFmpegConverter:
|
||||
"""Utility to convert audio files using ffmpeg."""
|
||||
|
||||
@staticmethod
|
||||
def convert_mp3_to_wav(src: Path, dest: Path) -> None:
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
str(src),
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-vn",
|
||||
"-f",
|
||||
"wav",
|
||||
str(dest),
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
class VoskModelManager:
|
||||
"""Loads a Vosk model and provides transcription methods.
|
||||
|
||||
Loading captures model stderr into Python logs to show progress.
|
||||
"""
|
||||
|
||||
def __init__(self, model_path: Path):
|
||||
self.logger = logging.getLogger("vosk.model_manager")
|
||||
if Model is None or KaldiRecognizer is None:
|
||||
raise RuntimeError("vosk is not installed in runtime")
|
||||
self.model = self._load_model_with_progress(model_path)
|
||||
|
||||
def _load_model_with_progress(self, path: Path):
|
||||
logger = logging.getLogger("vosk.loader")
|
||||
r_fd, w_fd = os.pipe()
|
||||
saved_stderr_fd = os.dup(sys.stderr.fileno())
|
||||
os.dup2(w_fd, sys.stderr.fileno())
|
||||
|
||||
def _reader(fd):
|
||||
with os.fdopen(fd, "rb") as fh:
|
||||
for raw in iter(fh.readline, b""):
|
||||
try:
|
||||
logger.info(raw.decode(errors="ignore").rstrip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
reader_thread = threading.Thread(target=_reader, args=(r_fd,), daemon=True)
|
||||
reader_thread.start()
|
||||
|
||||
try:
|
||||
model = Model(str(path))
|
||||
finally:
|
||||
try:
|
||||
os.dup2(saved_stderr_fd, sys.stderr.fileno())
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.close(saved_stderr_fd)
|
||||
except Exception:
|
||||
pass
|
||||
reader_thread.join(timeout=2)
|
||||
|
||||
return model
|
||||
|
||||
def transcribe_wav(self, wav_path: Path) -> str:
|
||||
if not wav_path.exists():
|
||||
raise FileNotFoundError(wav_path)
|
||||
try:
|
||||
with wave.open(str(wav_path), "rb") as wf:
|
||||
if wf.getnchannels() != 1:
|
||||
raise ValueError("Audio must be mono (1 channel)")
|
||||
recognizer = KaldiRecognizer(self.model, wf.getframerate())
|
||||
try:
|
||||
recognizer.SetWords(True)
|
||||
except Exception:
|
||||
# older vosk bindings may not expose SetWords; ignore if absent
|
||||
pass
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if not data:
|
||||
break
|
||||
recognizer.AcceptWaveform(data)
|
||||
result = json.loads(recognizer.FinalResult())
|
||||
return result.get("text", "")
|
||||
except wave.Error as e:
|
||||
raise ValueError(f"Invalid WAV file: {e}")
|
||||
|
||||
def transcribe_wav_with_words(self, wav_path: Path):
|
||||
"""Transcribe WAV and return dict with 'text' and 'words' (word-level timestamps).
|
||||
|
||||
Returns: { 'text': str, 'words': [{'word': str, 'start': float, 'end': float}, ...] }
|
||||
"""
|
||||
if not wav_path.exists():
|
||||
raise FileNotFoundError(wav_path)
|
||||
try:
|
||||
with wave.open(str(wav_path), "rb") as wf:
|
||||
if wf.getnchannels() != 1:
|
||||
raise ValueError("Audio must be mono (1 channel)")
|
||||
recognizer = KaldiRecognizer(self.model, wf.getframerate())
|
||||
try:
|
||||
recognizer.SetWords(True)
|
||||
except Exception:
|
||||
pass
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if not data:
|
||||
break
|
||||
recognizer.AcceptWaveform(data)
|
||||
res = json.loads(recognizer.FinalResult())
|
||||
words = res.get("result", [])
|
||||
# words are dictionaries with word, start, end
|
||||
text = res.get("text", "")
|
||||
return {"text": text, "words": words}
|
||||
except wave.Error as e:
|
||||
raise ValueError(f"Invalid WAV file: {e}")
|
||||
69
api_legacy/lib/diarize_embeddings.py
Normal file
69
api_legacy/lib/diarize_embeddings.py
Normal file
@@ -0,0 +1,69 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
|
||||
def diarize_with_resemblyzer(wav_path: Path, window_s: float = 1.5, hop_s: float = 0.75, distance_threshold: float = 0.6) -> List[Dict]:
|
||||
"""Lightweight embedding-based diarization using resemblyzer + sklearn.
|
||||
|
||||
Returns list of segments: {'start': float, 'end': float, 'speaker': 'spk_N'}
|
||||
"""
|
||||
try:
|
||||
import numpy as np
|
||||
import librosa
|
||||
from resemblyzer import VoiceEncoder
|
||||
from sklearn.cluster import AgglomerativeClustering
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Missing dependency for embedding diarization: {e}")
|
||||
|
||||
sr = 16000
|
||||
wav, sr_loaded = librosa.load(str(wav_path), sr=sr)
|
||||
n = wav.shape[0]
|
||||
win = int(window_s * sr)
|
||||
hop = int(hop_s * sr)
|
||||
if n <= win:
|
||||
# short file: embed whole
|
||||
encoder = VoiceEncoder()
|
||||
emb = encoder.embed_utterance(wav)
|
||||
labels = [0]
|
||||
centers = [ (0.0 + n / sr) / 2.0 ]
|
||||
windows = [(0.0, n / sr)]
|
||||
else:
|
||||
encoder = VoiceEncoder()
|
||||
embeddings = []
|
||||
centers = []
|
||||
windows = []
|
||||
for start in range(0, n - win + 1, hop):
|
||||
chunk = wav[start:start+win]
|
||||
try:
|
||||
e = encoder.embed_utterance(chunk)
|
||||
except Exception:
|
||||
# fallback: mean pooling
|
||||
e = np.mean(chunk)
|
||||
embeddings.append(e)
|
||||
t0 = start / sr
|
||||
t1 = (start + win) / sr
|
||||
centers.append((t0 + t1) / 2.0)
|
||||
windows.append((t0, t1))
|
||||
|
||||
X = np.vstack(embeddings)
|
||||
# Agglomerative clustering with distance threshold
|
||||
model = AgglomerativeClustering(n_clusters=None, distance_threshold=distance_threshold, affinity='euclidean', linkage='average')
|
||||
labels = model.fit_predict(X)
|
||||
|
||||
# merge consecutive windows with same label into segments
|
||||
segs = []
|
||||
if len(labels) == 0:
|
||||
return segs
|
||||
cur_label = labels[0]
|
||||
cur_start = windows[0][0]
|
||||
cur_end = windows[0][1]
|
||||
for i, lab in enumerate(labels[1:], start=1):
|
||||
if lab == cur_label:
|
||||
cur_end = windows[i][1]
|
||||
else:
|
||||
segs.append({"start": cur_start, "end": cur_end, "speaker": f"spk_{int(cur_label)+1}"})
|
||||
cur_label = lab
|
||||
cur_start = windows[i][0]
|
||||
cur_end = windows[i][1]
|
||||
segs.append({"start": cur_start, "end": cur_end, "speaker": f"spk_{int(cur_label)+1}"})
|
||||
|
||||
return segs
|
||||
236
api_legacy/lib/transcript_utils.py
Normal file
236
api_legacy/lib/transcript_utils.py
Normal file
@@ -0,0 +1,236 @@
|
||||
from pathlib import Path
|
||||
import wave
|
||||
import json
|
||||
import math
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
def _rms(frames: bytes, sample_width: int) -> float:
|
||||
# compute RMS of raw frames (little-endian signed samples)
|
||||
if sample_width == 2:
|
||||
import struct
|
||||
|
||||
count = len(frames) // 2
|
||||
if count == 0:
|
||||
return 0.0
|
||||
fmt = "<%dh" % count
|
||||
vals = struct.unpack(fmt, frames)
|
||||
s = 0
|
||||
for v in vals:
|
||||
s += v * v
|
||||
return math.sqrt(s / count)
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
|
||||
def diarize_by_silence(wav_path: Path, win_ms: int = 30, silence_thresh: float = 500.0, min_silence_ms: int = 400) -> List[Dict]:
|
||||
"""Naive diarization by splitting on long silence.
|
||||
|
||||
Returns list of segments: {'start': float, 'end': float, 'speaker': str}
|
||||
Speakers are assigned alternately: Speaker 1, Speaker 2, ...
|
||||
"""
|
||||
segs = []
|
||||
with wave.open(str(wav_path), "rb") as wf:
|
||||
sr = wf.getframerate()
|
||||
sw = wf.getsampwidth()
|
||||
n_channels = wf.getnchannels()
|
||||
# we expect mono
|
||||
frames_per_win = int(sr * win_ms / 1000)
|
||||
min_silence_frames = int(sr * min_silence_ms / 1000)
|
||||
total_frames = wf.getnframes()
|
||||
pos = 0
|
||||
silent_acc = 0
|
||||
current_start = 0
|
||||
speaker_idx = 1
|
||||
|
||||
while pos < total_frames:
|
||||
to_read = min(frames_per_win, total_frames - pos)
|
||||
raw = wf.readframes(to_read)
|
||||
pos += to_read
|
||||
level = _rms(raw, sw)
|
||||
if level < silence_thresh:
|
||||
silent_acc += to_read
|
||||
else:
|
||||
# if there was long silence before speech, start a new segment
|
||||
if silent_acc >= min_silence_frames and pos / sr > current_start:
|
||||
end_time = (pos - silent_acc) / sr
|
||||
segs.append({"start": current_start, "end": end_time, "speaker": f"Speaker {speaker_idx}"})
|
||||
speaker_idx = (speaker_idx % 2) + 1
|
||||
current_start = end_time
|
||||
silent_acc = 0
|
||||
|
||||
# finish last segment
|
||||
if current_start < total_frames / sr:
|
||||
segs.append({"start": current_start, "end": total_frames / sr, "speaker": f"Speaker {speaker_idx}"})
|
||||
|
||||
return segs
|
||||
|
||||
|
||||
def assign_words_to_speakers(words: List[Dict], segments: List[Dict]) -> List[Dict]:
|
||||
"""Assign word dicts (with start/end) to speaker segments. Returns list of speaker blocks with text."""
|
||||
result = []
|
||||
seg_idx = 0
|
||||
current_block = {"speaker": segments[0]["speaker"] if segments else "Speaker 1", "start": None, "end": None, "words": []}
|
||||
|
||||
for w in words:
|
||||
t = w.get("start", 0)
|
||||
# move seg_idx until this word falls into segment
|
||||
while seg_idx + 1 < len(segments) and t >= segments[seg_idx]["end"]:
|
||||
# flush current
|
||||
if current_block["words"]:
|
||||
current_block["end"] = segments[seg_idx]["end"]
|
||||
result.append(current_block)
|
||||
seg_idx += 1
|
||||
current_block = {"speaker": segments[seg_idx]["speaker"], "start": None, "end": None, "words": []}
|
||||
|
||||
# append word
|
||||
if current_block["start"] is None:
|
||||
current_block["start"] = w.get("start")
|
||||
current_block["words"].append(w)
|
||||
|
||||
if current_block["words"]:
|
||||
# set end
|
||||
current_block["end"] = current_block["words"][-1].get("end")
|
||||
result.append(current_block)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def chapter_by_wordcount(words: List[Dict], speaker_blocks: List[Dict] = None, words_per_chapter: int = 100, min_words_on_speaker_change: int = 30) -> List[Dict]:
|
||||
"""Chaptering that prefers boundaries at speaker changes.
|
||||
|
||||
- If `speaker_blocks` provided, will try to split a chapter when speaker changes
|
||||
and the current chapter has at least `min_words_on_speaker_change` words.
|
||||
- Also split when `words_per_chapter` is reached.
|
||||
Returns list of chapters: {'start': float, 'end': float, 'words': [...]}.
|
||||
"""
|
||||
chapters = []
|
||||
chapter = {"start": None, "end": None, "words": []}
|
||||
count = 0
|
||||
|
||||
def speaker_for_time(t: float):
|
||||
if not speaker_blocks or t is None:
|
||||
return None
|
||||
for b in speaker_blocks:
|
||||
bstart = b.get("start")
|
||||
bend = b.get("end")
|
||||
if bstart is None or bend is None:
|
||||
continue
|
||||
try:
|
||||
if bstart <= t <= bend:
|
||||
return b.get("speaker")
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
prev_speaker = None
|
||||
for w in words:
|
||||
wstart = w.get("start")
|
||||
if chapter["start"] is None:
|
||||
chapter["start"] = wstart
|
||||
chapter["words"].append(w)
|
||||
chapter["end"] = w.get("end")
|
||||
count += 1
|
||||
|
||||
# check speaker change boundary
|
||||
cur_speaker = speaker_for_time(wstart)
|
||||
if prev_speaker is None:
|
||||
prev_speaker = cur_speaker
|
||||
|
||||
if cur_speaker is not None and prev_speaker is not None and cur_speaker != prev_speaker:
|
||||
if count >= min_words_on_speaker_change:
|
||||
# close chapter at previous word
|
||||
chapters.append(chapter)
|
||||
chapter = {"start": None, "end": None, "words": []}
|
||||
count = 0
|
||||
prev_speaker = cur_speaker
|
||||
continue
|
||||
else:
|
||||
# don't split if too short to make a chapter
|
||||
prev_speaker = cur_speaker
|
||||
|
||||
# check fixed-size boundary
|
||||
if count >= words_per_chapter:
|
||||
chapters.append(chapter)
|
||||
chapter = {"start": None, "end": None, "words": []}
|
||||
count = 0
|
||||
prev_speaker = None
|
||||
|
||||
if chapter["words"]:
|
||||
chapters.append(chapter)
|
||||
return chapters
|
||||
|
||||
|
||||
def save_transcript(text_dir: Path, base_name: str, speaker_blocks: List[Dict], chapters: List[Dict]):
|
||||
text_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_json = text_dir / f"{base_name}.json"
|
||||
out_txt = text_dir / f"{base_name}.txt"
|
||||
# Automatic host labeling: speaker with most words -> 'Ведущий', others -> 'Участник N'
|
||||
try:
|
||||
# Prefer selecting host by total spoken duration (better for long monologues).
|
||||
durations = []
|
||||
for b in speaker_blocks:
|
||||
dur = 0.0
|
||||
for w in b.get("words", []):
|
||||
s = w.get("start")
|
||||
e = w.get("end")
|
||||
if s is None or e is None:
|
||||
continue
|
||||
try:
|
||||
dur += float(e) - float(s)
|
||||
except Exception:
|
||||
continue
|
||||
durations.append(dur)
|
||||
|
||||
if any(durations):
|
||||
host_idx = int(max(range(len(durations)), key=lambda i: durations[i]))
|
||||
else:
|
||||
# fallback to word counts if timestamps missing
|
||||
counts = [sum(1 for w in b.get("words", [])) for b in speaker_blocks]
|
||||
host_idx = int(max(range(len(counts)), key=lambda i: counts[i])) if counts else 0
|
||||
|
||||
new_blocks = []
|
||||
participant_idx = 1
|
||||
for i, b in enumerate(speaker_blocks):
|
||||
b_copy = dict(b)
|
||||
if i == host_idx:
|
||||
b_copy["speaker"] = "Ведущий"
|
||||
else:
|
||||
b_copy["speaker"] = f"Участник {participant_idx}"
|
||||
participant_idx += 1
|
||||
new_blocks.append(b_copy)
|
||||
speaker_blocks = new_blocks
|
||||
except Exception:
|
||||
# fall back to provided labels on error
|
||||
pass
|
||||
|
||||
data = {"speakers": speaker_blocks, "chapters": chapters}
|
||||
with open(out_json, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, indent=2)
|
||||
|
||||
# plain text with speaker labels and chapter headings
|
||||
with open(out_txt, "w", encoding="utf-8") as fh:
|
||||
for i, ch in enumerate(chapters, start=1):
|
||||
fh.write(f"=== Глава {i} ===\n")
|
||||
# collect speaker text overlapping this chapter by word membership
|
||||
# build a simple view: iterate speaker blocks and print their words that fall into chapter
|
||||
for b in speaker_blocks:
|
||||
# collect words in this chapter that belong to this speaker block
|
||||
b_words = []
|
||||
for w in ch["words"]:
|
||||
# a word belongs to speaker block if its start is within block start/end (if available)
|
||||
wstart = w.get("start")
|
||||
if wstart is None:
|
||||
continue
|
||||
bstart = b.get("start")
|
||||
bend = b.get("end")
|
||||
if bstart is None or bend is None:
|
||||
continue
|
||||
if bstart <= wstart <= bend:
|
||||
b_words.append(w["word"])
|
||||
if b_words:
|
||||
fh.write(f"{b['speaker']}: ")
|
||||
fh.write(" ".join(b_words) + "\n")
|
||||
fh.write("\n")
|
||||
|
||||
return out_json, out_txt
|
||||
22
api_legacy/main.py
Normal file
22
api_legacy/main.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import asyncio
|
||||
from fastapi import FastAPI
|
||||
from api.route import default, subscription
|
||||
# renamed route modules for clarity
|
||||
from api.route import users, installing, media_convert, mp3_ffmpeg_stream, moviepy
|
||||
#from api.db.subscription import init_db
|
||||
#from api.db.connection import wait_for_db
|
||||
|
||||
app = FastAPI(title="LLM-infa API")
|
||||
|
||||
#@app.on_event("startup")
|
||||
#async def on_startup():
|
||||
# await wait_for_db()
|
||||
# await init_db()
|
||||
|
||||
app.include_router(default.router)
|
||||
#app.include_router(subscription.router)
|
||||
#app.include_router(users.router)
|
||||
app.include_router(installing.router)
|
||||
app.include_router(media_convert.router)
|
||||
app.include_router(mp3_ffmpeg_stream.router)
|
||||
app.include_router(moviepy.router)
|
||||
3
api_legacy/models/base.py
Normal file
3
api_legacy/models/base.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
14
api_legacy/models/subscription.py
Normal file
14
api_legacy/models/subscription.py
Normal file
@@ -0,0 +1,14 @@
|
||||
from sqlalchemy import Column, String, DateTime, ForeignKey, Boolean, Text
|
||||
from api.models.base import Base
|
||||
|
||||
|
||||
class Subscription(Base):
|
||||
__tablename__ = 'subscriptions'
|
||||
uuid = Column(String, primary_key=True, index=True)
|
||||
# one subscription per user (unique)
|
||||
user_id = Column(String, ForeignKey('users.id'), nullable=False, index=True, unique=True)
|
||||
until = Column(DateTime, nullable=False)
|
||||
active = Column(Boolean, nullable=False, default=True)
|
||||
# avoid NULL: store empty string when no link provided
|
||||
subscription_link = Column(Text, nullable=False, default="")
|
||||
connect_link = Column(Text, nullable=False, default="")
|
||||
12
api_legacy/models/user.py
Normal file
12
api_legacy/models/user.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from sqlalchemy import Column, String, DateTime
|
||||
from sqlalchemy.sql import func
|
||||
from api.models.base import Base
|
||||
import uuid
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
telegram_id = Column(String, unique=True, index=True, nullable=False)
|
||||
name = Column(String, nullable=False, default="")
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
|
||||
17
api_legacy/route/default.py
Normal file
17
api_legacy/route/default.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from fastapi import APIRouter
|
||||
import toml
|
||||
from pathlib import Path
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/")
|
||||
async def root():
|
||||
version = "dev"
|
||||
pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
|
||||
if pyproject_path.exists():
|
||||
try:
|
||||
data = toml.load(pyproject_path)
|
||||
version = data.get("project", {}).get("version", "dev")
|
||||
except Exception:
|
||||
pass
|
||||
return {"status": "ok", "msg": "LLM-infa API", "version": version}
|
||||
48
api_legacy/route/installing.py
Normal file
48
api_legacy/route/installing.py
Normal file
@@ -0,0 +1,48 @@
|
||||
# Импортируем необходимые модули
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
import yt_dlp
|
||||
import tempfile
|
||||
from fastapi.responses import FileResponse
|
||||
import os
|
||||
|
||||
router = APIRouter(prefix="/yt-dlp", tags=["yt-dlp"])
|
||||
|
||||
class DownloadRequest(BaseModel):
|
||||
url: str
|
||||
format: str = "best"
|
||||
audio_only: bool = False
|
||||
|
||||
@router.post("/download")
|
||||
async def download_video(data: DownloadRequest):
|
||||
"""
|
||||
Скачать видео по ссылке через yt-dlp и вернуть файл пользователю.
|
||||
"""
|
||||
save_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data'))
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
# choose output dir and format
|
||||
if data.audio_only:
|
||||
out_dir = os.path.join(save_dir, 'audio')
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
outtmpl = os.path.join(out_dir, '%(title)s.%(ext)s')
|
||||
ydl_format = data.format if data.format != "best" else "bestaudio[ext=m4a]/bestaudio"
|
||||
else:
|
||||
outtmpl = os.path.join(save_dir, '%(title)s.%(ext)s')
|
||||
ydl_format = data.format
|
||||
|
||||
ydl_opts = {
|
||||
'outtmpl': outtmpl,
|
||||
'format': ydl_format,
|
||||
# Use aria2c for parallel segmented downloading to speed up large files
|
||||
'external_downloader': 'aria2c',
|
||||
'external_downloader_args': ['-x', '16', '-s', '16', '-k', '1M'],
|
||||
}
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(data.url, download=True)
|
||||
filename = ydl.prepare_filename(info)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Ошибка загрузки: {e}")
|
||||
if not os.path.exists(filename):
|
||||
raise HTTPException(status_code=404, detail="Файл не найден после загрузки")
|
||||
return FileResponse(filename, filename=os.path.basename(filename), media_type='application/octet-stream')
|
||||
344
api_legacy/route/media_convert.py
Normal file
344
api_legacy/route/media_convert.py
Normal file
@@ -0,0 +1,344 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from fastapi.responses import FileResponse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
from typing import List
|
||||
import asyncio
|
||||
import subprocess
|
||||
import shutil
|
||||
import uuid
|
||||
import wave
|
||||
|
||||
router = APIRouter(prefix="/convert", tags=["convert"])
|
||||
|
||||
|
||||
@router.get("/list-audio", response_model=List[str])
|
||||
async def list_audio_files():
|
||||
base_dir = Path(__file__).resolve().parent.parent.parent / "data" / "audio"
|
||||
if not base_dir.exists():
|
||||
return []
|
||||
files = [f.name for f in base_dir.iterdir() if f.is_file()]
|
||||
return files
|
||||
|
||||
|
||||
class DownloadAudioRequest(BaseModel):
|
||||
filename: str
|
||||
|
||||
|
||||
@router.post("/download-audio")
|
||||
async def download_audio_file(data: DownloadAudioRequest):
|
||||
base_dir = Path(__file__).resolve().parent.parent.parent / "data" / "audio"
|
||||
src = (base_dir / data.filename).resolve()
|
||||
try:
|
||||
src.relative_to(base_dir.parent)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists():
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
return FileResponse(str(src), filename=src.name, media_type="audio/mpeg")
|
||||
|
||||
|
||||
class VideoToMp3Request(BaseModel):
|
||||
filename: str
|
||||
|
||||
|
||||
def _run_moviepy(src: Path, dest: Path):
|
||||
from moviepy import VideoFileClip
|
||||
|
||||
clip = VideoFileClip(str(src))
|
||||
try:
|
||||
if clip.audio is None:
|
||||
raise RuntimeError("Source file has no audio stream")
|
||||
clip.audio.write_audiofile(str(dest), codec="libmp3lame", bitrate="192k")
|
||||
finally:
|
||||
clip.close()
|
||||
|
||||
|
||||
@router.post("/video-to-mp3")
|
||||
async def video_to_mp3(req: VideoToMp3Request):
|
||||
try:
|
||||
import moviepy # noqa: F401
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Rebuild image or install moviepy/imageio-ffmpeg")
|
||||
|
||||
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
src = (data_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(data_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
out_dir = data_dir / "audio"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / f"{src.stem}.mp3"
|
||||
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy, src, dest)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
|
||||
|
||||
return FileResponse(str(dest), filename=dest.name, media_type="audio/mpeg")
|
||||
|
||||
|
||||
class VideoToWavRequest(BaseModel):
|
||||
filename: str
|
||||
|
||||
|
||||
def _run_moviepy_to_wav(src: Path, dest: Path):
|
||||
from moviepy import VideoFileClip
|
||||
|
||||
clip = VideoFileClip(str(src))
|
||||
try:
|
||||
if clip.audio is None:
|
||||
raise RuntimeError("Source file has no audio stream")
|
||||
clip.audio.write_audiofile(str(dest), fps=16000, nbytes=2)
|
||||
finally:
|
||||
clip.close()
|
||||
|
||||
|
||||
@router.post("/video-to-wav")
|
||||
async def video_to_wav(req: VideoToWavRequest):
|
||||
try:
|
||||
import moviepy # noqa: F401
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Rebuild image or install moviepy/imageio-ffmpeg")
|
||||
|
||||
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
src = (data_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(data_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
out_dir = data_dir / "audio"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / f"{src.stem}.wav"
|
||||
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy_to_wav, src, dest)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
|
||||
|
||||
return FileResponse(str(dest), filename=dest.name, media_type="audio/wav")
|
||||
|
||||
# MoviePy video-to-audio endpoints moved to api/route/moviepy.py
|
||||
|
||||
|
||||
class Mp3ToTextRequest(BaseModel):
|
||||
filename: str
|
||||
model_path: str
|
||||
|
||||
|
||||
def _run_ffmpeg(src: Path, dest: Path):
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
str(src),
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-vn",
|
||||
"-f",
|
||||
"wav",
|
||||
str(dest),
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
@router.post("/mp3-to-text")
|
||||
async def mp3_to_text(req: Mp3ToTextRequest):
|
||||
logger = __import__("logging").getLogger("convert.mp3-to-text")
|
||||
try:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg in the environment/container.")
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg in the environment/container.")
|
||||
|
||||
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
audio_dir = data_dir / "audio"
|
||||
src = (audio_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(audio_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
audio_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = audio_dir / f"{src.stem}_{uuid.uuid4().hex}.wav"
|
||||
|
||||
try:
|
||||
logger.info("Starting ffmpeg conversion %s -> %s", src.name, dest.name)
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_ffmpeg, src, dest)
|
||||
logger.info("ffmpeg conversion finished")
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise HTTPException(status_code=500, detail=f"ffmpeg conversion failed: {e}")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
|
||||
|
||||
try:
|
||||
from api.lib.audio_processing import VoskModelManager
|
||||
except Exception as e:
|
||||
try:
|
||||
dest.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
|
||||
|
||||
try:
|
||||
logger.info("Starting transcription stage for %s", dest.name)
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
models_dir = project_root / "models"
|
||||
model_path = Path(req.model_path)
|
||||
if not model_path.is_absolute():
|
||||
model_path = project_root / req.model_path
|
||||
model_path = model_path.resolve()
|
||||
try:
|
||||
model_path.relative_to(models_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
|
||||
|
||||
mgr = VoskModelManager(model_path)
|
||||
res = mgr.transcribe_wav_with_words(dest)
|
||||
text = res.get("text", "")
|
||||
# Save plain transcript (no speaker split, no chapters)
|
||||
text_dir = project_root / "data" / "text"
|
||||
text_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_name = dest.stem
|
||||
json_path = text_dir / f"{base_name}.json"
|
||||
txt_path = text_dir / f"{base_name}.txt"
|
||||
with open(json_path, "w", encoding="utf-8") as fh:
|
||||
import json as _json
|
||||
|
||||
_json.dump({"text": text}, fh, ensure_ascii=False, indent=2)
|
||||
with open(txt_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
logger.info("Transcription stage finished for %s", dest.name)
|
||||
finally:
|
||||
try:
|
||||
dest.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"text": text}
|
||||
|
||||
|
||||
class WavToTextRequest(BaseModel):
|
||||
filename: str
|
||||
model_path: str
|
||||
use_embedding: bool = False
|
||||
|
||||
|
||||
@router.post("/wav-to-text")
|
||||
async def wav_to_text(req: WavToTextRequest):
|
||||
logger = __import__("logging").getLogger("convert.wav-to-text")
|
||||
|
||||
data_dir = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
audio_dir = data_dir / "audio"
|
||||
src = (audio_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(audio_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
try:
|
||||
from api.lib.audio_processing import VoskModelManager
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
models_dir = project_root / "models"
|
||||
model_path = Path(req.model_path)
|
||||
if not model_path.is_absolute():
|
||||
model_path = project_root / req.model_path
|
||||
model_path = model_path.resolve()
|
||||
try:
|
||||
model_path.relative_to(models_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
|
||||
|
||||
temp_dest = None
|
||||
to_cleanup = False
|
||||
try:
|
||||
if shutil.which("ffmpeg"):
|
||||
temp_dest = audio_dir / f"{src.stem}_mono_{uuid.uuid4().hex}.wav"
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
str(src),
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-sample_fmt",
|
||||
"s16",
|
||||
str(temp_dest),
|
||||
]
|
||||
try:
|
||||
logger.info("Normalizing WAV via ffmpeg: %s -> %s", src.name, temp_dest.name)
|
||||
subprocess.run(cmd, check=True)
|
||||
src_for_transcription = temp_dest
|
||||
to_cleanup = True
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise HTTPException(status_code=500, detail=f"ffmpeg normalization failed: {e}")
|
||||
else:
|
||||
try:
|
||||
with wave.open(str(src), "rb") as wf:
|
||||
channels = wf.getnchannels()
|
||||
rate = wf.getframerate()
|
||||
sampwidth = wf.getsampwidth()
|
||||
if channels != 1 or rate != 16000 or sampwidth != 2:
|
||||
raise HTTPException(status_code=415, detail=("WAV must be mono PCM16 16kHz. "
|
||||
"Install ffmpeg in the runtime or convert the file before calling this endpoint."))
|
||||
src_for_transcription = src
|
||||
except wave.Error as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid WAV file: {e}")
|
||||
|
||||
mgr = VoskModelManager(model_path)
|
||||
res = mgr.transcribe_wav_with_words(src_for_transcription)
|
||||
text = res.get("text", "")
|
||||
words = res.get("words", [])
|
||||
|
||||
# Save plain transcript (no speaker split, no chapters)
|
||||
text_dir = project_root / "data" / "text"
|
||||
text_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_name = src.stem
|
||||
json_path = text_dir / f"{base_name}.json"
|
||||
txt_path = text_dir / f"{base_name}.txt"
|
||||
import json as _json
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as fh:
|
||||
_json.dump({"text": text}, fh, ensure_ascii=False, indent=2)
|
||||
with open(txt_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
logger.info("WAV transcription finished: %s", src.name)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Transcription failed: {e}")
|
||||
finally:
|
||||
if to_cleanup and temp_dest is not None:
|
||||
try:
|
||||
temp_dest.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"text": text, "json": str(json_path.name), "txt": str(txt_path.name)}
|
||||
97
api_legacy/route/moviepy.py
Normal file
97
api_legacy/route/moviepy.py
Normal file
@@ -0,0 +1,97 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from fastapi.responses import FileResponse
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
|
||||
router = APIRouter(prefix="/moviepy", tags=["moviepy"])
|
||||
|
||||
|
||||
class VideoToMp3Request(BaseModel):
|
||||
filename: str
|
||||
|
||||
|
||||
class VideoToWavRequest(BaseModel):
|
||||
filename: str
|
||||
|
||||
|
||||
def _run_moviepy(src: Path, dest: Path):
|
||||
from moviepy import VideoFileClip
|
||||
|
||||
clip = VideoFileClip(str(src))
|
||||
try:
|
||||
if clip.audio is None:
|
||||
raise RuntimeError("Source file has no audio stream")
|
||||
clip.audio.write_audiofile(str(dest), codec="libmp3lame", bitrate="192k")
|
||||
finally:
|
||||
clip.close()
|
||||
|
||||
|
||||
def _run_moviepy_to_wav(src: Path, dest: Path):
|
||||
from moviepy import VideoFileClip
|
||||
|
||||
clip = VideoFileClip(str(src))
|
||||
try:
|
||||
if clip.audio is None:
|
||||
raise RuntimeError("Source file has no audio stream")
|
||||
clip.audio.write_audiofile(str(dest), fps=16000, nbytes=2)
|
||||
finally:
|
||||
clip.close()
|
||||
|
||||
|
||||
@router.post("/video-to-mp3")
|
||||
async def video_to_mp3(req: VideoToMp3Request):
|
||||
try:
|
||||
import moviepy # noqa: F401
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Install moviepy/imageio-ffmpeg")
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
data_dir = project_root / "data"
|
||||
src = (data_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(data_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
out_dir = data_dir / "audio"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / f"{src.stem}.mp3"
|
||||
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy, src, dest)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
|
||||
|
||||
return FileResponse(str(dest), filename=dest.name, media_type="audio/mpeg")
|
||||
|
||||
|
||||
@router.post("/video-to-wav")
|
||||
async def video_to_wav(req: VideoToWavRequest):
|
||||
try:
|
||||
import moviepy # noqa: F401
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="moviepy is not installed in the runtime. Install moviepy/imageio-ffmpeg")
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
data_dir = project_root / "data"
|
||||
src = (data_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(data_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
out_dir = data_dir / "audio"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / f"{src.stem}.wav"
|
||||
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_moviepy_to_wav, src, dest)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"conversion failed: {e}")
|
||||
|
||||
return FileResponse(str(dest), filename=dest.name, media_type="audio/wav")
|
||||
138
api_legacy/route/mp3_ffmpeg_stream.py
Normal file
138
api_legacy/route/mp3_ffmpeg_stream.py
Normal file
@@ -0,0 +1,138 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import shutil
|
||||
import json
|
||||
import logging
|
||||
|
||||
try:
|
||||
from vosk import Model, KaldiRecognizer
|
||||
except Exception:
|
||||
Model = None
|
||||
KaldiRecognizer = None
|
||||
|
||||
router = APIRouter(prefix="/convert", tags=["convert"])
|
||||
|
||||
|
||||
class Mp3ToTextRequest(BaseModel):
|
||||
filename: str
|
||||
model_path: str
|
||||
|
||||
|
||||
@router.post("/mp3-to-text-ffmpeg")
|
||||
async def mp3_to_text_ffmpeg(req: Mp3ToTextRequest):
|
||||
"""Convert MP3 -> PCM via ffmpeg (stdout) and transcribe with Vosk.
|
||||
|
||||
This endpoint requires `ffmpeg` available in PATH. It decodes the input
|
||||
MP3 to raw PCM s16le 16k mono and streams it into Vosk recognizer.
|
||||
"""
|
||||
logger = logging.getLogger("convert.mp3")
|
||||
|
||||
if Model is None or KaldiRecognizer is None:
|
||||
raise HTTPException(status_code=503, detail="vosk is not installed in runtime")
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
audio_dir = project_root / "data" / "audio"
|
||||
models_dir = project_root / "models"
|
||||
|
||||
src = (audio_dir / req.filename).resolve()
|
||||
try:
|
||||
src.relative_to(audio_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
if not src.exists() or not src.is_file():
|
||||
raise HTTPException(status_code=404, detail="Source file not found")
|
||||
|
||||
model_path = Path(req.model_path)
|
||||
if not model_path.is_absolute():
|
||||
model_path = project_root / req.model_path
|
||||
model_path = model_path.resolve()
|
||||
try:
|
||||
model_path.relative_to(models_dir)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="model_path must point inside the project's models directory")
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=400, detail=f"Model path not found: {model_path}")
|
||||
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg to use this endpoint.")
|
||||
|
||||
try:
|
||||
from api.lib.audio_processing import FFmpegConverter, VoskModelManager
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Internal import error: {e}")
|
||||
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise HTTPException(status_code=503, detail="ffmpeg is not available in the runtime. Install ffmpeg to use this endpoint.")
|
||||
|
||||
# Stream decode via ffmpeg to temporary process and feed recognizer
|
||||
try:
|
||||
mgr = VoskModelManager(model_path)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to load Vosk model: {e}")
|
||||
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
str(src),
|
||||
"-f",
|
||||
"s16le",
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-",
|
||||
]
|
||||
|
||||
logger.info("Starting ffmpeg streaming decode")
|
||||
try:
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to start ffmpeg: {e}")
|
||||
|
||||
if proc.stdout is None:
|
||||
proc.kill()
|
||||
raise HTTPException(status_code=500, detail="ffmpeg did not provide stdout")
|
||||
|
||||
recognizer = None
|
||||
try:
|
||||
recognizer = KaldiRecognizer(mgr.model, 16000)
|
||||
try:
|
||||
recognizer.SetWords(True)
|
||||
except Exception:
|
||||
pass
|
||||
bytes_read = 0
|
||||
total = None
|
||||
try:
|
||||
total = src.stat().st_size
|
||||
except Exception:
|
||||
total = None
|
||||
|
||||
last_pct = 0
|
||||
while True:
|
||||
chunk = proc.stdout.read(4000)
|
||||
if not chunk:
|
||||
break
|
||||
recognizer.AcceptWaveform(chunk)
|
||||
bytes_read += len(chunk)
|
||||
if total:
|
||||
pct = int(bytes_read * 100 / total)
|
||||
if pct - last_pct >= 5:
|
||||
last_pct = pct
|
||||
logger.info("Decoding+transcription progress: %s%%", pct)
|
||||
|
||||
result = json.loads(recognizer.FinalResult())
|
||||
text = result.get("text", "")
|
||||
logger.info("Transcription finished; result length=%d", len(text))
|
||||
return {"text": text}
|
||||
finally:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
94
api_legacy/route/subscription.py
Normal file
94
api_legacy/route/subscription.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List
|
||||
from api.db.subscription import (
|
||||
create_subscription,
|
||||
get_subscription_by_user,
|
||||
get_subscription_by_uuid,
|
||||
update_subscription,
|
||||
delete_subscription,
|
||||
list_subscriptions,
|
||||
extend_subscription_by_user,
|
||||
)
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
router = APIRouter(prefix="/subscription", tags=["subscription"])
|
||||
|
||||
|
||||
class SubscriptionInfo(BaseModel):
|
||||
user_id: str
|
||||
uuid: str
|
||||
until: datetime
|
||||
active: bool
|
||||
subscription_link: Optional[str] = None
|
||||
connect_link: Optional[str] = None
|
||||
|
||||
|
||||
class SubscriptionCreate(BaseModel):
|
||||
user_id: str
|
||||
months: int
|
||||
subscription_link: Optional[str] = None
|
||||
connect_link: Optional[str] = None
|
||||
|
||||
|
||||
class SubscriptionUpdate(BaseModel):
|
||||
until: Optional[datetime]
|
||||
user_id: Optional[str]
|
||||
active: Optional[bool] = None
|
||||
subscription_link: Optional[str] = None
|
||||
connect_link: Optional[str] = None
|
||||
|
||||
|
||||
@router.get("/{user_id}", response_model=Optional[SubscriptionInfo])
|
||||
async def get_user_subscription(user_id: str):
|
||||
sub = await get_subscription_by_user(user_id)
|
||||
if not sub:
|
||||
return None
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until)
|
||||
|
||||
|
||||
@router.post("/new", response_model=SubscriptionInfo)
|
||||
async def create_subscription_endpoint(data: SubscriptionCreate):
|
||||
sub_id = str(uuid.uuid4())
|
||||
until = datetime.now() + timedelta(days=30 * data.months)
|
||||
sub = await create_subscription(data.user_id, sub_id, until, subscription_link=data.subscription_link, connect_link=data.connect_link, active=True)
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until)
|
||||
|
||||
|
||||
@router.post("/extend", response_model=SubscriptionInfo)
|
||||
async def extend_user_subscription(data: SubscriptionCreate):
|
||||
sub = await extend_subscription_by_user(data.user_id, data.months)
|
||||
if not sub:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found")
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
|
||||
|
||||
|
||||
@router.get("/by-uuid/{sub_uuid}", response_model=Optional[SubscriptionInfo])
|
||||
async def get_subscription_uuid(sub_uuid: str):
|
||||
sub = await get_subscription_by_uuid(sub_uuid)
|
||||
if not sub:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found")
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
|
||||
|
||||
|
||||
@router.patch("/{sub_uuid}", response_model=SubscriptionInfo)
|
||||
async def update_subscription_endpoint(sub_uuid: str, data: SubscriptionUpdate):
|
||||
sub = await update_subscription(sub_uuid, until=data.until, user_id=data.user_id, active=data.active, subscription_link=data.subscription_link, connect_link=data.connect_link)
|
||||
if not sub:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found")
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
|
||||
|
||||
|
||||
@router.delete("/{sub_uuid}")
|
||||
async def delete_subscription_endpoint(sub_uuid: str):
|
||||
ok = await delete_subscription(sub_uuid)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/", response_model=List[SubscriptionInfo])
|
||||
async def list_all_subscriptions():
|
||||
subs = await list_subscriptions()
|
||||
return [SubscriptionInfo(user_id=s.user_id, uuid=s.uuid, until=s.until) for s in subs]
|
||||
107
api_legacy/route/users.py
Normal file
107
api_legacy/route/users.py
Normal file
@@ -0,0 +1,107 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List
|
||||
from api.db.user import create_user, get_user, get_user_by_telegram, update_user, delete_user, list_users
|
||||
from api.db.subscription import (
|
||||
create_subscription,
|
||||
get_subscription_by_user,
|
||||
update_subscription,
|
||||
delete_subscription,
|
||||
)
|
||||
from api.route.subscription import SubscriptionInfo
|
||||
|
||||
router = APIRouter(prefix="/users", tags=["users"])
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
id: str
|
||||
telegram_id: str
|
||||
name: Optional[str]
|
||||
subscription: Optional[SubscriptionInfo] = None
|
||||
|
||||
|
||||
class UserCreate(BaseModel):
|
||||
telegram_id: str
|
||||
name: Optional[str]
|
||||
|
||||
|
||||
class UserUpdate(BaseModel):
|
||||
name: Optional[str]
|
||||
|
||||
|
||||
@router.post("/", response_model=UserOut)
|
||||
async def api_create_user(data: UserCreate):
|
||||
existing = await get_user_by_telegram(data.telegram_id)
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="User already exists")
|
||||
u = await create_user(data.telegram_id, data.name)
|
||||
return UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
|
||||
|
||||
|
||||
@router.get("/{user_id}", response_model=UserOut)
|
||||
async def api_get_user(user_id: str, expand: Optional[str] = None):
|
||||
u = await get_user(user_id)
|
||||
if not u:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
out = UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
|
||||
if expand == "subscription":
|
||||
sub = await get_subscription_by_user(u.id)
|
||||
if sub:
|
||||
out.subscription = SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
|
||||
return out
|
||||
|
||||
|
||||
@router.post("/{user_id}/subscription", response_model=SubscriptionInfo)
|
||||
async def api_create_user_subscription(user_id: str, data: dict):
|
||||
# expected keys: months, subscription_link (opt), connect_link (opt)
|
||||
months = int(data.get("months", 1))
|
||||
subscription_link = data.get("subscription_link")
|
||||
connect_link = data.get("connect_link")
|
||||
import uuid as _uuid
|
||||
sub_id = str(_uuid.uuid4())
|
||||
from datetime import datetime, timedelta
|
||||
until = datetime.now() + timedelta(days=30 * months)
|
||||
sub = await create_subscription(user_id, sub_id, until, subscription_link=subscription_link, connect_link=connect_link, active=True)
|
||||
return SubscriptionInfo(user_id=sub.user_id, uuid=sub.uuid, until=sub.until, active=sub.active, subscription_link=sub.subscription_link, connect_link=sub.connect_link)
|
||||
|
||||
|
||||
@router.patch("/{user_id}/subscription", response_model=SubscriptionInfo)
|
||||
async def api_update_user_subscription(user_id: str, data: dict):
|
||||
sub = await get_subscription_by_user(user_id)
|
||||
if not sub:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found for user")
|
||||
updated = await update_subscription(sub.uuid, until=data.get("until"), user_id=data.get("user_id"), active=data.get("active"), subscription_link=data.get("subscription_link"), connect_link=data.get("connect_link"))
|
||||
return SubscriptionInfo(user_id=updated.user_id, uuid=updated.uuid, until=updated.until, active=updated.active, subscription_link=updated.subscription_link, connect_link=updated.connect_link)
|
||||
|
||||
|
||||
@router.delete("/{user_id}/subscription")
|
||||
async def api_delete_user_subscription(user_id: str):
|
||||
sub = await get_subscription_by_user(user_id)
|
||||
if not sub:
|
||||
raise HTTPException(status_code=404, detail="Subscription not found for user")
|
||||
ok = await delete_subscription(sub.uuid)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=500, detail="Failed to delete subscription")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/", response_model=List[UserOut])
|
||||
async def api_list_users():
|
||||
users = await list_users()
|
||||
return [UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name) for u in users]
|
||||
|
||||
|
||||
@router.patch("/{user_id}", response_model=UserOut)
|
||||
async def api_update_user(user_id: str, data: UserUpdate):
|
||||
u = await update_user(user_id, name=data.name)
|
||||
if not u:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
return UserOut(id=u.id, telegram_id=u.telegram_id, name=u.name)
|
||||
|
||||
|
||||
@router.delete("/{user_id}")
|
||||
async def api_delete_user(user_id: str):
|
||||
ok = await delete_user(user_id)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
return {"ok": True}
|
||||
Reference in New Issue
Block a user