Рабочий пайплайн: YouTube -> транскрипция -> выжимка -> Postgres
- новый api/: /pipeline/process (yt-dlp, субтитры или Vosk, LLM/экстрактивная суммаризация), /videos, /llm; старый код перенесён в api_legacy/ - web/: Flet UI (flet 0.28.3 + flet-web, порт 8550) - Dockerfile.api: ffmpeg слоем из mwader/static-ffmpeg, pip с кэш-маунтом - requirements/pyproject: убраны moviepy, imageio-ffmpeg, битый asyncio - README с инструкцией запуска и загрузкой Vosk-модели Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jze9
0
api/db/__init__.py
Normal file
0
api/db/__init__.py
Normal file
@@ -1,27 +1,63 @@
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy import text
|
||||
import asyncio
|
||||
from api.config import settings
|
||||
import logging
|
||||
|
||||
DATABASE_URL = settings.DATABASE_URL
|
||||
engine = create_async_engine(DATABASE_URL, echo=False, future=True)
|
||||
import asyncpg
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from api.config import settings
|
||||
from api.models.base import Base
|
||||
import api.models.video # noqa: F401 - register model with Base.metadata
|
||||
|
||||
|
||||
logger = logging.getLogger("db")
|
||||
|
||||
engine = create_async_engine(settings.database_url, echo=False, future=True)
|
||||
SessionLocal = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
||||
|
||||
|
||||
async def wait_for_db(retries: int = 30, delay: float = 1.0) -> None:
|
||||
"""Wait until the database is available. Retries with exponential backoff.
|
||||
async def wait_for_postgres(retries: int = 30, delay: float = 1.0) -> None:
|
||||
"""Wait for the postgres server to accept connections (using the default 'postgres' db)."""
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
conn = await asyncpg.connect(
|
||||
user=settings.DB_USER,
|
||||
password=settings.DB_PASS,
|
||||
database="postgres",
|
||||
host=settings.DB_HOST,
|
||||
port=int(settings.DB_PORT),
|
||||
)
|
||||
await conn.close()
|
||||
return
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
wait = min(delay * (2 ** (attempt - 1)), 5)
|
||||
logger.info("Waiting for postgres (attempt %s/%s): %s", attempt, retries, e)
|
||||
await asyncio.sleep(wait)
|
||||
raise RuntimeError(f"Could not connect to postgres after {retries} attempts") from last_exc
|
||||
|
||||
Raises RuntimeError if DB is still unavailable after retries.
|
||||
"""
|
||||
last_exc = None
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
return
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
wait = min(delay * (2 ** (attempt - 1)), 5)
|
||||
await asyncio.sleep(wait)
|
||||
raise RuntimeError(f"Could not connect to DB after {retries} attempts") from last_exc
|
||||
|
||||
async def _create_database_if_missing() -> None:
|
||||
admin = await asyncpg.connect(
|
||||
user=settings.DB_USER,
|
||||
password=settings.DB_PASS,
|
||||
database="postgres",
|
||||
host=settings.DB_HOST,
|
||||
port=int(settings.DB_PORT),
|
||||
)
|
||||
try:
|
||||
exists = await admin.fetchval(
|
||||
"SELECT 1 FROM pg_database WHERE datname=$1", settings.DB_NAME
|
||||
)
|
||||
if not exists:
|
||||
await admin.execute(f'CREATE DATABASE "{settings.DB_NAME}"')
|
||||
finally:
|
||||
await admin.close()
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
await _create_database_if_missing()
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
from api.db.connection import engine, SessionLocal
|
||||
from api.models.subscription import Subscription
|
||||
from api.models.base import Base
|
||||
from api.config import settings
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy import select
|
||||
import asyncpg
|
||||
|
||||
|
||||
async def init_db():
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
return
|
||||
except Exception as exc: # try to create database if it doesn't exist
|
||||
msg = str(exc).lower()
|
||||
if "does not exist" in msg or "invalidcatalogname" in msg or "database" in msg:
|
||||
try:
|
||||
admin_conn = await asyncpg.connect(
|
||||
user=settings.DB_USER,
|
||||
password=settings.DB_PASS,
|
||||
database="postgres",
|
||||
host=settings.DB_HOST,
|
||||
port=int(settings.DB_PORT),
|
||||
)
|
||||
try:
|
||||
await admin_conn.execute(f'CREATE DATABASE "{settings.DB_NAME}"')
|
||||
except Exception:
|
||||
pass
|
||||
await admin_conn.close()
|
||||
except Exception:
|
||||
raise
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
async def create_subscription(user_id: str, uuid: str, until: datetime, subscription_link: str | None = None, connect_link: str | None = None, active: bool = True):
|
||||
async with SessionLocal() as session:
|
||||
# if user already has a subscription, remove it (replace)
|
||||
q = select(Subscription).where(Subscription.user_id == user_id)
|
||||
res = await session.execute(q)
|
||||
existing = res.scalars().first()
|
||||
if existing:
|
||||
await session.delete(existing)
|
||||
# avoid NULL in DB: replace None links with empty strings
|
||||
safe_subscription_link = subscription_link if subscription_link is not None else ""
|
||||
safe_connect_link = connect_link if connect_link is not None else ""
|
||||
sub = Subscription(user_id=user_id, uuid=uuid, until=until, active=active, subscription_link=safe_subscription_link, connect_link=safe_connect_link)
|
||||
session.add(sub)
|
||||
await session.commit()
|
||||
await session.refresh(sub)
|
||||
return sub
|
||||
|
||||
|
||||
async def get_subscription_by_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.user_id == user_id)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
return sub
|
||||
|
||||
|
||||
async def get_subscription_by_uuid(sub_uuid: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def update_subscription(sub_uuid: str, until: datetime = None, user_id: str | None = None, active: bool | None = None, subscription_link: str | None = None, connect_link: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
if not sub:
|
||||
return None
|
||||
if until is not None:
|
||||
sub.until = until
|
||||
if user_id is not None:
|
||||
sub.user_id = user_id
|
||||
if active is not None:
|
||||
sub.active = active
|
||||
if subscription_link is not None:
|
||||
sub.subscription_link = subscription_link
|
||||
if connect_link is not None:
|
||||
sub.connect_link = connect_link
|
||||
await session.commit()
|
||||
return sub
|
||||
|
||||
|
||||
async def delete_subscription(sub_uuid: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription).where(Subscription.uuid == sub_uuid)
|
||||
res = await session.execute(q)
|
||||
sub = res.scalars().first()
|
||||
if not sub:
|
||||
return False
|
||||
await session.delete(sub)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_subscriptions():
|
||||
async with SessionLocal() as session:
|
||||
q = select(Subscription)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().all()
|
||||
|
||||
|
||||
async def extend_subscription_by_user(user_id: str, months: int):
|
||||
sub = await get_subscription_by_user(user_id)
|
||||
if sub:
|
||||
new_until = max(sub.until, datetime.now()) + timedelta(days=30 * months)
|
||||
return await update_subscription(sub.uuid, until=new_until, active=True)
|
||||
return None
|
||||
|
||||
|
||||
async def deactivate_subscription(sub_uuid: str):
|
||||
return await update_subscription(sub_uuid, active=False)
|
||||
@@ -1,64 +0,0 @@
|
||||
from api.db.connection import SessionLocal
|
||||
from api.models.user import User
|
||||
from sqlalchemy import select
|
||||
|
||||
|
||||
import uuid
|
||||
|
||||
|
||||
async def create_user(telegram_id: str, name: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
# ensure no NULL in DB: replace None with empty string
|
||||
safe_name = name if name is not None else ""
|
||||
user = User(id=str(uuid.uuid4()), telegram_id=telegram_id, name=safe_name)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def get_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def get_user_by_telegram(telegram_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.telegram_id == telegram_id)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def update_user(user_id: str, name: str | None = None):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
user = res.scalars().first()
|
||||
if not user:
|
||||
return None
|
||||
if name is not None:
|
||||
# replace None handled by caller; here update only when provided
|
||||
user.name = name
|
||||
await session.commit()
|
||||
return user
|
||||
|
||||
|
||||
async def delete_user(user_id: str):
|
||||
async with SessionLocal() as session:
|
||||
q = select(User).where(User.id == user_id)
|
||||
res = await session.execute(q)
|
||||
user = res.scalars().first()
|
||||
if not user:
|
||||
return False
|
||||
await session.delete(user)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_users():
|
||||
async with SessionLocal() as session:
|
||||
q = select(User)
|
||||
res = await session.execute(q)
|
||||
return res.scalars().all()
|
||||
53
api/db/video.py
Normal file
53
api/db/video.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from sqlalchemy import select
|
||||
|
||||
from api.db.connection import SessionLocal
|
||||
from api.models.video import Video
|
||||
|
||||
|
||||
async def create_video(
|
||||
*,
|
||||
uuid: str,
|
||||
source_url: str,
|
||||
title: str,
|
||||
video_path: str,
|
||||
text_full_path: str,
|
||||
text_summary_path: str,
|
||||
transcription_method: str,
|
||||
) -> Video:
|
||||
async with SessionLocal() as session:
|
||||
v = Video(
|
||||
uuid=uuid,
|
||||
source_url=source_url,
|
||||
title=title or "",
|
||||
video_path=video_path or "",
|
||||
text_full_path=text_full_path or "",
|
||||
text_summary_path=text_summary_path or "",
|
||||
transcription_method=transcription_method or "",
|
||||
)
|
||||
session.add(v)
|
||||
await session.commit()
|
||||
await session.refresh(v)
|
||||
return v
|
||||
|
||||
|
||||
async def get_video(video_uuid: str) -> Video | None:
|
||||
async with SessionLocal() as session:
|
||||
res = await session.execute(select(Video).where(Video.uuid == video_uuid))
|
||||
return res.scalars().first()
|
||||
|
||||
|
||||
async def list_videos() -> list[Video]:
|
||||
async with SessionLocal() as session:
|
||||
res = await session.execute(select(Video).order_by(Video.created_at.desc()))
|
||||
return list(res.scalars().all())
|
||||
|
||||
|
||||
async def delete_video(video_uuid: str) -> bool:
|
||||
async with SessionLocal() as session:
|
||||
res = await session.execute(select(Video).where(Video.uuid == video_uuid))
|
||||
v = res.scalars().first()
|
||||
if not v:
|
||||
return False
|
||||
await session.delete(v)
|
||||
await session.commit()
|
||||
return True
|
||||
Reference in New Issue
Block a user