diff --git a/.env.example b/.env.example index 3ddeab8..2b2b00d 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,6 @@ ALLOWED_ORIGINS=http://localhost,http://localhost:80 ADMIN_USERNAME=admin ADMIN_PASSWORD=changeme - # VK импорт (необязательно) # Сервисный токен приложения: vk.com/apps → Настройки → Сервисный ключ доступа VK_ACCESS_TOKEN= diff --git a/api/alembic.ini b/api/alembic.ini new file mode 100644 index 0000000..08f1d9f --- /dev/null +++ b/api/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/api/bd/models.py b/api/bd/models.py index 7802c7c..86143ae 100644 --- a/api/bd/models.py +++ b/api/bd/models.py @@ -72,6 +72,9 @@ class Media(Base): media_type: Mapped[str] = mapped_column(String(20)) # image | video size_bytes: Mapped[int] = mapped_column(Integer, default=0) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + article_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("articles.id", ondelete="SET NULL"), nullable=True + ) class VkSource(Base): diff --git a/api/main.py b/api/main.py index 0ebda36..b34dd2c 100644 --- a/api/main.py +++ b/api/main.py @@ -10,7 +10,7 @@ from slowapi import _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded from rate_limiter import limiter -from bd.database import wait_for_db, engine, Base +from bd.database import wait_for_db from route.public import router as public_router from route.admin_auth import router as auth_router, ensure_default_admin from route.admin_articles import router as articles_router @@ -32,11 +32,22 @@ def _verify_docs(creds: HTTPBasicCredentials = Depends(_basic)): return creds.username +def _run_migrations(): + import subprocess, sys + result = subprocess.run( + ["alembic", "upgrade", "head"], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f"[Alembic] Ошибка миграции:\n{result.stderr}", file=sys.stderr) + else: + print(f"[Alembic] {result.stdout.strip() or 'Миграции применены'}") + + @asynccontextmanager async def lifespan(app: FastAPI): await wait_for_db() - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) + _run_migrations() await ensure_default_admin() scheduler.start() yield diff --git a/api/migrations/README b/api/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/api/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/api/migrations/env.py b/api/migrations/env.py new file mode 100644 index 0000000..9f8a7fb --- /dev/null +++ b/api/migrations/env.py @@ -0,0 +1,63 @@ +import asyncio +import os +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подключаем модели для autogenerate +from bd.database import Base +import bd.models # noqa: F401 + +target_metadata = Base.metadata + +# URL из ENV (приоритет) или из alembic.ini +DATABASE_URL = os.getenv("DATABASE_URL", config.get_main_option("sqlalchemy.url", "")) +config.set_main_option("sqlalchemy.url", DATABASE_URL) + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/api/migrations/script.py.mako b/api/migrations/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/api/migrations/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/api/migrations/versions/25d6ed4a524a_initial.py b/api/migrations/versions/25d6ed4a524a_initial.py new file mode 100644 index 0000000..a8ae281 --- /dev/null +++ b/api/migrations/versions/25d6ed4a524a_initial.py @@ -0,0 +1,109 @@ +"""initial + +Revision ID: 25d6ed4a524a +Revises: +Create Date: 2026-05-15 14:46:07.892008 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '25d6ed4a524a' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('admins', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('username', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.String(length=300), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username') + ) + op.create_table('categories', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(length=100), nullable=False), + sa.Column('slug', sa.String(length=100), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_index(op.f('ix_categories_slug'), 'categories', ['slug'], unique=True) + op.create_table('tags', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(length=100), nullable=False), + sa.Column('slug', sa.String(length=100), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_index(op.f('ix_tags_slug'), 'tags', ['slug'], unique=True) + op.create_table('vk_sources', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('group_id', sa.String(length=50), nullable=False), + sa.Column('group_name', sa.String(length=200), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('last_run', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('group_id') + ) + op.create_table('articles', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('title', sa.String(length=500), nullable=False), + sa.Column('slug', sa.String(length=500), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('excerpt', sa.String(length=1000), nullable=False), + sa.Column('cover_url', sa.String(length=1000), nullable=True), + sa.Column('font_family', sa.String(length=100), nullable=False), + sa.Column('source_url', sa.String(length=1000), nullable=True), + sa.Column('status', sa.Enum('draft', 'published', name='articlestatus'), nullable=False), + sa.Column('view_count', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('published_at', sa.DateTime(), nullable=True), + sa.Column('category_id', sa.UUID(), nullable=True), + sa.ForeignKeyConstraint(['category_id'], ['categories.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_articles_slug'), 'articles', ['slug'], unique=True) + op.create_table('article_tag', + sa.Column('article_id', sa.UUID(), nullable=True), + sa.Column('tag_id', sa.UUID(), nullable=True), + sa.ForeignKeyConstraint(['article_id'], ['articles.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tags.id'], ondelete='CASCADE') + ) + op.create_table('media', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('filename', sa.String(length=500), nullable=False), + sa.Column('url', sa.String(length=1000), nullable=False), + sa.Column('media_type', sa.String(length=20), nullable=False), + sa.Column('size_bytes', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('article_id', sa.UUID(), nullable=True), + sa.ForeignKeyConstraint(['article_id'], ['articles.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('media') + op.drop_table('article_tag') + op.drop_index(op.f('ix_articles_slug'), table_name='articles') + op.drop_table('articles') + op.drop_table('vk_sources') + op.drop_index(op.f('ix_tags_slug'), table_name='tags') + op.drop_table('tags') + op.drop_index(op.f('ix_categories_slug'), table_name='categories') + op.drop_table('categories') + op.drop_table('admins') + # ### end Alembic commands ### diff --git a/api/requirements.txt b/api/requirements.txt index 0119fd3..9ac1d31 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,3 +1,4 @@ +alembic==1.13.3 fastapi==0.115.0 uvicorn[standard]==0.30.6 sqlalchemy[asyncio]==2.0.36 diff --git a/api/route/admin_articles.py b/api/route/admin_articles.py index bfc7726..b574af9 100644 --- a/api/route/admin_articles.py +++ b/api/route/admin_articles.py @@ -1,3 +1,4 @@ +import os import uuid from datetime import datetime from typing import Optional @@ -9,9 +10,11 @@ from sqlalchemy.orm import selectinload import redis.asyncio as aioredis from bd.database import get_db, get_redis -from bd.models import Article, Category, Tag, ArticleStatus +from bd.models import Article, Category, Tag, ArticleStatus, Media from route.deps import require_admin +MINIO_BUCKET = os.getenv("MINIO_BUCKET", "news-media") + router = APIRouter() @@ -208,6 +211,26 @@ async def delete_article( article = (await db.execute(select(Article).where(Article.id == uuid.UUID(article_id)))).scalar_one_or_none() if not article: raise HTTPException(status_code=404, detail="Not found") + + # Удаляем все связанные медиафайлы из MinIO и БД + media_items = (await db.execute( + select(Media).where(Media.article_id == article.id) + )).scalars().all() + + if media_items: + from route.admin_media import get_minio + try: + client = get_minio() + for m in media_items: + obj = m.url.split(f"/{MINIO_BUCKET}/", 1)[-1] + try: + client.remove_object(MINIO_BUCKET, obj) + except Exception: + pass + await db.delete(m) + except Exception: + pass + await db.delete(article) await db.commit() await _invalidate_cache(redis) diff --git a/api/route/admin_media.py b/api/route/admin_media.py index 7c32268..8281010 100644 --- a/api/route/admin_media.py +++ b/api/route/admin_media.py @@ -7,9 +7,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select from minio import Minio from minio.error import S3Error +from slugify import slugify from bd.database import get_db -from bd.models import Media +from bd.models import Media, Article from route.deps import require_admin router = APIRouter() @@ -50,6 +51,7 @@ def ensure_bucket(client: Minio): @router.post("/upload") async def upload_media( file: UploadFile = File(...), + article_id: str | None = Query(None), db: AsyncSession = Depends(get_db), _: str = Depends(require_admin), ): @@ -63,7 +65,27 @@ async def upload_media( media_type = "image" if content_type in ALLOWED_IMAGE else "video" ext = (file.filename or "file").rsplit(".", 1)[-1].lower() - object_name = f"{media_type}s/{uuid.uuid4().hex}.{ext}" + + # Определяем путь в MinIO + art_uuid = None + if article_id: + try: + art_uuid = uuid.UUID(article_id) + # Проверяем существование статьи + art = (await db.execute(select(Article).where(Article.id == art_uuid))).scalar_one_or_none() + if not art: + art_uuid = None + except ValueError: + art_uuid = None + + if art_uuid: + folder = "images" if media_type == "image" else "videos" + art_slug = slugify(art.title) if art and art.title else str(art_uuid)[:8] + art_dir = f"{art_slug}-{str(art_uuid)[:8]}" + base_name = f"{art_slug}-{uuid.uuid4().hex[:6]}.{ext}" + object_name = f"articles/{art_dir}/{folder}/{base_name}" + else: + object_name = f"{media_type}s/{uuid.uuid4().hex}.{ext}" client = get_minio() ensure_bucket(client) @@ -78,10 +100,11 @@ async def upload_media( public_url = f"{MINIO_PUBLIC_URL}/{MINIO_BUCKET}/{object_name}" media = Media( - filename=file.filename or object_name, + filename=object_name.split("/")[-1], url=public_url, media_type=media_type, size_bytes=len(data), + article_id=art_uuid, ) db.add(media) await db.commit() @@ -101,20 +124,40 @@ async def list_media( offset: int = Query(0, ge=0), limit: int = Query(40, ge=1, le=200), media_type: str | None = Query(None), + q: str | None = Query(None, max_length=200), db: AsyncSession = Depends(get_db), _: str = Depends(require_admin), ): from sqlalchemy import func + from sqlalchemy.orm import outerjoin base = select(Media) if media_type: base = base.where(Media.media_type == media_type) + if q: + base = base.where(Media.filename.ilike(f"%{q}%")) total = (await db.execute(select(func.count()).select_from(base.subquery()))).scalar_one() items = (await db.execute(base.order_by(Media.created_at.desc()).offset(offset).limit(limit))).scalars().all() + + # Fetch article titles for media that have an article_id + art_ids = {m.article_id for m in items if m.article_id} + art_titles: dict = {} + if art_ids: + arts = (await db.execute(select(Article.id, Article.title).where(Article.id.in_(art_ids)))).all() + art_titles = {a.id: a.title for a in arts} + return { "total": total, "items": [ - {"id": str(m.id), "url": m.url, "filename": m.filename, "media_type": m.media_type, - "size_bytes": m.size_bytes, "created_at": m.created_at.isoformat()} + { + "id": str(m.id), + "url": m.url, + "filename": m.filename, + "media_type": m.media_type, + "size_bytes": m.size_bytes, + "created_at": m.created_at.isoformat(), + "article_id": str(m.article_id) if m.article_id else None, + "article_title": art_titles.get(m.article_id) if m.article_id else None, + } for m in items ], } diff --git a/api/route/admin_vk.py b/api/route/admin_vk.py index 9e0a7e4..86b13f4 100644 --- a/api/route/admin_vk.py +++ b/api/route/admin_vk.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException # BackgroundTasks используется в history endpoints from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -88,9 +88,9 @@ async def delete_source(source_id: str, db: AsyncSession = Depends(get_db), _: s # ── Ручной запуск импорта ───────────────────────────────────────────────────── @router.post("/import") -async def vk_manual_import(background_tasks: BackgroundTasks, _: str = Depends(require_admin)): - background_tasks.add_task(run_import) - return {"ok": True, "message": "Импорт последних постов запущен в фоне"} +async def vk_manual_import(_: str = Depends(require_admin)): + result = await run_import() + return result @router.post("/import/history") diff --git a/api/route/public.py b/api/route/public.py index 7626896..69990a0 100644 --- a/api/route/public.py +++ b/api/route/public.py @@ -75,13 +75,15 @@ async def list_news( if date_from: try: - base = base.where(Article.published_at >= datetime.fromisoformat(date_from)) + dt = datetime.fromisoformat(date_from.replace("Z", "+00:00")) + base = base.where(Article.published_at >= dt.replace(tzinfo=None)) except ValueError: pass if date_to: try: - base = base.where(Article.published_at <= datetime.fromisoformat(date_to)) + dt = datetime.fromisoformat(date_to.replace("Z", "+00:00")) + base = base.where(Article.published_at <= dt.replace(tzinfo=None)) except ValueError: pass diff --git a/api/vk_parser.py b/api/vk_parser.py index f1b494a..03a7f46 100644 --- a/api/vk_parser.py +++ b/api/vk_parser.py @@ -60,82 +60,60 @@ def _sanitize(html: str) -> str: # ── Теги по ключевым словам ─────────────────────────────────────────────────── # Ключ = название тега, значение = список подстрок (регистронезависимо) TAG_KEYWORDS: dict[str, list[str]] = { - # ── Учёба и поступление ─────────────────────────────────────────────────── - "экзамены": ["экзамен", "зачёт", "зачет", "сессия", "огэ", "егэ", - "гиа", "защит", "диплом", "аттестац", "промежуточн"], - "абитуриентам": ["абитуриент", "поступ", "приёмн", "приемн", "зачислен", - "набор студент", "подай документ", "приглашаем поступ", - "бюджетн мест", "целевой приём", "контрольные цифры"], - "день открытых дверей": ["день открытых дверей", "открытые двери", "день открытых", - "экскурси", "познакомиться с колледж"], - "практика": ["практик", "стажировк", "производственн", "учебно-производ", - "на предприяти", "работодател", "наставник"], - "достижения": ["победител", "призёр", "призер", "награда", "грамот", - "диплом лауреат", "1 место", "2 место", "3 место", - "лучший студент", "гордост", "поздравляем", "медал"], - "студенческая жизнь": ["студсовет", "студенческ жизн", "общежити", "капустник", - "квн", "студенч", "первокурсник", "посвящение", - "студент год", "актив"], - "профессионалитет": ["профессионалитет", "фп профессионалитет", - "кластер", "федеральн проект"], - - # ── Направления колледжей ───────────────────────────────────────────────── - "медицина": ["медицин", "сестринск", "фельдшер", "фармацевт", "лечебн", - "анатоми", "патологи", "здравоохранен", "санитар", - "первая помощ", "реанимац", "клиническ"], - "педагогика": ["педагог", "воспитател", "учител", "начальн класс", - "дошкольн", "детский сад", "логопед", "коррекцион", - "инклюзив", "тьютор"], - "юриспруденция": ["юрист", "юридическ", "правоохранительн", "право", - "законодательств", "суд", "прокурат", "полиц", - "юриспруденц", "правовед"], - "IT и технологии": ["программирован", "it-", "ит-", "информационн технолог", - "цифров", "кибербезопасн", "веб-разраб", "1с", - "компьютерн", "разработчик", "python", "frontend", - "backend", "хакатон", "ворлдскиллс"], - "кулинария и торговля": ["кулинар", "повар", "кондитер", "гастроном", "блюд", - "рецепт", "ресторан", "кафе", "торговл", "продавец", - "мерчандайзинг", "товаровед", "общепит"], - "строительство": ["строительств", "архитектур", "проектирован", "чертёж", - "чертеж", "монтаж", "сварк", "электромонтаж", - "сантехник", "отделочн"], - "техника и механика": ["механик", "двигател", "автомобил", "станок", - "металлообраб", "токар", "слесар", "техническ обслуж", - "ремонт оборудован", "машиностроен", "технолог производ"], - "сельское хозяйство": ["агроном", "агроинженер", "сельск хоз", "животновод", - "агротехник", "землепользован", "агро"], - "экономика и бухучёт": ["бухгалтер", "экономик", "финансов", "налог", - "аудит", "бизнес", "предпринимател", "менеджмент", - "маркетинг", "логистик"], - "социальная работа": ["социальн работ", "соцработник", "психолог", - "реабилитац", "инвалид", "ограниченн возможн", - "социальн помощ", "опека"], - - # ── Общественные события ────────────────────────────────────────────────── - "спорт": ["спорт", "соревнован", "олимпиад", "футбол", "баскетбол", - "волейбол", "чемпион", "кубок", "турнир", "атлет", - "фитнес", "зарядк", "ворлдскиллс хайскул"], - "военка": ["военн", "армия", "призыв", "нво", "сво", "оборон", - "патриот", "защитник", "военно-патриот", "стрельб", - "зарниц", "юнармия", "допризывн"], - "воздушная опасность": ["воздушн тревог", "воздушн опасност", "ракет", - "бпла", "дрон", "обстрел", "укрытие", "сирен", - "эвакуац", "бомбоубежищ"], + "день открытых дверей": ["день открытых дверей", "день открытых", + "приходи в колледж", "познакомиться с колледж", + "двери открыты", "приглашаем на экскурс"], "общественная деятельность": ["волонтёр", "волонтер", "благотвор", "субботник", "экологическ акц", "посадк дерев", "уборк территор", - "донорств", "гуманитарн", "помощ фронт"], - "праздники и события": ["праздник", "концерт", "фестиваль", "выставк", - "торжеств", "церемони", "день колледж", "юбилей", - "8 марта", "23 февраля", "новый год", "масленица", - "день знаний", "последний звонок"], - "конкурсы и олимпиады": ["конкурс", "олимпиад", "чемпионат профессий", - "worldskills", "ворлдскиллс", "хакатон", "викторин", - "интеллектуальн", "акселератор"], + "донорств", "гуманитарн помощ", "добровольч", + "общественн деятельн", "социальн акц", "акция добра"], + "чемпионаты и конкурсы": ["чемпионат", + "олимпиад", + "хакатон", + "всероссийск конкурс", "региональн конкурс", + "областн конкурс", "международн конкурс", + "городск конкурс", "межвузовск конкурс", + "конкурс профессионал", "конкурс мастерств", + "конкурс молодых специалист", + "заняли 1 место", "заняли 2 место", "заняли 3 место", + "1 место среди", "2 место среди", "3 место среди", + "заняли первое место", "заняли второе место", "заняли третье место", + "золото на", "серебро на", "бронзу на", + "завоевали золот", "завоевали серебр", "завоевали бронз"], + "профессионалы": ["профессионалитет", "worldskills", "ворлдскиллс", + "ворлдскиллс хайскул", "хайскул", + "компетенц", "демонстрацион экзамен", + "федеральн проект профессионалите"], + "абилимпикс": ["абилимпикс", "abilympics", + "ограниченн возможн здоровь", + "особенн образовател потребн", + "инклюзивн образован"], + "экзамены": ["расписани экзамен", "подготовк к экзамен", + "экзаменацион сессия", "зачётн неделя", "зачетн неделя", + "промежуточн аттестац", "государственн итог аттестац", + "государственн экзамен", "итоговая аттестац", + "защит диплом", "защита выпускн", + "огэ", "егэ", "гиа"], } +# Слова-исключения: если они есть в тексте — пост скипается полностью +# (посты о ВОВ/Победе не относятся ни к одной из 6 категорий) +_SKIP_KEYWORDS = [ + "великая отечественная война", + "великой отечественной войн", + "день победы", + "9 мая", + "вов ", + "ветеран войн", + "павш за родин", +] + def _detect_tags(text: str) -> list[str]: lower = text.lower() + # Пропускаем посты о ВОВ/Победе — они не попадают ни в одну категорию + if any(kw in lower for kw in _SKIP_KEYWORDS): + return [] found = [] for tag_name, keywords in TAG_KEYWORDS.items(): if any(kw in lower for kw in keywords): @@ -163,7 +141,9 @@ def _ensure_bucket(mc: Minio): pass -async def _upload_from_url(url: str, db: AsyncSession | None = None) -> str | None: +async def _upload_from_url(url: str, db: AsyncSession | None = None, + article_dir: str | None = None, + index: int = 1) -> str | None: try: async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: r = await client.get(url) @@ -172,7 +152,13 @@ async def _upload_from_url(url: str, db: AsyncSession | None = None) -> str | No data = r.content ct = r.headers.get("content-type", "image/jpeg").split(";")[0].strip() ext = ct.split("/")[-1].replace("jpeg", "jpg") - object_name = f"vk/images/{uuid.uuid4().hex}.{ext}" + if article_dir: + prefix = f"articles/{article_dir}/images" + filename = f"{article_dir}-{index}.{ext}" + else: + prefix = "vk/images" + filename = f"{uuid.uuid4().hex}.{ext}" + object_name = f"{prefix}/{filename}" mc = _minio_client() _ensure_bucket(mc) mc.put_object(MINIO_BUCKET, object_name, io.BytesIO(data), len(data), content_type=ct) @@ -180,10 +166,11 @@ async def _upload_from_url(url: str, db: AsyncSession | None = None) -> str | No if db is not None: from bd.models import Media db.add(Media( - filename=object_name.split("/")[-1], + filename=filename, url=public_url, media_type="image", size_bytes=len(data), + # article_id выставляется после коммита статьи (см. _process_post) )) return public_url except Exception as exc: @@ -257,7 +244,9 @@ def _ytdlp_extract_url(vk_url: str, max_mb: int = 150) -> str | None: async def _download_vk_video(owner_id: int, video_id: int, db: AsyncSession | None = None, - max_mb: int = 150) -> str | None: + max_mb: int = 150, + article_dir: str | None = None, + index: int = 1) -> str | None: """Скачать VK-видео в MinIO и вернуть публичный URL. Использует yt-dlp для получения прямой mp4-ссылки. Стриминг — не держит весь файл в памяти. @@ -303,7 +292,13 @@ async def _download_vk_video(owner_id: int, video_id: int, return None chunks.append(chunk) data = b"".join(chunks) - object_name = f"vk/videos/{uuid.uuid4().hex}.mp4" + if article_dir: + prefix = f"articles/{article_dir}/videos" + filename = f"{article_dir}-{index}.mp4" + else: + prefix = "vk/videos" + filename = f"{uuid.uuid4().hex}.mp4" + object_name = f"{prefix}/{filename}" mc = _minio_client() _ensure_bucket(mc) mc.put_object(MINIO_BUCKET, object_name, io.BytesIO(data), len(data), content_type="video/mp4") @@ -312,10 +307,11 @@ async def _download_vk_video(owner_id: int, video_id: int, if db is not None: from bd.models import Media db.add(Media( - filename=object_name.split("/")[-1], + filename=filename, url=public_url, media_type="video", size_bytes=len(data), + # article_id выставляется после коммита статьи (см. _process_post) )) return public_url except Exception as exc: @@ -397,6 +393,14 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession) -> boo attachments = post.get("attachments", []) ts = post["date"] + # Фильтр: импортируем только посты с разрешёнными тегами + tag_names = _detect_tags(text) + if not tag_names: + return False + + # Заранее генерируем UUID и slug — используются в путях MinIO + article_id = uuid.uuid4() + # VK Clips и некоторые видео-посты: post.text пустой, описание лежит в video.description if not text: for att in attachments: @@ -406,6 +410,15 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession) -> boo text = desc break + # Генерируем slug до загрузки медиа: используем его как имя директории и файлов + _early_title = _extract_title(text, ts) + _early_slug = (slugify(_early_title) or vk_slug)[:60] # макс 60 символов + # Уникальный префикс: slug + 8 символов UUID (защита от коллизий одинаковых заголовков) + article_dir = f"{_early_slug}-{str(article_id)[:8]}" + + img_idx = 0 # счётчик изображений для нумерации файлов + vid_idx = 0 # счётчик видео + cover_url = None content_parts = [] @@ -418,7 +431,8 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession) -> boo if att_type == "photo": url = _best_photo(att["photo"].get("sizes", [])) if url: - minio_url = await _upload_from_url(url, db=db) + img_idx += 1 + minio_url = await _upload_from_url(url, db=db, article_dir=article_dir, index=img_idx) final_url = minio_url or url # fallback на оригинальный URL если MinIO недоступен if cover_url is None: cover_url = final_url @@ -456,7 +470,8 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession) -> boo # Всегда скачиваем превью в MinIO minio_thumb = None if thumb_url: - minio_thumb = await _upload_from_url(thumb_url, db=db) + img_idx += 1 + minio_thumb = await _upload_from_url(thumb_url, db=db, article_dir=article_dir, index=img_idx) final_thumb = minio_thumb or thumb_url if cover_url is None: cover_url = final_thumb @@ -466,7 +481,8 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession) -> boo # Скачиваем само видео через yt-dlp → MinIO minio_video_url = None if vid_id and vid_oid: - minio_video_url = await _download_vk_video(vid_oid, vid_id, db=db) + vid_idx += 1 + minio_video_url = await _download_vk_video(vid_oid, vid_id, db=db, article_dir=article_dir, index=vid_idx) if minio_video_url: # 1. Видео лежит у нас — нативный