fix: корректные source_url для VK постов (screen_name формат)
- VkSource: добавлено поле screen_name, заполняется из VK API при добавлении источника
- vk_parser: source_url теперь в формате vk.com/{screen_name}?w=wall{owner}_{post}
вместо устаревшего vk.com/wall{owner}_{post}
- _vk_post_exists: проверка через LIKE чтобы находить оба формата URL
- admin_vk: автоматически получает screen_name при создании/обновлении источника
- admin_system: исправлен regex в repair-media-urls — только /news-media/ URL
- Alembic миграция: b2c3d4e5f6a7 добавляет колонку screen_name в vk_sources
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,7 @@ class VkSource(Base):
|
|||||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
group_id: Mapped[str] = mapped_column(String(50), unique=True) # числовой ID без минуса
|
group_id: Mapped[str] = mapped_column(String(50), unique=True) # числовой ID без минуса
|
||||||
group_name: Mapped[str] = mapped_column(String(200)) # станет названием категории
|
group_name: Mapped[str] = mapped_column(String(200)) # станет названием категории
|
||||||
|
screen_name: Mapped[str | None] = mapped_column(String(200), nullable=True) # screen_name для корректных URL
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||||
strict_filter: Mapped[bool] = mapped_column(Boolean, default=True) # True = только посты с тегами
|
strict_filter: Mapped[bool] = mapped_column(Boolean, default=True) # True = только посты с тегами
|
||||||
last_run: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
last_run: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""add screen_name to vk_sources
|
||||||
|
|
||||||
|
Revision ID: b2c3d4e5f6a7
|
||||||
|
Revises: a1b2c3d4e5f6
|
||||||
|
Create Date: 2026-05-20 18:00:00.000000
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision: str = 'b2c3d4e5f6a7'
|
||||||
|
down_revision: Union[str, None] = 'a1b2c3d4e5f6'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column('vk_sources',
|
||||||
|
sa.Column('screen_name', sa.String(200), nullable=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column('vk_sources', 'screen_name')
|
||||||
@@ -138,12 +138,13 @@ async def repair_media_urls(db: AsyncSession = Depends(get_db), _: str = Depends
|
|||||||
RETURNING id
|
RETURNING id
|
||||||
"""), {"new_base": new_base, "prefix": new_base + "%"})
|
"""), {"new_base": new_base, "prefix": new_base + "%"})
|
||||||
|
|
||||||
# HTML-контент: src="http://host/news-media/path" → src="new_base/path"
|
# HTML-контент: только MinIO URL вида src="http://host/news-media/path"
|
||||||
|
# Точный паттерн: URL содержит /news-media/ — не трогаем другие ссылки
|
||||||
r_content = await db.execute(text(r"""
|
r_content = await db.execute(text(r"""
|
||||||
UPDATE articles
|
UPDATE articles
|
||||||
SET content = regexp_replace(
|
SET content = regexp_replace(
|
||||||
content,
|
content,
|
||||||
'https?://[^/]+(:[0-9]+)?/[^/]+/([^"'' ]+)',
|
'https?://[^/''"]+(:[0-9]+)?/news-media/([^''">]+)',
|
||||||
:new_base || '/\2',
|
:new_base || '/\2',
|
||||||
'g'
|
'g'
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import httpx
|
||||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException # BackgroundTasks используется в history endpoints
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException # BackgroundTasks используется в history endpoints
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -18,11 +19,30 @@ class SourceIn(BaseModel):
|
|||||||
strict_filter: bool = True
|
strict_filter: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_screen_name(group_id: str) -> str | None:
|
||||||
|
if not VK_TOKEN:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=5) as client:
|
||||||
|
r = await client.get(
|
||||||
|
"https://api.vk.com/method/groups.getById",
|
||||||
|
params={"group_ids": group_id, "access_token": VK_TOKEN, "v": "5.131"},
|
||||||
|
)
|
||||||
|
data = r.json()
|
||||||
|
groups = data.get("response", [])
|
||||||
|
if groups:
|
||||||
|
return groups[0].get("screen_name")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _source_dict(s: VkSource) -> dict:
|
def _source_dict(s: VkSource) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": str(s.id),
|
"id": str(s.id),
|
||||||
"group_id": s.group_id,
|
"group_id": s.group_id,
|
||||||
"group_name": s.group_name,
|
"group_name": s.group_name,
|
||||||
|
"screen_name": s.screen_name,
|
||||||
"enabled": s.enabled,
|
"enabled": s.enabled,
|
||||||
"strict_filter": s.strict_filter,
|
"strict_filter": s.strict_filter,
|
||||||
"last_run": s.last_run.isoformat() if s.last_run else None,
|
"last_run": s.last_run.isoformat() if s.last_run else None,
|
||||||
@@ -58,7 +78,8 @@ async def add_source(data: SourceIn, db: AsyncSession = Depends(get_db), _: str
|
|||||||
exists = (await db.execute(select(VkSource).where(VkSource.group_id == group_id))).scalar_one_or_none()
|
exists = (await db.execute(select(VkSource).where(VkSource.group_id == group_id))).scalar_one_or_none()
|
||||||
if exists:
|
if exists:
|
||||||
raise HTTPException(status_code=409, detail="Источник с таким group_id уже есть")
|
raise HTTPException(status_code=409, detail="Источник с таким group_id уже есть")
|
||||||
source = VkSource(group_id=group_id, group_name=data.group_name, enabled=data.enabled, strict_filter=data.strict_filter)
|
screen_name = await _fetch_screen_name(group_id)
|
||||||
|
source = VkSource(group_id=group_id, group_name=data.group_name, screen_name=screen_name, enabled=data.enabled, strict_filter=data.strict_filter)
|
||||||
db.add(source)
|
db.add(source)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await db.refresh(source)
|
await db.refresh(source)
|
||||||
@@ -74,6 +95,8 @@ async def update_source(source_id: str, data: SourceIn, db: AsyncSession = Depen
|
|||||||
source.group_name = data.group_name
|
source.group_name = data.group_name
|
||||||
source.enabled = data.enabled
|
source.enabled = data.enabled
|
||||||
source.strict_filter = data.strict_filter
|
source.strict_filter = data.strict_filter
|
||||||
|
if not source.screen_name:
|
||||||
|
source.screen_name = await _fetch_screen_name(source.group_id)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await db.refresh(source)
|
await db.refresh(source)
|
||||||
return _source_dict(source)
|
return _source_dict(source)
|
||||||
|
|||||||
@@ -361,8 +361,8 @@ async def _slug_exists(slug: str, db: AsyncSession) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
async def _vk_post_exists(owner_id: int, post_id: int, db: AsyncSession) -> bool:
|
async def _vk_post_exists(owner_id: int, post_id: int, db: AsyncSession) -> bool:
|
||||||
source_url = f"https://vk.com/wall{owner_id}_{post_id}"
|
pattern = f"%wall{owner_id}_{post_id}%"
|
||||||
r = await db.execute(select(Article.id).where(Article.source_url == source_url))
|
r = await db.execute(select(Article.id).where(Article.source_url.like(pattern)))
|
||||||
return r.scalar_one_or_none() is not None
|
return r.scalar_one_or_none() is not None
|
||||||
|
|
||||||
|
|
||||||
@@ -388,7 +388,7 @@ async def _get_or_create_tag(name: str, db: AsyncSession) -> Tag:
|
|||||||
|
|
||||||
# ── Post processing ───────────────────────────────────────────────────────────
|
# ── Post processing ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async def _process_post(post: dict, category_name: str, db: AsyncSession, strict_filter: bool = True) -> bool:
|
async def _process_post(post: dict, category_name: str, db: AsyncSession, strict_filter: bool = True, screen_name: str | None = None) -> bool:
|
||||||
owner_id = post["owner_id"]
|
owner_id = post["owner_id"]
|
||||||
post_id = post["id"]
|
post_id = post["id"]
|
||||||
vk_slug = _vk_slug(owner_id, post_id)
|
vk_slug = _vk_slug(owner_id, post_id)
|
||||||
@@ -562,7 +562,10 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession, strict
|
|||||||
created_at = datetime.utcfromtimestamp(ts)
|
created_at = datetime.utcfromtimestamp(ts)
|
||||||
published_at = created_at if status == ArticleStatus.published else None
|
published_at = created_at if status == ArticleStatus.published else None
|
||||||
|
|
||||||
source_url = f"https://vk.com/wall{owner_id}_{post_id}"
|
if screen_name:
|
||||||
|
source_url = f"https://vk.com/{screen_name}?w=wall{owner_id}_{post_id}"
|
||||||
|
else:
|
||||||
|
source_url = f"https://vk.com/wall{owner_id}_{post_id}"
|
||||||
|
|
||||||
article = Article(
|
article = Article(
|
||||||
id=article_id,
|
id=article_id,
|
||||||
@@ -673,7 +676,7 @@ async def run_import() -> dict:
|
|||||||
batch_new = 0
|
batch_new = 0
|
||||||
for post in posts:
|
for post in posts:
|
||||||
try:
|
try:
|
||||||
ok = await _process_post(post, source.group_name, db, source.strict_filter)
|
ok = await _process_post(post, source.group_name, db, source.strict_filter, screen_name=source.screen_name)
|
||||||
if ok:
|
if ok:
|
||||||
imported += 1; grp_new += 1; batch_new += 1
|
imported += 1; grp_new += 1; batch_new += 1
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user