jze9/news_all_spo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: корректные source_url для VK постов (screen_name формат)

Browse Source 
- VkSource: добавлено поле screen_name, заполняется из VK API при добавлении источника
- vk_parser: source_url теперь в формате vk.com/{screen_name}?w=wall{owner}_{post}
  вместо устаревшего vk.com/wall{owner}_{post}
- _vk_post_exists: проверка через LIKE чтобы находить оба формата URL
- admin_vk: автоматически получает screen_name при создании/обновлении источника
- admin_system: исправлен regex в repair-media-urls — только /news-media/ URL
- Alembic миграция: b2c3d4e5f6a7 добавляет колонку screen_name в vk_sources

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jze9
2026-05-20 19:32:28 +05:00
parent 9e277de65d
commit 615c819e97
5 changed files with 62 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/bd/models.py
View File

@@ -82,6 +82,7 @@ class VkSource(Base):
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
group_id: Mapped[str] = mapped_column(String(50), unique=True) # числовой ID без минуса
group_name: Mapped[str] = mapped_column(String(200)) # станет названием категории
screen_name: Mapped[str | None] = mapped_column(String(200), nullable=True) # screen_name для корректных URL
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
strict_filter: Mapped[bool] = mapped_column(Boolean, default=True) # True = только посты с тегами
last_run: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)

26
api/migrations/versions/b2c3d4e5f6a7_add_screen_name_to_vk_sources.py Normal file
View File

@@ -0,0 +1,26 @@
"""add screen_name to vk_sources
Revision ID: b2c3d4e5f6a7
Revises: a1b2c3d4e5f6
Create Date: 2026-05-20 18:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = 'b2c3d4e5f6a7'
down_revision: Union[str, None] = 'a1b2c3d4e5f6'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column('vk_sources',
sa.Column('screen_name', sa.String(200), nullable=True)
)
def downgrade() -> None:
op.drop_column('vk_sources', 'screen_name')

5
api/route/admin_system.py
View File

@@ -138,12 +138,13 @@ async def repair_media_urls(db: AsyncSession = Depends(get_db), _: str = Depends
RETURNING id
"""), {"new_base": new_base, "prefix": new_base + "%"})
# HTML-контент: src="http://host/news-media/path" → src="new_base/path"
# HTML-контент: только MinIO URL вида src="http://host/news-media/path"
# Точный паттерн: URL содержит /news-media/ — не трогаем другие ссылки
r_content = await db.execute(text(r"""
UPDATE articles
SET content = regexp_replace(
content,
'https?://[^/]+(:[0-9]+)?/[^/]+/([^"'' ]+)',
'https?://[^/''"]+(:[0-9]+)?/news-media/([^''">]+)',
:new_base || '/\2',
'g'
)

25
api/route/admin_vk.py
View File

@@ -1,3 +1,4 @@
import httpx
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException # BackgroundTasks используется в history endpoints
from pydantic import BaseModel
from sqlalchemy import select
@@ -18,11 +19,30 @@ class SourceIn(BaseModel):
strict_filter: bool = True
async def _fetch_screen_name(group_id: str) -> str | None:
if not VK_TOKEN:
return None
try:
async with httpx.AsyncClient(timeout=5) as client:
r = await client.get(
"https://api.vk.com/method/groups.getById",
params={"group_ids": group_id, "access_token": VK_TOKEN, "v": "5.131"},
)
data = r.json()
groups = data.get("response", [])
if groups:
return groups[0].get("screen_name")
except Exception:
pass
return None
def _source_dict(s: VkSource) -> dict:
return {
"id": str(s.id),
"group_id": s.group_id,
"group_name": s.group_name,
"screen_name": s.screen_name,
"enabled": s.enabled,
"strict_filter": s.strict_filter,
"last_run": s.last_run.isoformat() if s.last_run else None,
@@ -58,7 +78,8 @@ async def add_source(data: SourceIn, db: AsyncSession = Depends(get_db), _: str
exists = (await db.execute(select(VkSource).where(VkSource.group_id == group_id))).scalar_one_or_none()
if exists:
raise HTTPException(status_code=409, detail="Источник с таким group_id уже есть")
source = VkSource(group_id=group_id, group_name=data.group_name, enabled=data.enabled, strict_filter=data.strict_filter)
screen_name = await _fetch_screen_name(group_id)
source = VkSource(group_id=group_id, group_name=data.group_name, screen_name=screen_name, enabled=data.enabled, strict_filter=data.strict_filter)
db.add(source)
await db.commit()
await db.refresh(source)
@@ -74,6 +95,8 @@ async def update_source(source_id: str, data: SourceIn, db: AsyncSession = Depen
source.group_name = data.group_name
source.enabled = data.enabled
source.strict_filter = data.strict_filter
if not source.screen_name:
source.screen_name = await _fetch_screen_name(source.group_id)
await db.commit()
await db.refresh(source)
return _source_dict(source)

11
api/vk_parser.py
View File

@@ -361,8 +361,8 @@ async def _slug_exists(slug: str, db: AsyncSession) -> bool:
async def _vk_post_exists(owner_id: int, post_id: int, db: AsyncSession) -> bool:
source_url = f"https://vk.com/wall{owner_id}_{post_id}"
r = await db.execute(select(Article.id).where(Article.source_url == source_url))
pattern = f"%wall{owner_id}_{post_id}%"
r = await db.execute(select(Article.id).where(Article.source_url.like(pattern)))
return r.scalar_one_or_none() is not None
@@ -388,7 +388,7 @@ async def _get_or_create_tag(name: str, db: AsyncSession) -> Tag:
# ── Post processing ───────────────────────────────────────────────────────────
async def _process_post(post: dict, category_name: str, db: AsyncSession, strict_filter: bool = True) -> bool:
async def _process_post(post: dict, category_name: str, db: AsyncSession, strict_filter: bool = True, screen_name: str | None = None) -> bool:
owner_id = post["owner_id"]
post_id = post["id"]
vk_slug = _vk_slug(owner_id, post_id)
@@ -562,6 +562,9 @@ async def _process_post(post: dict, category_name: str, db: AsyncSession, strict
created_at = datetime.utcfromtimestamp(ts)
published_at = created_at if status == ArticleStatus.published else None
if screen_name:
source_url = f"https://vk.com/{screen_name}?w=wall{owner_id}_{post_id}"
else:
source_url = f"https://vk.com/wall{owner_id}_{post_id}"
article = Article(
@@ -673,7 +676,7 @@ async def run_import() -> dict:
batch_new = 0
for post in posts:
try:
ok = await _process_post(post, source.group_name, db, source.strict_filter)
ok = await _process_post(post, source.group_name, db, source.strict_filter, screen_name=source.screen_name)
if ok:
imported += 1; grp_new += 1; batch_new += 1
else: