infinity serch

api/vk_parser.py

@@ -359,6 +359,12 @@ async def _slug_exists(slug: str, db: AsyncSession) -> bool:
     return r.scalar_one_or_none() is not None
 
 
+async def _vk_post_exists(owner_id: int, post_id: int, db: AsyncSession) -> bool:
+    source_url = f"https://vk.com/wall{owner_id}_{post_id}"
+    r = await db.execute(select(Article.id).where(Article.source_url == source_url))
+    return r.scalar_one_or_none() is not None
+
+
 async def _get_or_create_category(name: str, db: AsyncSession) -> Category:
     s = slugify(name)
     cat = (await db.execute(select(Category).where(Category.slug == s))).scalar_one_or_none()
@@ -381,21 +387,39 @@ async def _get_or_create_tag(name: str, db: AsyncSession) -> Tag:
 
 # ── Post processing ───────────────────────────────────────────────────────────
 
-async def _process_post(post: dict, category_name: str, db: AsyncSession) -> bool:
+async def _process_post(post: dict, category_name: str, db: AsyncSession, strict_filter: bool = True) -> bool:
     owner_id = post["owner_id"]
     post_id  = post["id"]
     vk_slug  = _vk_slug(owner_id, post_id)
 
-    if await _slug_exists(vk_slug, db):
+    if await _vk_post_exists(owner_id, post_id, db):
         return False
 
     text        = post.get("text", "")
     attachments = post.get("attachments", [])
     ts          = post["date"]
 
-    # Фильтр: импортируем только посты с разрешёнными тегами
+    # Обработка репостов (copy_history)
+    copy_history = post.get("copy_history", [])
+    if copy_history:
+        original     = copy_history[0]
+        orig_gid     = str(abs(original.get("owner_id", 0)))
+        in_db = (await db.execute(
+            select(VkSource.id).where(VkSource.group_id == orig_gid)
+        )).scalar_one_or_none()
+        if in_db:
+            # Оригинал придёт сам из своей группы — пропускаем
+            return False
+        # Берём содержимое оригинала; комментарий репостера добавляем в начало
+        orig_text  = original.get("text", "")
+        orig_atts  = original.get("attachments", [])
+        text       = (text + "\n" + orig_text).strip() if text else orig_text
+        attachments = orig_atts if orig_atts else attachments
+
     tag_names = _detect_tags(text)
-    if not tag_names:
+    # strict_filter=True: пропускаем пост если нет тегов
+    # strict_filter=False: импортируем всё, теги назначаем если найдены
+    if strict_filter and not tag_names:
         return False
 
     # Заранее генерируем UUID и slug — используются в путях MinIO
@@ -648,7 +672,7 @@ async def run_import() -> dict:
                 batch_new = 0
                 for post in posts:
                     try:
-                        ok = await _process_post(post, source.group_name, db)
+                        ok = await _process_post(post, source.group_name, db, source.strict_filter)
                         if ok:
                             imported += 1; grp_new += 1; batch_new += 1
                         else:
@@ -721,7 +745,7 @@ async def run_history_import(group_id: str | None = None, since_days: int = 365)
                         hit_cutoff = True
                         continue
                     try:
-                        ok = await _process_post(post, source.group_name, db)
+                        ok = await _process_post(post, source.group_name, db, source.strict_filter)
                         if ok:
                             imported += 1; grp_new += 1; batch_new += 1
                         else: