This commit is contained in:
2026-04-29 16:14:39 +05:00
commit 02d52b2850
27 changed files with 65576 additions and 0 deletions

581
main.py Normal file
View File

@@ -0,0 +1,581 @@
#!/usr/bin/env python3
"""ФИАС: поиск кадастровых номеров и скачивание выписок ГАР."""
from __future__ import annotations # str | None etc. on Python 3.8
import os
import re
import sys
import time
import queue
import shutil
import logging
import platform
import threading
import requests
from datetime import datetime
from pathlib import Path
import tkinter as tk
from tkinter import filedialog, scrolledtext, messagebox
import openpyxl
import pdfplumber
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options as FirefoxOptions
from selenium.webdriver.firefox.service import Service as FirefoxService
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, WebDriverException
from webdriver_manager.firefox import GeckoDriverManager
FIAS_URL = "https://fias.nalog.ru/Search/Extended"
_CRYPTOPRO_ID = "ru.cryptopro.nmcades@cryptopro.ru"
# ---------------------------------------------------------------------------
# КриптоПро XPI finder
# ---------------------------------------------------------------------------
def _find_cryptopro_xpi() -> str | None:
candidates: list[Path] = []
system = platform.system()
if system == "Windows":
appdata = Path(os.environ.get("APPDATA", ""))
localappdata = Path(os.environ.get("LOCALAPPDATA", ""))
# Zen Browser (Windows)
for base in (appdata / "Zen" / "Profiles", localappdata / "Zen" / "Profiles"):
if base.is_dir():
candidates.extend(base.glob(f"*/extensions/{_CRYPTOPRO_ID}.xpi"))
# Firefox (Windows)
for base in (appdata / "Mozilla" / "Firefox" / "Profiles",):
if base.is_dir():
candidates.extend(base.glob(f"*/extensions/{_CRYPTOPRO_ID}.xpi"))
else:
# Zen Browser (Linux Flatpak)
for d in Path.home().glob(".var/app/app.zen_browser.zen/.zen/*/extensions/"):
p = d / f"{_CRYPTOPRO_ID}.xpi"
if p.exists():
candidates.append(p)
# Firefox (Linux / snap)
for base in (
Path.home() / ".mozilla" / "firefox",
Path.home() / "snap" / "firefox" / "common" / ".mozilla" / "firefox",
):
if base.is_dir():
candidates.extend(base.glob(f"*/extensions/{_CRYPTOPRO_ID}.xpi"))
# Downloads folder — works on both platforms
for folder in ("Downloads", "Загрузки"):
d = Path.home() / folder
if d.is_dir():
candidates.extend(d.glob("*.xpi"))
exact = [p for p in candidates if _CRYPTOPRO_ID in p.name]
if exact:
return str(max(exact, key=lambda p: p.stat().st_mtime))
fuzzy = [p for p in candidates if any(k in p.name.lower() for k in ("cryptopro", "cades"))]
if fuzzy:
return str(max(fuzzy, key=lambda p: p.stat().st_mtime))
return None
# ---------------------------------------------------------------------------
# Browser automation
# ---------------------------------------------------------------------------
class FIASSearcher:
def __init__(self, log=print):
self.log = log
self.driver: webdriver.Firefox | None = None
self.wait: WebDriverWait | None = None
def setup(self):
opts = FirefoxOptions()
opts.set_preference("pdfjs.disabled", True)
opts.set_preference("browser.download.manager.showWhenStarting", False)
opts.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/pdf")
try:
service = FirefoxService(GeckoDriverManager().install())
self.driver = webdriver.Firefox(service=service, options=opts)
except Exception:
self.driver = webdriver.Firefox(options=opts)
self.wait = WebDriverWait(self.driver, 20)
self.driver.get(FIAS_URL)
if self._popup_visible():
self._try_install_cryptopro()
self.driver.refresh()
time.sleep(2)
self._dismiss_popup()
self.wait.until(EC.element_to_be_clickable((By.ID, "ByIdentifiersTab"))).click()
time.sleep(1.5)
self.log("Браузер запущен, открыта вкладка «По уникальным идентификаторам»")
def _popup_visible(self) -> bool:
try:
WebDriverWait(self.driver, 3).until(
EC.presence_of_element_located((
By.XPATH,
"//button[contains(@class,'close') or @aria-label='Close' or normalize-space()='×']",
))
)
return True
except TimeoutException:
return False
def _try_install_cryptopro(self):
xpi = _find_cryptopro_xpi()
if not xpi:
self.log(" XPI КриптоПро не найден, попап будет закрыт кнопкой")
return
try:
self.log(f" Установка расширения: {Path(xpi).name}")
self.driver.install_addon(xpi, temporary=True)
time.sleep(2)
self.log(" КриптоПро установлен")
except Exception as e:
self.log(f" Не удалось установить расширение: {e}")
def _dismiss_popup(self):
try:
btn = WebDriverWait(self.driver, 3).until(
EC.element_to_be_clickable((
By.XPATH,
"//button[contains(@class,'close') or @aria-label='Close' or normalize-space()='×']",
))
)
btn.click()
time.sleep(0.3)
except (TimeoutException, WebDriverException):
pass
def search(self, kn: str) -> dict | None:
self._dismiss_popup()
self._clear_form()
# Wait until the previous results panel is fully hidden before proceeding.
# Without this, the old SearchResultDiv is still visible and the next
# wait.until(visibility_of...) returns immediately with stale data.
try:
WebDriverWait(self.driver, 10).until(
EC.invisibility_of_element_located((By.ID, "SearchResultDiv"))
)
except TimeoutException:
pass # First search — div never appeared, that's fine
field = self.wait.until(EC.element_to_be_clickable((By.ID, "CadastralNumberInput")))
field.clear()
field.send_keys(kn)
self.driver.find_element(
By.XPATH, "//button[contains(@class,'btn_primary')][@title='Найти']"
).click()
try:
self.wait.until(EC.visibility_of_element_located((By.ID, "SearchResultDiv")))
except TimeoutException:
self.log(f" Таймаут ожидания результатов")
return None
time.sleep(0.5)
rows = self.driver.find_elements(By.CSS_SELECTOR, "#ResultPagerDataPlaceholder tr")
if not rows:
return None
cells = rows[0].find_elements(By.TAG_NAME, "td")
if len(cells) < 6:
self.log(f" Неожиданное число столбцов: {len(cells)}")
return None
data = {
"full_name": cells[0].text.strip(),
"status": cells[1].text.strip(),
"obj_type": cells[2].text.strip(),
"fias_id": cells[5].text.strip(),
"pdf_url": None,
}
self.log(f" Статус: {data['status']} | Тип: {data['obj_type']} | ID: {data['fias_id']}")
try:
row_html = (rows[0].get_attribute("innerHTML") or "").replace("&amp;", "&")
m = re.search(r"(/Export/ExportPdfStatement\?[^'\"]+)", row_html)
if m:
data["pdf_url"] = "https://fias.nalog.ru" + m.group(1).rstrip("'\"")
self.log(" PDF URL найден")
else:
self.log(" PDF URL не найден в HTML")
except Exception as e:
self.log(f" Ошибка извлечения PDF URL: {e}")
return data
def download_pdf(self, pdf_url: str, save_path: str) -> bool:
try:
session = requests.Session()
for c in self.driver.get_cookies():
session.cookies.set(c["name"], c["value"])
session.headers.update({
"User-Agent": self.driver.execute_script("return navigator.userAgent;"),
"Referer": FIAS_URL,
"Accept": "application/pdf,*/*",
})
resp = session.get(pdf_url, timeout=40)
resp.raise_for_status()
content = resp.content
if content[:4] != b"%PDF":
self.log(f" Ответ не PDF: Content-Type={resp.headers.get('Content-Type','?')}")
return False
with open(save_path, "wb") as f:
f.write(content)
self.log(f" Скачано {len(content):,} байт")
return True
except Exception as e:
self.log(f" Ошибка загрузки PDF: {e}")
return False
def _clear_form(self):
try:
self.driver.find_element(
By.XPATH,
"//button[@title='Очистить форму'] | //button[normalize-space()='Очистить форму']",
).click()
except Exception:
pass
def close(self):
if self.driver:
try:
self.driver.quit()
except Exception:
pass
self.driver = None
# ---------------------------------------------------------------------------
# PDF parsing
# ---------------------------------------------------------------------------
def parse_pdf_extract(pdf_path: str, log=print) -> tuple[str | None, str | None]:
try:
with pdfplumber.open(pdf_path) as pdf:
text = "\n".join(p.extract_text() or "" for p in pdf.pages)
except Exception as e:
log(f" Ошибка чтения PDF: {e}")
return None, None
log(f" PDF (начало): {text[:200].replace(chr(10),' ')}")
date = None
m = re.search(
r"(?:дат[аеу]|выписк[аи]|формирован)[^\n\d]{0,30}(\d{2}\.\d{2}\.\d{4})",
text, re.IGNORECASE,
)
if not m:
m = re.search(r"\b(\d{2}\.\d{2}\.\d{4})\b", text)
if m:
date = m.group(1)
uuid_pat = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
number = None
m = re.search(r"(?:номер|идентификатор)[^\n]{0,40}(" + uuid_pat + ")", text, re.IGNORECASE)
if not m:
m = re.search(uuid_pat, text)
if m:
number = m.group(1) if m.lastindex else m.group(0)
return date, number
# ---------------------------------------------------------------------------
# File helpers
# ---------------------------------------------------------------------------
def rename_extract(pdf_path: str, kn: str, date_str: str | None, output_dir: str) -> str:
kn_safe = kn.replace(":", "_")
date_part = date_str or datetime.now().strftime("%d.%m.%Y")
dest = os.path.join(output_dir, f"ГАР_{kn_safe}_{date_part}.pdf")
n = 1
while os.path.exists(dest):
dest = os.path.join(output_dir, f"ГАР_{kn_safe}_{date_part}_{n}.pdf")
n += 1
shutil.move(pdf_path, dest)
return dest
# ---------------------------------------------------------------------------
# Main processing
# ---------------------------------------------------------------------------
def process_all(input_xlsx: str, output_dir: str, log=print) -> None:
# ---- Session directory: named by start time ----
session_ts = datetime.now().strftime("%d.%m.%Y_%H-%M")
session_dir = os.path.join(output_dir, session_ts)
os.makedirs(session_dir, exist_ok=True)
# File logger inside session dir — survives window close
log_file = os.path.join(session_dir, "fias_log.txt")
file_handler = logging.FileHandler(log_file, encoding="utf-8")
file_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s", "%H:%M:%S"))
file_logger = logging.getLogger("fias")
file_logger.setLevel(logging.DEBUG)
file_logger.handlers.clear()
file_logger.addHandler(file_handler)
def logged(msg: str):
log(msg)
file_logger.info(msg)
logged(f"Папка сессии: {session_dir}")
# ---- Load input ----
wb_in = openpyxl.load_workbook(input_xlsx)
ws_in = wb_in.active
kn_col = vid_col = None
for row_vals in ws_in.iter_rows(min_row=1, max_row=5, values_only=True):
for i, cell in enumerate(row_vals):
s = str(cell or "").strip().lower()
if s in ("кн", "кадастровый номер"):
kn_col = i
if s in ("вид он", "вид объекта", "вид"):
vid_col = i
if kn_col is not None:
break
if kn_col is None:
logged("ОШИБКА: колонка «КН» не найдена")
return
entries = [
(str(row[kn_col]).strip(), row[vid_col] if vid_col is not None else None)
for row in ws_in.iter_rows(min_row=2, values_only=True)
if row[kn_col] and str(row[kn_col]).strip()
]
logged(f"Загружено КН: {len(entries)}")
# ---- Output workbook — named with session timestamp ----
output_xlsx_path = os.path.join(session_dir, f"Результаты_ФИАС_{session_ts}.xlsx")
wb_out = openpyxl.Workbook()
ws_out = wb_out.active
ws_out.title = "Результаты"
ws_out.append([
"КН", "Вид ОН",
"Уникальный номер в ГАР (ID FIAS)", "Статус записи",
"Наименование (адрес)", "Тип",
"Дата выписки ГАР", "Номер выписки ГАР",
])
ws_out.freeze_panes = "A2"
done_kns: set[str] = set()
# ---- Run automation ----
searcher = FIASSearcher(log=logged)
try:
searcher.setup()
total = len(entries)
for idx, (kn, vid_on) in enumerate(entries):
if kn in done_kns:
continue
logged(f"\n[{idx + 1}/{total}] {kn}")
# Per-KN try/except — one bad KN doesn't kill the run
try:
data = searcher.search(kn)
except WebDriverException as e:
logged(f" WebDriver ошибка: {e} — пропуск КН")
ws_out.append([kn, vid_on,
"ошибка", "ошибка", str(e)[:100], "", None, None])
wb_out.save(output_xlsx_path)
continue
except Exception as e:
logged(f" Непредвиденная ошибка: {e} — пропуск КН")
ws_out.append([kn, vid_on,
"ошибка", "ошибка", str(e)[:100], "", None, None])
wb_out.save(output_xlsx_path)
continue
if data is None:
logged(" → не найден в ФИАС")
ws_out.append([kn, vid_on,
"отсутствует", "отсутствует",
"отсутствует", "отсутствует",
None, None])
else:
logged(f"{data['full_name']}")
date_str = number_str = None
if data.get("pdf_url"):
tmp_pdf = os.path.join(session_dir, f"_tmp_{kn.replace(':', '_')}.pdf")
try:
if searcher.download_pdf(data["pdf_url"], tmp_pdf):
date_str, number_str = parse_pdf_extract(tmp_pdf, logged)
final_pdf = rename_extract(tmp_pdf, kn, date_str, session_dir)
logged(f" Выписка: {Path(final_pdf).name}")
logged(f" Дата: {date_str} | Номер: {number_str}")
except Exception as e:
logged(f" Ошибка при скачивании/разборе выписки: {e}")
else:
logged(" PDF URL отсутствует")
ws_out.append([
kn, vid_on,
data.get("fias_id"),
data.get("status"),
data.get("full_name"),
data.get("obj_type"),
date_str,
number_str,
])
wb_out.save(output_xlsx_path)
finally:
searcher.close()
file_handler.close()
logged(f"\nГотово!\nПапка: {session_dir}\nТаблица: {Path(output_xlsx_path).name}")
# ---------------------------------------------------------------------------
# GUI — thread-safe via queue
# ---------------------------------------------------------------------------
class App:
BG = "#2C3E50"
FG = "#ECF0F1"
BTN = "#2980B9"
def __init__(self):
self.root = tk.Tk()
self.root.title("ФИАС — Поиск адресов и выгрузка выписок")
self.root.configure(bg=self.BG)
self.root.resizable(True, True)
self._log_queue: queue.Queue[str | None] = queue.Queue()
self._build_ui()
# Poll the log queue from the main thread — the only safe way
self._poll_log()
def _build_ui(self):
pad = {"padx": 8, "pady": 4}
frame = tk.Frame(self.root, bg=self.BG, padx=16, pady=16)
frame.pack(fill="both", expand=True)
tk.Label(frame, text="Исходный файл (КН):", bg=self.BG, fg=self.FG).grid(
row=0, column=0, sticky="w", **pad)
self.v_input = tk.StringVar()
tk.Entry(frame, textvariable=self.v_input, width=55).grid(
row=0, column=1, sticky="ew", **pad)
tk.Button(frame, text="Выбрать…", command=self._pick_input,
bg=self.BTN, fg="white", relief="flat").grid(row=0, column=2, **pad)
tk.Label(frame, text="Папка результатов:", bg=self.BG, fg=self.FG).grid(
row=1, column=0, sticky="w", **pad)
self.v_output = tk.StringVar()
tk.Entry(frame, textvariable=self.v_output, width=55).grid(
row=1, column=1, sticky="ew", **pad)
tk.Button(frame, text="Выбрать…", command=self._pick_output,
bg=self.BTN, fg="white", relief="flat").grid(row=1, column=2, **pad)
self.btn_start = tk.Button(
frame, text="▶ Запустить", command=self._start,
bg="#27AE60", fg="white", relief="flat",
font=("", 11, "bold"), padx=20, pady=6,
)
self.btn_start.grid(row=2, column=0, columnspan=3, pady=12)
self.log_box = scrolledtext.ScrolledText(
frame, width=90, height=28,
bg="#1A252F", fg=self.FG, font=("Courier", 9), state="disabled",
)
self.log_box.grid(row=3, column=0, columnspan=3, sticky="nsew", **pad)
frame.rowconfigure(3, weight=1)
frame.columnconfigure(1, weight=1)
def _poll_log(self):
"""Drain the log queue in the main thread — safe for tkinter."""
try:
while True:
msg = self._log_queue.get_nowait()
if msg is None:
# Sentinel: processing finished
self.btn_start.config(state="normal", text="▶ Запустить")
else:
self.log_box.configure(state="normal")
self.log_box.insert("end", msg + "\n")
self.log_box.see("end")
self.log_box.configure(state="disabled")
except queue.Empty:
pass
self.root.after(100, self._poll_log)
def _log(self, msg: str):
"""Thread-safe: put message in queue, never touch tkinter directly."""
self._log_queue.put(msg)
def _pick_input(self):
path = filedialog.askopenfilename(
title="Файл с кадастровыми номерами",
filetypes=[("Excel", "*.xlsx *.xls"), ("Все файлы", "*")],
)
if path:
self.v_input.set(path)
if not self.v_output.get():
self.v_output.set(str(Path(path).parent))
def _pick_output(self):
path = filedialog.askdirectory(title="Папка для результатов")
if path:
self.v_output.set(path)
def _start(self):
inp = self.v_input.get().strip()
out = self.v_output.get().strip()
if not inp or not os.path.isfile(inp):
messagebox.showerror("Ошибка", "Выберите существующий файл Excel с КН")
return
if not out or not os.path.isdir(out):
messagebox.showerror("Ошибка", "Выберите существующую папку")
return
self.btn_start.config(state="disabled", text="⏳ Выполняется…")
def worker():
try:
process_all(inp, out, log=self._log)
# Schedule GUI notifications on main thread
self.root.after(0, lambda: messagebox.showinfo(
"Готово", "Обработка завершена!\nСм. Результаты_ФИАС.xlsx"
))
except Exception as exc:
self._log(f"\nКРИТИЧЕСКАЯ ОШИБКА: {exc}")
self.root.after(0, lambda msg=str(exc): messagebox.showerror("Ошибка", msg))
finally:
self._log_queue.put(None) # Sentinel → re-enable button
threading.Thread(target=worker, daemon=True).start()
def run(self):
self.root.mainloop()
# ---------------------------------------------------------------------------
def main():
App().run()
if __name__ == "__main__":
main()