From 83c3e4232b149b480a3ab2b7c34e319b009612bc Mon Sep 17 00:00:00 2001 From: jze9 Date: Thu, 28 May 2026 20:53:50 +0500 Subject: [PATCH] Initial commit: GPU temperature monitor with email alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSHes into VM 109, checks nvidia-smi every 60s, sends Gmail alert at warn (80°C) and critical (88°C) thresholds. Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 15 +++++ gpu-monitor.service | 15 +++++ monitor.py | 152 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 183 insertions(+) create mode 100644 .env.example create mode 100644 gpu-monitor.service create mode 100644 monitor.py create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2e78a61 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +GPU_HOST=192.168.20.47 +GPU_USER=user +GPU_PASS=your_vm_password + +# Gmail: нужен App Password (не основной пароль) +# https://myaccount.google.com/apppasswords +SMTP_HOST=smtp.gmail.com +SMTP_PORT=587 +SMTP_USER=jze9programer@gmail.com +SMTP_PASS=your_gmail_app_password +ALERT_TO=jze9programer@gmail.com + +WARN_TEMP=80 +CRIT_TEMP=88 +CHECK_EVERY=60 diff --git a/gpu-monitor.service b/gpu-monitor.service new file mode 100644 index 0000000..c91f1f7 --- /dev/null +++ b/gpu-monitor.service @@ -0,0 +1,15 @@ +[Unit] +Description=GPU temperature monitor +After=network.target + +[Service] +Type=simple +User=jze9 +WorkingDirectory=/home/jze9/Documents/git/gpu-monitor +EnvironmentFile=/home/jze9/Documents/git/gpu-monitor/.env +ExecStart=/home/jze9/Documents/git/gpu-monitor/venv/bin/python monitor.py +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/monitor.py b/monitor.py new file mode 100644 index 0000000..baae248 --- /dev/null +++ b/monitor.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +GPU temperature monitor. +SSHes into VM 109, checks nvidia-smi, sends email if critical. +""" +import smtplib +import subprocess +import time +import logging +import os +from dataclasses import dataclass +from email.mime.text import MIMEText +from dotenv import load_dotenv + +load_dotenv() + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + handlers=[ + logging.FileHandler("/var/log/gpu-monitor.log"), + logging.StreamHandler(), + ] +) +log = logging.getLogger(__name__) + +# ── config ──────────────────────────────────────────────────────────────────── + +GPU_HOST = os.getenv("GPU_HOST", "192.168.20.47") +GPU_USER = os.getenv("GPU_USER", "user") +GPU_PASS = os.getenv("GPU_PASS", "") + +SMTP_HOST = os.getenv("SMTP_HOST", "smtp.gmail.com") +SMTP_PORT = int(os.getenv("SMTP_PORT", "587")) +SMTP_USER = os.getenv("SMTP_USER", "") +SMTP_PASS = os.getenv("SMTP_PASS", "") +ALERT_TO = os.getenv("ALERT_TO", "") + +WARN_TEMP = int(os.getenv("WARN_TEMP", "80")) +CRIT_TEMP = int(os.getenv("CRIT_TEMP", "88")) +CHECK_EVERY = int(os.getenv("CHECK_EVERY", "60")) # seconds + +# ── GPU query ───────────────────────────────────────────────────────────────── + +@dataclass +class GpuStats: + temp: int + fan: str + power: str + util: str + mem_used: str + mem_total: str + +def get_gpu_stats() -> GpuStats | None: + cmd = [ + "sshpass", "-p", GPU_PASS, + "ssh", "-o", "StrictHostKeyChecking=no", + "-o", "ConnectTimeout=10", + f"{GPU_USER}@{GPU_HOST}", + "nvidia-smi --query-gpu=temperature.gpu,fan.speed,power.draw," + "utilization.gpu,memory.used,memory.total --format=csv,noheader" + ] + try: + out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL, timeout=15) + parts = [p.strip() for p in out.decode().strip().split(",")] + return GpuStats( + temp=int(parts[0]), + fan=parts[1], + power=parts[2], + util=parts[3], + mem_used=parts[4], + mem_total=parts[5], + ) + except Exception as e: + log.error(f"Failed to query GPU: {e}") + return None + +# ── email ───────────────────────────────────────────────────────────────────── + +def send_alert(subject: str, body: str): + if not SMTP_USER or not ALERT_TO: + log.warning("Email not configured, skipping alert") + return + try: + msg = MIMEText(body, "plain", "utf-8") + msg["Subject"] = subject + msg["From"] = SMTP_USER + msg["To"] = ALERT_TO + with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=15) as s: + s.starttls() + s.login(SMTP_USER, SMTP_PASS) + s.send_message(msg) + log.info(f"Alert sent: {subject}") + except Exception as e: + log.error(f"Failed to send email: {e}") + +# ── main loop ───────────────────────────────────────────────────────────────── + +def main(): + log.info(f"GPU monitor started — checking {GPU_HOST} every {CHECK_EVERY}s") + log.info(f"Thresholds: warn={WARN_TEMP}°C crit={CRIT_TEMP}°C") + + last_alert_temp = 0 # avoid spamming — only re-alert if temp changes by 3°C + + while True: + stats = get_gpu_stats() + + if stats is None: + time.sleep(CHECK_EVERY) + continue + + log.info( + f"GPU {stats.temp}°C | fan {stats.fan} | {stats.power} | " + f"util {stats.util} | mem {stats.mem_used}/{stats.mem_total}" + ) + + if stats.temp >= CRIT_TEMP: + if abs(stats.temp - last_alert_temp) >= 3: + send_alert( + f"🔥 КРИТИЧНО: GPU {stats.temp}°C на {GPU_HOST}", + f"GPU перегрев!\n\n" + f"Температура : {stats.temp}°C (критично от {CRIT_TEMP}°C)\n" + f"Вентилятор : {stats.fan}\n" + f"Потребление : {stats.power}\n" + f"Загрузка GPU: {stats.util}\n" + f"Память : {stats.mem_used} / {stats.mem_total}\n\n" + f"Проверь охлаждение немедленно." + ) + last_alert_temp = stats.temp + + elif stats.temp >= WARN_TEMP: + if abs(stats.temp - last_alert_temp) >= 3: + send_alert( + f"⚠️ ВНИМАНИЕ: GPU {stats.temp}°C на {GPU_HOST}", + f"GPU горячий.\n\n" + f"Температура : {stats.temp}°C (предупреждение от {WARN_TEMP}°C)\n" + f"Вентилятор : {stats.fan}\n" + f"Потребление : {stats.power}\n" + f"Загрузка GPU: {stats.util}\n" + f"Память : {stats.mem_used} / {stats.mem_total}\n" + ) + last_alert_temp = stats.temp + else: + # reset cooldown when temp is back to normal + if last_alert_temp > 0: + log.info(f"GPU temp back to normal ({stats.temp}°C)") + last_alert_temp = 0 + + time.sleep(CHECK_EVERY) + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0eed382 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +python-dotenv>=1.0.0