Initial commit: GPU temperature monitor with email alerts

SSHes into VM 109, checks nvidia-smi every 60s, sends Gmail alert
at warn (80°C) and critical (88°C) thresholds.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -0,0 +1,15 @@
+GPU_HOST=192.168.20.47
+GPU_USER=user
+GPU_PASS=your_vm_password
+
+# Gmail: нужен App Password (не основной пароль)
+# https://myaccount.google.com/apppasswords
+SMTP_HOST=smtp.gmail.com
+SMTP_PORT=587
+SMTP_USER=jze9programer@gmail.com
+SMTP_PASS=your_gmail_app_password
+ALERT_TO=jze9programer@gmail.com
+
+WARN_TEMP=80
+CRIT_TEMP=88
+CHECK_EVERY=60

gpu-monitor.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=GPU temperature monitor
+After=network.target
+
+[Service]
+Type=simple
+User=jze9
+WorkingDirectory=/home/jze9/Documents/git/gpu-monitor
+EnvironmentFile=/home/jze9/Documents/git/gpu-monitor/.env
+ExecStart=/home/jze9/Documents/git/gpu-monitor/venv/bin/python monitor.py
+Restart=on-failure
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

monitor.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+GPU temperature monitor.
+SSHes into VM 109, checks nvidia-smi, sends email if critical.
+"""
+import smtplib
+import subprocess
+import time
+import logging
+import os
+from dataclasses import dataclass
+from email.mime.text import MIMEText
+from dotenv import load_dotenv
+
+load_dotenv()
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(message)s",
+    handlers=[
+        logging.FileHandler("/var/log/gpu-monitor.log"),
+        logging.StreamHandler(),
+    ]
+)
+log = logging.getLogger(__name__)
+
+# ── config ────────────────────────────────────────────────────────────────────
+
+GPU_HOST     = os.getenv("GPU_HOST",     "192.168.20.47")
+GPU_USER     = os.getenv("GPU_USER",     "user")
+GPU_PASS     = os.getenv("GPU_PASS",     "")
+
+SMTP_HOST    = os.getenv("SMTP_HOST",    "smtp.gmail.com")
+SMTP_PORT    = int(os.getenv("SMTP_PORT", "587"))
+SMTP_USER    = os.getenv("SMTP_USER",    "")
+SMTP_PASS    = os.getenv("SMTP_PASS",    "")
+ALERT_TO     = os.getenv("ALERT_TO",     "")
+
+WARN_TEMP    = int(os.getenv("WARN_TEMP",  "80"))
+CRIT_TEMP    = int(os.getenv("CRIT_TEMP",  "88"))
+CHECK_EVERY  = int(os.getenv("CHECK_EVERY", "60"))   # seconds
+
+# ── GPU query ─────────────────────────────────────────────────────────────────
+
+@dataclass
+class GpuStats:
+    temp: int
+    fan: str
+    power: str
+    util: str
+    mem_used: str
+    mem_total: str
+
+def get_gpu_stats() -> GpuStats | None:
+    cmd = [
+        "sshpass", "-p", GPU_PASS,
+        "ssh", "-o", "StrictHostKeyChecking=no",
+        "-o", "ConnectTimeout=10",
+        f"{GPU_USER}@{GPU_HOST}",
+        "nvidia-smi --query-gpu=temperature.gpu,fan.speed,power.draw,"
+        "utilization.gpu,memory.used,memory.total --format=csv,noheader"
+    ]
+    try:
+        out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL, timeout=15)
+        parts = [p.strip() for p in out.decode().strip().split(",")]
+        return GpuStats(
+            temp=int(parts[0]),
+            fan=parts[1],
+            power=parts[2],
+            util=parts[3],
+            mem_used=parts[4],
+            mem_total=parts[5],
+        )
+    except Exception as e:
+        log.error(f"Failed to query GPU: {e}")
+        return None
+
+# ── email ─────────────────────────────────────────────────────────────────────
+
+def send_alert(subject: str, body: str):
+    if not SMTP_USER or not ALERT_TO:
+        log.warning("Email not configured, skipping alert")
+        return
+    try:
+        msg = MIMEText(body, "plain", "utf-8")
+        msg["Subject"] = subject
+        msg["From"]    = SMTP_USER
+        msg["To"]      = ALERT_TO
+        with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=15) as s:
+            s.starttls()
+            s.login(SMTP_USER, SMTP_PASS)
+            s.send_message(msg)
+        log.info(f"Alert sent: {subject}")
+    except Exception as e:
+        log.error(f"Failed to send email: {e}")
+
+# ── main loop ─────────────────────────────────────────────────────────────────
+
+def main():
+    log.info(f"GPU monitor started — checking {GPU_HOST} every {CHECK_EVERY}s")
+    log.info(f"Thresholds: warn={WARN_TEMP}°C  crit={CRIT_TEMP}°C")
+
+    last_alert_temp = 0   # avoid spamming — only re-alert if temp changes by 3°C
+
+    while True:
+        stats = get_gpu_stats()
+
+        if stats is None:
+            time.sleep(CHECK_EVERY)
+            continue
+
+        log.info(
+            f"GPU {stats.temp}°C | fan {stats.fan} | {stats.power} | "
+            f"util {stats.util} | mem {stats.mem_used}/{stats.mem_total}"
+        )
+
+        if stats.temp >= CRIT_TEMP:
+            if abs(stats.temp - last_alert_temp) >= 3:
+                send_alert(
+                    f"🔥 КРИТИЧНО: GPU {stats.temp}°C на {GPU_HOST}",
+                    f"GPU перегрев!\n\n"
+                    f"Температура : {stats.temp}°C  (критично от {CRIT_TEMP}°C)\n"
+                    f"Вентилятор  : {stats.fan}\n"
+                    f"Потребление : {stats.power}\n"
+                    f"Загрузка GPU: {stats.util}\n"
+                    f"Память      : {stats.mem_used} / {stats.mem_total}\n\n"
+                    f"Проверь охлаждение немедленно."
+                )
+                last_alert_temp = stats.temp
+
+        elif stats.temp >= WARN_TEMP:
+            if abs(stats.temp - last_alert_temp) >= 3:
+                send_alert(
+                    f"⚠️ ВНИМАНИЕ: GPU {stats.temp}°C на {GPU_HOST}",
+                    f"GPU горячий.\n\n"
+                    f"Температура : {stats.temp}°C  (предупреждение от {WARN_TEMP}°C)\n"
+                    f"Вентилятор  : {stats.fan}\n"
+                    f"Потребление : {stats.power}\n"
+                    f"Загрузка GPU: {stats.util}\n"
+                    f"Память      : {stats.mem_used} / {stats.mem_total}\n"
+                )
+                last_alert_temp = stats.temp
+        else:
+            # reset cooldown when temp is back to normal
+            if last_alert_temp > 0:
+                log.info(f"GPU temp back to normal ({stats.temp}°C)")
+                last_alert_temp = 0
+
+        time.sleep(CHECK_EVERY)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1 @@
+python-dotenv>=1.0.0