From cedae7ab926c1fe2e56bc1726c89a62527120429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=B8=85?= <2920339316@qq.com> Date: Thu, 5 Nov 2026 18:39:20 +0800 Subject: [PATCH] Initial hardware dashboard and scheduled task service --- .gitattributes | 12 + .gitignore | 24 + README.md | 58 ++ config/hardware-watch.example.json | 11 + deploy/install.sh | 44 ++ deploy/journald/60-hardware-watch.conf | 5 + deploy/systemd/hardware-dashboard.service | 29 + deploy/systemd/hardware-taskd.service | 23 + deploy/systemd/hardware-watch.service | 23 + deploy/systemd/hardware-watch.timer | 11 + docs/DEPLOYMENT.zh-CN.md | 263 ++++++++ docs/REPOSITORY.md | 26 + pyproject.toml | 10 + scripts/hwlog | 438 +++++++++++++ server.py | 730 ++++++++++++++++++++++ static/app.js | 171 +++++ static/favicon.svg | 1 + static/index.html | 81 +++ static/style.css | 465 ++++++++++++++ static/tasks.js | 176 ++++++ task_core.py | 386 ++++++++++++ task_exec.py | 104 +++ taskd.py | 193 ++++++ tests/test_dashboard.py | 147 +++++ tests/test_tasks.py | 99 +++ 25 files changed, 3530 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config/hardware-watch.example.json create mode 100755 deploy/install.sh create mode 100644 deploy/journald/60-hardware-watch.conf create mode 100644 deploy/systemd/hardware-dashboard.service create mode 100644 deploy/systemd/hardware-taskd.service create mode 100644 deploy/systemd/hardware-watch.service create mode 100644 deploy/systemd/hardware-watch.timer create mode 100644 docs/DEPLOYMENT.zh-CN.md create mode 100644 docs/REPOSITORY.md create mode 100644 pyproject.toml create mode 100755 scripts/hwlog create mode 100644 server.py create mode 100644 static/app.js create mode 100644 static/favicon.svg create mode 100644 static/index.html create mode 100644 static/style.css create mode 100644 static/tasks.js create mode 100644 task_core.py create mode 100644 task_exec.py create mode 100644 taskd.py create mode 100644 tests/test_dashboard.py create mode 100644 tests/test_tasks.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4a5d341 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +* text=auto +*.py text eol=lf +*.js text eol=lf +*.css text eol=lf +*.html text eol=lf +*.md text eol=lf +*.json text eol=lf +*.service text eol=lf +*.timer text eol=lf +*.conf text eol=lf +*.svg text eol=lf +scripts/hwlog text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d72dfc2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.venv/ +venv/ + +# Runtime state and local deployment secrets +*.sqlite3 +*.sqlite3-shm +*.sqlite3-wal +*.log +*.pid +config/hardware-watch.json +config/hardware-task-admin.json +hardware-task-admin-login*.txt + +# Local archives and editor state +*.tar.gz +.idea/ +.vscode/ +.DS_Store +Thumbs.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..c7247cb --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# Hardware Dashboard + +这是运行在 192.168.1.37 上的局域网设备监控与维护面板。页面提供 CPU、内存、双 P100 显卡、磁盘 SMART、性能历史曲线、硬件问题记录和定时任务管理。后端只使用 Python 标准库;前端使用原生 HTML、CSS 和 JavaScript,不依赖外部 CDN。 + +当前访问地址是 http://192.168.1.37:8090。定时任务可以执行服务器命令,必须使用独立的管理员账号登录。该账号与 SSH root 密码相互独立。 + +## 目录结构 + +``` +. +├── server.py # 监控网页、实时指标和历史数据 API +├── task_core.py # 定时任务定义、校验和 systemd 管理 +├── taskd.py # root 侧任务管理服务 +├── task_exec.py # 单个任务的执行器和输出记录 +├── static/ # 前端页面、样式和脚本 +├── scripts/hwlog # 硬件事件采集与命令行查看工具 +├── config/ +| └── hardware-watch.example.json +├── deploy/systemd/ # dashboard、taskd 和硬件采集服务文件 +├── deploy/journald/ # 原始内核日志持久化配置 +├── docs/DEPLOYMENT.zh-CN.md # 服务器部署与技术说明 +└── tests/ # 自动化测试 +``` + +## 本地检查 + +项目只需要 Python 3.12 或更新版本,没有第三方 Python 依赖。 + +``` +python -m unittest discover -s tests -p "test_*.py" -v +python -m py_compile server.py task_core.py taskd.py task_exec.py scripts/hwlog +``` + +如需本地启动网页: + +``` +python server.py --host 127.0.0.1 --port 8090 +``` + +服务器上的完整部署方式、文件路径、权限、任务管理和维护命令见 docs/DEPLOYMENT.zh-CN.md。 + +## 服务器配置 + +仓库中不保存真实硬件基准配置和任务管理凭据。部署前复制示例配置: + +``` +sudo cp config/hardware-watch.example.json /etc/hardware-watch.json +sudo nano /etc/hardware-watch.json +``` + +把 expected_gpu_count、GPU UUID 和磁盘名改成当前服务器的实际值。修改后重启监控采集服务。定时任务管理服务首次初始化会生成独立管理员密码文件;不要把生成的密码、任务数据库、日志或系统配置哈希提交到仓库。 + +## 安全边界 + +- 监控页面本身是只读的,可查看状态、曲线和日志。 +- 定时任务能以 root 身份执行命令,新增、编辑、启用和立即执行均需要任务管理员登录。 +- 不要把这个服务直接暴露到公网。当前设计只适合可信局域网。 +- 远程访问应通过可信 VPN 或带 HTTPS 与额外认证的反向代理,而不是直接端口映射。 diff --git a/config/hardware-watch.example.json b/config/hardware-watch.example.json new file mode 100644 index 0000000..70d695e --- /dev/null +++ b/config/hardware-watch.example.json @@ -0,0 +1,11 @@ +{ + "expected_gpu_count": 2, + "expected_gpu_uuids": [ + "GPU-00000000-0000-0000-0000-000000000001", + "GPU-00000000-0000-0000-0000-000000000002" + ], + "expected_disks": ["nvme0n1"], + "gpu_warn_c": 85, + "history_since": "7 days ago", + "retention_days": 90 +} diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100755 index 0000000..7130cdf --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +INSTALL_ROOT="${INSTALL_ROOT:-/opt/hardware-dashboard}" +CONFIG_FILE="${CONFIG_FILE:-/etc/hardware-watch.json}" +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "必须使用 root 运行部署脚本。" >&2 + exit 1 +fi + +if [[ ! -f "${CONFIG_FILE}" ]]; then + echo "缺少 ${CONFIG_FILE}。请先复制 config/hardware-watch.example.json 并填入当前服务器实际值。" >&2 + exit 1 +fi + +getent group hardware-watch >/dev/null || groupadd --system hardware-watch +getent passwd hardware-dashboard >/dev/null || useradd --system --gid hardware-watch --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin hardware-dashboard + +install -d -m 0755 "${INSTALL_ROOT}/static" +install -d -m 0750 /var/lib/hardware-dashboard /var/lib/hardware-watch /var/log/hardware-watch +install -d -m 0755 /etc/systemd/journald.conf.d + +install -m 0644 server.py task_core.py taskd.py task_exec.py "${INSTALL_ROOT}/" +install -m 0644 static/index.html static/app.js static/tasks.js static/style.css static/favicon.svg "${INSTALL_ROOT}/static/" +install -m 0755 scripts/hwlog /usr/local/sbin/hwlog +install -m 0640 config/hardware-watch.json "${CONFIG_FILE}" +install -m 0644 deploy/systemd/*.service deploy/systemd/*.timer "${SYSTEMD_DIR}/" +install -m 0644 deploy/journald/60-hardware-watch.conf /etc/systemd/journald.conf.d/60-hardware-watch.conf + +setfacl -m u:hardware-dashboard:rx /var/lib/hardware-watch /var/log/hardware-watch +setfacl -d -m u:hardware-dashboard:rx /var/lib/hardware-watch /var/log/hardware-watch +find /var/lib/hardware-watch /var/log/hardware-watch -maxdepth 1 -type f -exec setfacl -m u:hardware-dashboard:r {} + +setfacl -m u:hardware-dashboard:r "${CONFIG_FILE}" + +systemctl daemon-reload +systemctl enable --now hardware-watch.timer +systemctl restart hardware-watch.service +systemctl enable --now hardware-taskd.service +systemctl restart hardware-dashboard.service + +echo "部署完成。首次初始化定时任务管理员:" +echo "python3 ${INSTALL_ROOT}/taskd.py init-admin" diff --git a/deploy/journald/60-hardware-watch.conf b/deploy/journald/60-hardware-watch.conf new file mode 100644 index 0000000..f43b326 --- /dev/null +++ b/deploy/journald/60-hardware-watch.conf @@ -0,0 +1,5 @@ +[Journal] +Storage=persistent +SystemMaxUse=1G +SystemKeepFree=2G +MaxRetentionSec=90day diff --git a/deploy/systemd/hardware-dashboard.service b/deploy/systemd/hardware-dashboard.service new file mode 100644 index 0000000..47e36ab --- /dev/null +++ b/deploy/systemd/hardware-dashboard.service @@ -0,0 +1,29 @@ +[Unit] +Description=Hardware monitoring dashboard on LAN port 8090 +After=network.target hardware-watch.timer + +[Service] +Type=simple +User=hardware-dashboard +Group=hardware-watch +WorkingDirectory=/opt/hardware-dashboard +ExecStart=/usr/bin/python3 /opt/hardware-dashboard/server.py --port 8090 +Restart=always +RestartSec=5 +Environment=PYTHONDONTWRITEBYTECODE=1 +Environment=TZ=Asia/Shanghai +StateDirectory=hardware-dashboard +StateDirectoryMode=0750 +UMask=0027 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ProtectKernelTunables=true +ProtectControlGroups=true +ReadWritePaths=/var/lib/hardware-dashboard +MemoryMax=384M +Nice=10 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/hardware-taskd.service b/deploy/systemd/hardware-taskd.service new file mode 100644 index 0000000..d4a4dfe --- /dev/null +++ b/deploy/systemd/hardware-taskd.service @@ -0,0 +1,23 @@ +[Unit] +Description=Authenticated local scheduled-task broker +After=local-fs.target + +[Service] +Type=simple +User=root +Group=hardware-watch +ExecStart=/usr/bin/python3 /opt/hardware-dashboard/taskd.py serve +Restart=on-failure +RestartSec=5 +RuntimeDirectory=hardware-taskd +RuntimeDirectoryMode=0750 +StateDirectory=hardware-taskd +StateDirectoryMode=0700 +LogsDirectory=hardware-taskd +LogsDirectoryMode=0700 +UMask=0077 +Environment=TZ=Asia/Shanghai +Environment=PYTHONDONTWRITEBYTECODE=1 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/hardware-watch.service b/deploy/systemd/hardware-watch.service new file mode 100644 index 0000000..89f86d0 --- /dev/null +++ b/deploy/systemd/hardware-watch.service @@ -0,0 +1,23 @@ +[Unit] +Description=Hardware event journal and GPU / disk health sampling +After=systemd-journald.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/hwlog collect +TimeoutStartSec=55 +Nice=10 +IOSchedulingClass=idle +UMask=0027 +StateDirectory=hardware-watch +StateDirectoryMode=0750 +LogsDirectory=hardware-watch +LogsDirectoryMode=0750 +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true +ProtectKernelTunables=true +ProtectControlGroups=true +MemoryMax=384M +ReadWritePaths=/var/lib/hardware-watch /var/log/hardware-watch diff --git a/deploy/systemd/hardware-watch.timer b/deploy/systemd/hardware-watch.timer new file mode 100644 index 0000000..63d59e6 --- /dev/null +++ b/deploy/systemd/hardware-watch.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check hardware health every minute + +[Timer] +OnBootSec=45s +OnUnitActiveSec=60s +AccuracySec=5s +Unit=hardware-watch.service + +[Install] +WantedBy=timers.target diff --git a/docs/DEPLOYMENT.zh-CN.md b/docs/DEPLOYMENT.zh-CN.md new file mode 100644 index 0000000..9d2c281 --- /dev/null +++ b/docs/DEPLOYMENT.zh-CN.md @@ -0,0 +1,263 @@ +# 192.168.1.37 监控与定时任务技术说明 + +版本:v1.2,2026 年 11 月 4 日。 + +## 1. 访问与登录 + +设备监控页面:**[http://192.168.1.37:8090](http://192.168.1.37:8090)** + +定时任务入口在“问题与维护”下方,也可通过左侧“定时任务”菜单定位。手机底部有同名入口。 + +硬件监控仍可直接查看。定时任务能够实际执行服务器操作,因此使用独立的管理员账号登录: + +- 账号:`admin`。 +- 初始密码:见交付的 `定时任务管理员登录信息.txt`,本说明不包含密码。 +- 登录最长有效 8 小时;修改密码后所有任务管理登录均失效。 +- 此密码与 SSH root 密码相互独立,本次没有修改 SSH 密码。 + +页面只开放在原有局域网访问范围内。当前使用局域网 HTTP,适用于可信局域网;不应将这个具备 root 任务执行能力的入口直接映射到公网。将来如需远程公开访问,应先配置 HTTPS 和适当的访问控制。 + +## 2. 本次新增功能 + +“定时任务”支持: + +- 新增、编辑、启用、停用和删除任务。 +- 立即执行一次;这不会自动启用任务的定时计划。 +- 停止本次执行;与停用后续调度分开处理。 +- 查看下次计划时间、最近执行结果和是否正在执行。 +- 查看单任务执行记录,以及包含已删除任务的全局执行记录。 +- 查看开始/结束时间、耗时、退出码、标准输出和错误输出。 +- 管理员登录、退出及修改密码。 + +执行方式有三种: + +| 执行方式 | 如何填写 | 说明 | +|---|---|---| +| 直接输入命令 | 例如 `/usr/bin/nvidia-smi` | 使用 Bash 执行,支持管道和多行命令。 | +| 粘贴脚本内容 | 选择 Bash 或 Python 3,粘贴代码 | 可以导入本地 `.sh`、`.py` 或文本脚本到编辑框;保存后才会传给服务器。 | +| 服务器上的脚本文件 | 填写绝对路径和可选参数 | 可选择 Bash、Python 3 或按文件自身方式执行;最后一种要求文件具有执行权限。 | + +执行身份固定为 **root**,因为重启服务器和维护证书通常需要管理员权限。它不是隔离的脚本沙箱:应只运行自己理解并信任的命令。 + +工作目录默认 `/root`,使用非交互执行环境,不继承某个 SSH 窗口的虚拟环境。PATH 包含常用系统目录和 `/snap/bin`;如需 Conda、venv 或自定义环境变量,应在脚本内明确设置。 + +## 3. 第一次添加任务 + +1. 刷新页面,打开“定时任务”,点击“登录管理”。 +2. 使用交付文件中的 `admin` 与初始密码登录。 +3. 点击“新增任务”,填写任务名称。 +4. 选择执行方式,输入命令、脚本或服务器脚本路径。 +5. 设置执行周期,点击“检查执行周期”核对下次时间。 +6. 保持“保存后启用”不勾选,先保存为停用状态。 +7. 如适合测试,点击“执行一次”,确认执行内容,查看执行日志。 +8. 核对结果后,再启用计划。 + +保存任务本身不会立即执行脚本。新任务默认停用。退出登录、关闭网页或重启网页服务不会停止已经启用的系统计划。 + +### 常用模板 + +新增窗口提供: + +- **显卡状态检查**:读取 `nvidia-smi`,结果保存在执行日志中。 +- **服务器定时重启**:示例命令为 `systemctl reboot`,默认周期是每周日 03:00,默认停用。启用或立即执行前都会提示可能中断计算任务。 +- **SSL 证书续签示例**:检查已有的 Certbot/acme.sh 工具并调用其续签入口。它不会安装工具、申请新域名或替用户配置证书。先确认现有续签配置,以及是否已经有其他自动续签任务;Web 服务的重新加载应按实际服务配置。 + +本次部署没有创建或启用实际的服务器重启任务和证书续签任务,也没有执行这两类操作。 + +## 4. 执行周期 + +所有日历时间按 **Asia/Shanghai,北京时间 UTC+8** 处理,页面显示服务器当前时间。 + +| 周期 | 规则 | +|---|---| +| 每天 | 指定时、分。 | +| 每周 | 可选择一个或多个星期及时间。 | +| 每月 | 选择 1 至 31 日和时间;当月不存在该日期时跳过。 | +| 每隔若干分钟/小时 | 启用后先等待指定间隔;每次执行结束后重新计时,避免长任务造成重叠。 | +| 指定日期,仅执行一次 | 选择未来的日期和时间,精确到分钟;至少晚于保存时刻 30 秒。 | +| 自定义 Cron | 标准五段:分、时、日、月、周。支持 `*`、逗号、范围、步长及常见英文月/星期名。 | + +Cron 示例: + +```text +0 3 * * * 每天 03:00 +0 3 * * 0 每周日 03:00 +0 3 1 * * 每月 1 日 03:00 +*/30 * * * * 每小时的 00、30 分 +``` + +不支持 `L`、`W`、`#`、`?` 等扩展语法,也不接受带秒的六段 Cron。日和星期同时指定时,按传统 Cron 的“任一满足”语义处理。 + +Cron 是日历触发,和“上次结束后再间隔 N 分钟”不同。例如长任务执行 10 分钟,再间隔 5 分钟,下一次大约在本次开始 15 分钟后。 + +### 开机、错过与长任务 + +- 日历任务设置 `Persistent=false`,服务器关机期间错过的执行时间不补跑,避免开机后补执行重启类任务。 +- 同一个任务由固定 systemd 服务承载,并有文件锁,不会同时重叠执行两份。 +- 一个任务执行超时后会被停止。默认最长 1800 秒,可在高级设置中改为 10 至 86400 秒。 +- 关闭网页只会关闭查看界面,不会停止系统定时器。 +- 本机历史上出现过时钟回拨。启用重启等任务前,应核对页面显示的服务器时间;定时调度依赖服务器时钟和系统校时状态。 + +部署验证时,服务器时区为 `Asia/Shanghai`,系统报告 `NTPSynchronized=yes`。 + +## 5. 看懂执行结果 + +| 显示 | 意义 | +|---|---| +| 尚未执行 | 没有记录到执行。 | +| 执行中 | 本次进程仍在运行。 | +| 执行成功 | 命令正常退出,退出码为 0。 | +| 执行失败 | 命令退出码非 0,或启动过程中发生错误。 | +| 超时已停止 | 达到设置的最长执行时间,已停止本次进程。 | +| 已停止 | 本次进程被停止,可能是手动操作或系统停止服务。 | +| 执行被中断 | 执行期间系统重启或进程中断,未取得完整结果。 | + +成功只表示脚本返回了 0,不保证某个证书一定已更新、备份一定可恢复或某个业务结果一定正确;这些应结合脚本输出核对。 + +“停用”只取消将来的调度,不会强制中止已经开始的执行。“停止本次”会终止执行进程,脚本可能尚未完成修改,操作前会提示确认。 + +目前单任务窗口展示最近 40 次执行,全局窗口展示最近 80 次。服务器执行记录按 90 天清理;每次输出上限约 512 KiB,网页每次读取最后 128 KiB。输出与脚本快照还有合计 256 MiB 的空间限制,较旧文件可能提前清理,执行状态元数据仍保留至期限。 + +删除任务会停用计划并移除模块生成的定时器,保留已有执行记录。历史记录可在“全部执行记录”查看。 + +## 6. 参考与技术栈 + +界面参考 [宝塔 Linux 面板](https://www.bt.cn/) 计划任务的组织方式:任务列表、执行周期、Shell 脚本、启停、手动执行和执行日志。本次功能自行编写,沿用已有监控页面的深色风格,没有安装宝塔,也没有复制其面板源码。 + +原监控布局仍参考: + +- [Zabbix](https://github.com/zabbix/zabbix):设备总览和告警分级。 +- [Netdata](https://github.com/netdata/netdata):实时指标展示。 +- [Grafana](https://github.com/grafana/grafana):历史曲线、图例和时间范围。 + +| 部分 | 使用技术 | +|---|---| +| 监控与任务前端 | HTML、CSS、原生 JavaScript、原生 `` 对话框。 | +| 曲线 | 原生 SVG,保留原有 CPU、内存、GPU 温度、功耗和利用率历史。 | +| 网页后端 | Python 3.12 标准库,`ThreadingHTTPServer`。 | +| 数据库 | SQLite,WAL 模式;监控数据与任务数据分库存储。 | +| 系统调度 | systemd `.timer` 和 `.service`,不是浏览器内的倒计时。 | +| 网页与特权服务通信 | 本机 Unix socket,限制连接进程身份。 | +| 管理密码 | 随机盐、PBKDF2-HMAC-SHA256,240000 次迭代。 | +| 登录态 | 内存会话、HttpOnly/SameSite=Strict Cookie;写操作同时检查 Origin 与 CSRF。 | +| 执行控制 | 独立进程组、文件锁、超时、停止信号、限量输出、执行审计。 | + +没有新增 npm、pip 或外部 CDN 依赖。服务器不需要 Node.js;开发阶段使用 Node.js 检查 JavaScript 语法。 + +## 7. 为什么要拆成两个服务 + +```mermaid +flowchart LR + A[浏览器:监控与任务界面] --> B[hardware-dashboard\n普通账号\nHTTP 8090] + B --> C[(原监控数据库)] + B -->|本机 Unix socket\n验证会话与 CSRF| D[hardware-taskd\nroot 任务管理服务] + D --> E[(独立任务数据库\nroot 专用目录)] + D --> F[systemd 定时器] + F --> G[task_exec\n执行指定任务] + G --> H[执行日志与结果] + H --> D +``` + +硬件网页继续使用原来的 `hardware-dashboard` 普通账号。新增 root 服务只通过本机 Unix socket 接收经验证的任务操作,不额外开放网络端口。 + +任务内容不插入 systemd 的 `ExecStart`。生成的服务只启动固定的执行器和经过验证的任务 ID;执行器从 root 专用数据库读取任务内容。这避免任务名称、换行或参数变成 systemd 单元配置。 + +上传或粘贴的脚本文本会统一换行为 Linux 格式。服务器脚本路径和参数以独立参数列表传递;命令内容模式则按管理员输入交给 Bash 执行。 + +root 服务不是恶意脚本隔离环境。管理员既然有权输入 root 命令,就应当视为具有服务器管理员权限。 + +## 8. 文件与数据库位置 + +以下均在 `192.168.1.37`: + +| 路径 | 内容 | +|---|---| +| `/opt/hardware-dashboard/server.py` | 网页 HTTP 和监控 API;新增任务 API 代理。 | +| `/opt/hardware-dashboard/static/tasks.js` | 任务表单、登录、启停、执行和日志交互。 | +| `/opt/hardware-dashboard/task_core.py` | 校验、周期转换、任务数据库和 systemd 管理。 | +| `/opt/hardware-dashboard/taskd.py` | 本机任务服务、会话验证、审计和清理。 | +| `/opt/hardware-dashboard/task_exec.py` | 单次执行、输出、超时与退出结果。 | +| `/etc/systemd/system/hardware-taskd.service` | 新增任务管理常驻服务。 | +| `/etc/systemd/system/hwjob-任务ID.timer` | 本模块为已创建任务生成的定时器。 | +| `/etc/systemd/system/hwjob-任务ID.service` | 对应的执行服务。 | +| `/run/hardware-taskd/control.sock` | 普通网页与 root 服务之间的本机通信入口。 | +| `/etc/hardware-task-admin.json` | 管理密码的盐与散列,仅 root 可读。 | +| `/var/lib/hardware-taskd/tasks.sqlite3` | 任务定义、执行元数据和审计。 | +| `/var/lib/hardware-taskd/scripts/` | 脚本内容模式的执行快照。 | +| `/var/log/hardware-taskd/` | 每次执行的输出日志,仅经管理登录读取。 | +| `/var/lib/hardware-dashboard/monitor.sqlite3` | 原来的性能和硬件日志数据库,继续保留。 | +| `/root/hardware-task-admin-login.txt` | 初始登录信息的服务器副本,仅 root 可读。 | + +原有 `hwlog` 命令、硬件采集器及性能历史继续运行。本模块只管理自己创建的 `hwjob-*` 单元,不接管现有 crontab、系统 timer 或其他面板任务。 + +## 9. 常用维护命令 + +```bash +# 查看网页服务与任务服务 +systemctl status hardware-dashboard.service hardware-taskd.service + +# 查看任务管理服务自身报错 +journalctl -u hardware-taskd.service -n 50 --no-pager + +# 查看本模块的全部系统定时器 +systemctl list-timers --all 'hwjob-*.timer' + +# 查看某个任务的系统层日志,任务 ID 可从页面或数据库查出 +# 将下方 32 位 ID 替换成实际值,不要把说明文字直接输入命令 +systemctl status hwjob-0123456789abcdef0123456789abcdef.service + +# 重启网页服务:不会取消已启用的任务 +systemctl restart hardware-dashboard.service + +# 重启任务管理服务:已有网页登录会失效,系统定时器仍独立运行 +systemctl restart hardware-taskd.service +``` + +忘记任务管理密码时,可通过 SSH 以 root 执行以下命令。新密码在交互输入,不放进命令行参数: + +```bash +python3 /opt/hardware-dashboard/taskd.py password +systemctl restart hardware-taskd.service +``` + +只停止网页或任务管理进程,不等于停用计划。要停止后续执行,应先在页面将对应任务停用;需要立即中断时再使用“停止本次”。 + +## 10. 备份与回退 + +本次更新前的网页代码备份:`/opt/hardware-dashboard-backup.vNWXhLKD/`。 + +任务数据库与监控数据库分开。备份正在运行的 SQLite 应使用 SQLite 的备份接口,不能只复制主文件而忽略 WAL。备份任务数据库、管理密码散列和脚本/日志时,应继续保持 root 专用权限。 + +如需回退界面,先在页面停用所有不再需要的任务,再停止任务管理服务并恢复代码。只恢复旧网页代码不会自动删除或停用已经生成的系统定时器。 + +## 11. 已完成的验证 + +### 自动化测试 + +16 项本地和服务器测试通过,覆盖: + +- 原监控数据、问题合并、时钟回拨及过期状态。 +- 每日/每周/每月/间隔周期与 Cron 字段解析。 +- Cron 日期和星期的 OR 语义、非法范围和步长。 +- 任务名称与脚本内容不会成为 systemd 单元配置指令。 +- 未登录和错误 CSRF 不会调用特权任务操作。 +- 修改密码使所有旧会话失效。 +- 日志读取量上限与路径编号验证。 + +### 服务器真实执行验证 + +使用明确命名的“部署验证”任务,确认了: + +| 验证内容 | 实际结果 | +|---|---| +| 手动打印命令 | 成功,退出码 0。 | +| 指定日期自动触发 | 在计划时间由 systemd 自动执行,记录为 `scheduled`,退出码 0。 | +| Python 脚本内容 | 成功,输出 `PYTHON_TASK_OK`。 | +| 服务器已有文件执行 | 成功,输出 `EXISTING_FILE_OK`。 | +| 主动返回退出码 7 | 正确显示失败,保留错误输出。 | +| 10 秒超时 | 约 10 秒后停止,显示超时。 | +| 手动停止等待任务 | 正确停止并记录状态。 | +| 编辑、启用和停用 | 页面与系统定时器状态一致。 | +| 未登录、错误来源、错误 CSRF | 分别被 HTTP 401/403 拒绝。 | + +这些测试仅打印信息或短暂等待,没有执行重启、关机、证书续签或修改现有业务配置。测试计划在交付前移除,历史中保留有“部署验证”字样的执行记录。 diff --git a/docs/REPOSITORY.md b/docs/REPOSITORY.md new file mode 100644 index 0000000..c30754f --- /dev/null +++ b/docs/REPOSITORY.md @@ -0,0 +1,26 @@ +# Git 提交说明 + +仓库已按可发布项目结构整理。源代码和部署模板可提交;运行状态、日志、数据库和管理凭据不应提交。 + +## 提交前检查 + +``` +python -m unittest discover -s tests -p "test_*.py" -v +python -m py_compile server.py task_core.py taskd.py task_exec.py scripts/hwlog +``` + +确认没有新增以下内容: + +- /var/lib/hardware-dashboard/monitor.sqlite3 +- /var/log/hardware-watch/*.jsonl +- /etc/hardware-task-admin.json +- 管理员密码、SSH 密码、私钥或真实硬件 UUID + +真实配置文件 config/hardware-watch.json 已被 .gitignore 忽略。仓库只保留 config/hardware-watch.example.json。 + +## 初始化远端 + +``` +git remote add origin <你的仓库地址> +git push -u origin main +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..65576f3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "hardware-dashboard" +version = "1.2.0" +description = "LAN hardware monitoring dashboard with authenticated scheduled tasks" +requires-python = ">=3.12" +dependencies = [] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/scripts/hwlog b/scripts/hwlog new file mode 100755 index 0000000..6635edb --- /dev/null +++ b/scripts/hwlog @@ -0,0 +1,438 @@ +#!/usr/bin/python3 +"""Read-only hardware sampling and bounded, persistent event summaries.""" +import argparse +import csv +import datetime as dt +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import time + +BASE = Path('/var/lib/hardware-watch') +LOG = Path('/var/log/hardware-watch') +CONFIG = Path('/etc/hardware-watch.json') +VERSION = '1.0' +PATTERN = (r'NVRM|NVIDIA|AER:|PCIe Bus Error|BadTLP|BadDLLP|' + r'EDAC|Machine check|Hardware Error|mce:|' + r'I/O error|Buffer I/O|EXT4-fs error|BTRFS.*error|' + r'nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail|reset)|' + r'critical temperature|temperature above threshold|thermal.*(thrott|critical)') +NAMES = { + 'power': '显卡辅助供电检测失败', 'gpu_init': '显卡驱动初始化失败', + 'pcie_fatal': 'PCIe 不可纠正错误', 'pcie_correctable': 'PCIe 可纠正通信错误', + 'gpu_xid': 'NVIDIA Xid 错误', 'memory_cpu': 'CPU/内存硬件报错', + 'storage': '磁盘或存储通信错误', 'thermal': '硬件温度告警', + 'gpu_missing': 'PCIe 显卡数量不足', 'gpu_driver': '显卡管理接口异常', + 'gpu_identity': '预期显卡 UUID 缺失', 'gpu_temp': '显卡温度超过设定阈值', + 'disk_missing': '预期磁盘缺失', 'smart': '磁盘 SMART 健康告警', + 'sensor_temp': '传感器温度告警', 'journal': '系统日志采集异常', + 'monitor': '硬件日志采集器', 'boot': '系统启动', + 'clock': '系统时钟发生明显变化', +} + +def stamp(): + return dt.datetime.now().astimezone().isoformat(timespec='seconds') + +def read(path, default=''): + try: + return Path(path).read_text().strip() + except OSError: + return default + +def load(path, default): + try: + return json.loads(Path(path).read_text()) + except (OSError, ValueError): + return default + +def save(path, obj): + tmp = Path(str(path) + '.tmp') + tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2) + '\n') + os.chmod(tmp, 0o640) + os.replace(tmp, path) + +def command(args, timeout=12): + try: + p = subprocess.run(args, text=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout, env={**os.environ, 'LC_ALL': 'C'}) + return p.returncode, p.stdout, p.stderr + except (OSError, subprocess.TimeoutExpired) as exc: + return 124, '', str(exc) + +def emit(kind, severity='INFO', **values): + row = {'observed_at': stamp(), 'severity': severity, 'category': kind, + 'summary': NAMES.get(kind, kind), **values} + path = LOG / ('events-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl') + with path.open('a', encoding='utf-8') as f: + f.write(json.dumps(row, ensure_ascii=False) + '\n') + os.chmod(path, 0o640) + +def classify(message): + low = message.lower() + if 'necessary power cables' in low: + return 'power', 'ERROR' + if 'rminitadapter failed' in low or 'rm_init_adapter failed' in low: + return 'gpu_init', 'ERROR' + if 'no nvidia gpu found' in low: + return 'gpu_missing', 'WARN' + if 'fallen off' in low or re.search(r'\bXid\b', message): + return 'gpu_xid', 'ERROR' + if any(x in low for x in ('severity=uncorrectable', 'aer: device recovery failed', 'non-functional downstream link')): + return 'pcie_fatal', 'ERROR' + if 'severity=correctable' in low or 'badtlp' in low or 'baddllp' in low or re.search(r'\[\s*12\]\s+Timeout', message): + return 'pcie_correctable', 'WARN' + if re.search(r'machine check|hardware error|edac.*(\b[cu]e\b|error)|mce:.*(error|bank)', low): + return 'memory_cpu', 'ERROR' + if re.search(r'i/o error|buffer i/o|ext4-fs error|btrfs.*error|nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail)', low): + return 'storage', 'ERROR' + if re.search(r'critical temperature|temperature above threshold|thermal.*(thrott|critical)', low): + return 'thermal', 'WARN' + return None + +def kernel_events(state, cfg): + args = ['journalctl', '--no-pager', '-o', 'json', '_TRANSPORT=kernel', + '--grep=' + PATTERN, '--case-sensitive=no', '--lines=+20000'] + cursor = state.get('cursor') + args.append('--after-cursor=' + cursor if cursor else '--since=' + cfg.get('history_since', '7 days ago')) + code, out, err = command(args, 22) + if code not in (0, 1): + state['journal_error'] = err[:1000] or f'journalctl exit {code}' + emit('journal', 'ERROR', detail=err[:1000], action='下次继续重试;原始 journal 未删除。') + if 'cursor' in err.lower() and 'cursor' in state: + state.pop('cursor') + return + state.pop('journal_error', None) + state['journal_last_success'] = stamp() + entries = [] + for line in out.splitlines(): + try: + entries.append(json.loads(line)) + except ValueError: + continue + # A journal with a backwards clock step can replay records before its cursor. + recent = state.get('recent_source_ids', []) + seen = set(recent) + def source_id(entry): + return hashlib.sha256(entry.get('__CURSOR', '').encode()).hexdigest()[:24] + if not recent and cursor and entries and entries[-1].get('__CURSOR') == cursor: + recent = [source_id(entry) for entry in entries] + seen = set(recent) + emit('clock', 'WARN', detail='检测到历史校时造成 journal 游标重放;已跳过此前处理过的记录,避免误计为新故障。') + groups = {} + rows = 0 + for entry in entries: + rows += 1 + state['cursor'] = entry.get('__CURSOR', state.get('cursor')) + token = source_id(entry) + if token in seen: + continue + seen.add(token) + recent.append(token) + message = entry.get('MESSAGE', '') + if not isinstance(message, str): + continue + result = classify(message) + if not result: + continue + category, severity = result + match = re.search(r'(?:[0-9a-f]{4}:)?[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]', message, re.I) + device = match.group(0) if match else '' + normalized = re.sub(r'0x[0-9a-f]+|\b[0-9]+\b', '#', message, flags=re.I) + key = (entry.get('_BOOT_ID', ''), category, device, normalized) + realtime = int(entry.get('__REALTIME_TIMESTAMP', 0)) / 1000000 + when = dt.datetime.fromtimestamp(realtime).astimezone().isoformat(timespec='seconds') + observed = stamp() + monotonic = entry.get('__MONOTONIC_TIMESTAMP') + group = groups.setdefault(key, {'category': category, 'severity': severity, + 'source': 'kernel', 'device': device, 'boot_id': entry.get('_BOOT_ID', ''), + 'first_at': when, 'last_at': when, 'first_monotonic_us': entry.get('__MONOTONIC_TIMESTAMP'), + 'last_monotonic_us': monotonic, 'first_observed_at': observed, + 'last_observed_at': observed, 'count': 0, 'detail': message[:1600]}) + group['count'] += 1 + group['last_at'] = when + group['last_monotonic_us'] = monotonic + group['last_observed_at'] = observed + for row in groups.values(): + emit(row.pop('category'), row.pop('severity'), **row) + state['recent_source_ids'] = recent[-4096:] + pending = rows >= 20000 + if pending and not state.get('journal_backlog'): + emit('journal', 'WARN', detail='本轮读取 20000 条匹配记录;下轮从游标继续处理积压,未跳过中间记录。') + elif not pending and state.get('journal_backlog'): + emit('journal', 'INFO', detail='历史日志积压已处理完成,后续按游标增量采集。') + state['journal_backlog'] = pending + +def pci_gpus(): + items = [] + for path in sorted(Path('/sys/bus/pci/devices').glob('*')): + if read(path / 'vendor') != '0x10de' or not read(path / 'class').startswith('0x03'): + continue + driver = path / 'driver' + items.append({'pci': path.name, 'device_id': read(path / 'device'), + 'driver': driver.resolve().name if driver.exists() else None, + 'link_speed': read(path / 'current_link_speed'), 'link_width': read(path / 'current_link_width')}) + return items + +def gpu_query(): + args = ['nvidia-smi', '--query-gpu=pci.bus_id,name,uuid,temperature.gpu,power.draw,memory.total', + '--format=csv,noheader,nounits'] + code, out, err = command(args) + result = [] + if code == 0: + for row in csv.reader(out.splitlines(), skipinitialspace=True): + if len(row) != 6: + continue + try: + result.append({'pci': row[0], 'name': row[1], 'uuid': row[2], + 'temperature_c': float(row[3]), 'power_w': float(row[4]), 'memory_mib': float(row[5])}) + except ValueError: + result.append({'pci': row[0], 'name': row[1], 'uuid': row[2], 'raw': row}) + return result, None if code == 0 and result else (err or out or 'nvidia-smi 未返回设备')[:1200] + +def disk_names(): + return sorted(p.name for p in Path('/sys/block').glob('*') + if not p.name.startswith(('loop', 'ram', 'zram', 'dm-', 'md'))) + +def sensors(): + result = [] + for path in sorted(Path('/sys/class/hwmon').glob('hwmon*')): + for inp in path.glob('temp*_input'): + try: + prefix = inp.name[:-6] + result.append({'chip': read(path / 'name'), 'label': read(path / (prefix + '_label'), prefix), + 'celsius': int(read(inp)) / 1000, + 'critical_c': int(read(path / (prefix + '_crit'), '0')) / 1000}) + except ValueError: + pass + return result + +def active_problem(problems, key, kind, detail, severity='ERROR'): + problems[key] = {'category': kind, 'severity': severity, 'detail': detail} + +def smart_checks(state, problems, disks): + # Hourly, read-only SMART query. Never starts self-tests or changes disk settings. + cached = state.get('smart', {}) + if time.time() - state.get('smart_at', 0) >= 3600: + cached = {} + for name in disks: + code, out, err = command(['smartctl', '-a', '-j', '/dev/' + name], 7) + try: + data = json.loads(out) + except ValueError: + cached[name] = {'available': False, 'detail': (err or out)[:500]} + continue + nv = data.get('nvme_smart_health_information_log', {}) + cached[name] = {'available': 'smart_status' in data, + 'passed': data.get('smart_status', {}).get('passed'), + 'critical_warning': nv.get('critical_warning'), 'media_errors': nv.get('media_errors'), + 'temperature_c': data.get('temperature', {}).get('current'), + 'percentage_used': nv.get('percentage_used'), 'exit_status': code} + state['smart'], state['smart_at'] = cached, time.time() + for name, item in cached.items(): + if not item.get('available'): + active_problem(problems, 'smart-unavailable:' + name, 'smart', + f'{name}: SMART 读取不可用,当前仅监控内核存储报错。{item.get("detail", "")}', 'WARN') + elif item.get('passed') is False or item.get('critical_warning', 0): + active_problem(problems, 'smart:' + name, 'smart', f'{name}: {json.dumps(item, ensure_ascii=False)}') + previous = state.get('smart_media', {}).get(name) + media = item.get('media_errors') + if media is not None and previous is not None and media > previous: + emit('smart', 'ERROR', detail=f'{name}: NVMe 介质错误计数从 {previous} 增加到 {media}') + if media is not None: + state.setdefault('smart_media', {})[name] = media + return cached + +def collect(state, cfg): + now = stamp() + boot = read('/proc/sys/kernel/random/boot_id') + clock = {'wall': time.time(), 'monotonic': time.monotonic(), 'boot_id': boot} + old_clock = state.get('clock', {}) + if old_clock.get('boot_id') == boot: + delta = (clock['wall'] - old_clock['wall']) - (clock['monotonic'] - old_clock['monotonic']) + if abs(delta) > 60: + emit('clock', 'WARN', detail=f'同一次启动中墙上时钟相对单调时钟变化 {delta:.1f} 秒;核对事件时间时应结合启动 ID 和单调时钟。') + state['clock'] = clock + if boot != state.get('boot_id'): + emit('boot', boot_id=boot, previous_boot_id=state.get('boot_id'), + detail='监控首次观察到本次启动;准确启动信息可通过 journalctl 查看。') + state['boot_id'] = boot + state['smart_at'] = 0 + kernel_events(state, cfg) + pci = pci_gpus() + gpu, nv_error = gpu_query() if pci else ([], 'PCIe 总线没有 NVIDIA 显卡') + disks = disk_names() + temps = sensors() + problems = {} + if state.get('journal_error'): + active_problem(problems, 'journal-read', 'journal', state['journal_error'], 'WARN') + if len(pci) < cfg['expected_gpu_count']: + active_problem(problems, 'gpu-count', 'gpu_missing', + f'预期 {cfg["expected_gpu_count"]} 块 NVIDIA 显卡,PCIe 实际 {len(pci)} 块。') + if nv_error or len(gpu) < len(pci): + active_problem(problems, 'gpu-driver', 'gpu_driver', nv_error or f'PCIe {len(pci)} 块,nvidia-smi 仅 {len(gpu)} 块。') + if not nv_error: + missing = sorted(set(cfg.get('expected_gpu_uuids', [])) - {x['uuid'] for x in gpu}) + if missing: + active_problem(problems, 'gpu-uuid', 'gpu_identity', '缺少预期 GPU:' + ', '.join(missing)) + for item in gpu: + if item.get('temperature_c', 0) >= cfg.get('gpu_warn_c', 85): + active_problem(problems, 'gpu-temp:' + item['uuid'], 'gpu_temp', + f'{item["name"]} {item["pci"]}: {item["temperature_c"]}°C', 'WARN') + for name in set(cfg.get('expected_disks', [])) - set(disks): + active_problem(problems, 'disk:' + name, 'disk_missing', name) + for item in temps: + if item['critical_c'] > 0 and item['celsius'] >= item['critical_c']: + active_problem(problems, 'sensor:' + item['chip'] + item['label'], 'sensor_temp', + json.dumps(item, ensure_ascii=False), 'ERROR') + smart = smart_checks(state, problems, disks) + previous = state.get('active', {}) + for key, value in problems.items(): + old = previous.get(key) + value['since'] = old.get('since', now) if old else now + if not old or old.get('detail') != value['detail']: + emit(value['category'], value['severity'], event='FAULT', key=key, detail=value['detail'], since=value['since']) + for key, value in previous.items(): + if key not in problems: + emit(value['category'], 'INFO', event='RECOVERED', key=key, detail='此前异常在本轮检查中已消失。', since=value.get('since')) + state['active'] = problems + status = {'checked_at': now, 'boot_id': boot, 'pci_gpus': pci, 'gpus': gpu, + 'nvidia_error': nv_error, 'disks': disks, 'smart': smart, 'sensors': temps, 'active': problems, + 'journal_last_success': state.get('journal_last_success'), 'journal_backlog': state.get('journal_backlog', False)} + save(BASE / 'status.json', status) + if time.time() - state.get('snapshot_at', 0) >= 900: + path = LOG / ('samples-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl') + with path.open('a', encoding='utf-8') as f: + f.write(json.dumps(status, ensure_ascii=False) + '\n') + os.chmod(path, 0o640) + state['snapshot_at'] = time.time() + save(BASE / 'state.json', state) + # Age and total-size limits apply only to this monitor's own log files. + files = sorted(LOG.glob('*.jsonl'), key=lambda p: p.stat().st_mtime) + cutoff = time.time() - cfg.get('retention_days', 90) * 86400 + total = sum(p.stat().st_size for p in files) + for path in files: + if path.stat().st_mtime < cutoff or total > 256 * 1024 * 1024: + if path.name.endswith(dt.datetime.now().strftime('%Y-%m-%d') + '.jsonl'): + continue + total -= path.stat().st_size + path.unlink() + +def show_status(): + status = load(BASE / 'status.json', {}) + print('硬件管理日志 | 最近检查:' + status.get('checked_at', '尚未运行')) + age = (dt.datetime.now().astimezone() - dt.datetime.fromisoformat(status['checked_at'])).total_seconds() if status else 9999 + if age > 180: + print('警告:最近检查超过 3 分钟,请检查 systemctl status hardware-watch.timer hardware-watch.service') + for item in status.get('pci_gpus', []): + print(f'PCIe {item["pci"]} | {item["link_speed"]} x{item["link_width"]} | {item["driver"]}') + for item in status.get('gpus', []): + print(f'{item["name"]} | {item["pci"]} | {item.get("temperature_c", "?")}°C | {item.get("power_w", "?")}W | {item["uuid"]}') + for name, item in status.get('smart', {}).items(): + print(f'磁盘 {name} | SMART={item.get("passed")} | 温度={item.get("temperature_c")}°C | 介质错误={item.get("media_errors")}') + print('当前采样告警:' + str(len(status.get('active', {})))) + for value in status.get('active', {}).values(): + print(f' [{value["severity"]}] {NAMES[value["category"]]}:{value["detail"]}') + if status.get('journal_backlog'): + print('历史内核日志正在补录,实时 GPU/磁盘采样已启用。') + print('PCIe 等内核历史事件另见 hwlog events;当前在线不代表此前没有通信错误。') + +def show_events(args): + rows = [] + seen = set() + status = load(BASE / 'status.json', {}) + active = status.get('active', {}) + since = dt.datetime.fromisoformat(args.since).astimezone() if args.since else None + for path in sorted(LOG.glob('events-*.jsonl')): + for line in path.read_text().splitlines(): + try: + row = json.loads(line) + if row.get('source') == 'kernel': + key = (row.get('boot_id'), row.get('category'), row.get('device'), + row.get('first_monotonic_us'), row.get('last_monotonic_us'), row.get('detail')) + if key in seen: + continue + seen.add(key) + event_time = row.get('first_at', row['observed_at']) + source_last = row.get('last_at', event_time) + observed = row.get('observed_at', event_time) + if since and max(source_last, observed) < since.isoformat(): + continue + if args.category and row['category'] != args.category: + continue + rows.append(row) + except (ValueError, KeyError): + continue + # Recently collected records remain visible even if old source clocks were ahead. + rows.sort(key=lambda x: x['observed_at']) + for row in rows[-max(1, args.limit):]: + if args.json: + print(json.dumps(row, ensure_ascii=False)) + else: + print(f'{row.get("first_at", row["observed_at"])} [{row["severity"]}] {row["summary"]} {row.get("device", "")} {row.get("event", "")}') + if 'count' in row: + print(f' 原始故障时间:{row.get("first_at", "未知")} 至 {row.get("last_at", "未知")}') + print(f' 监控发现时间:{row.get("first_observed_at", row["observed_at"])} 至 {row.get("last_observed_at", row["observed_at"])}') + first_mono = row.get('first_monotonic_us') + last_mono = row.get('last_monotonic_us') + if first_mono and last_mono: + print(f' 本次启动经过:{int(first_mono)/1000000:.3f}s 至 {int(last_mono)/1000000:.3f}s;启动 {row.get("boot_id", "")}') + print(f' 本批重复次数:{row["count"]}') + current = False + for value in active.values(): + detail = value.get('detail', '') + if value.get('category') == row.get('category') and (not row.get('device') or row.get('device') in detail): + current = True + break + print(' 当前判断:当前采样仍有同类告警' if current else ' 当前判断:当前采样未复现,属于历史记录或已恢复') + if row['last_at'] < row['first_at']: + print(' 时间提示:源日志时钟发生回拨,以上首末时间不能按日历顺序判断;请结合启动 ID、单调时钟及原始 journal。') + if row['first_at'] > row['observed_at']: + print(' 时间提示:源日志日期晚于采集日期,源系统时钟曾异常;采集时间为 ' + row['observed_at']) + print(' ' + row.get('detail', '')) + +def main(): + parser = argparse.ArgumentParser(description='硬件管理日志:每分钟采样,查看状态和历史故障。') + sub = parser.add_subparsers(dest='action') + sub.add_parser('collect') + sub.add_parser('status') + events = sub.add_parser('events') + events.add_argument('--since', help='例如 2026-11-02 或 2026-11-02T12:00:00+08:00') + events.add_argument('--category', choices=sorted(NAMES)) + events.add_argument('--limit', type=int, default=50) + events.add_argument('--json', action='store_true') + sub.add_parser('self-test') + args = parser.parse_args() + if args.action == 'self-test': + cases = [('NVRM: GPU does not have the necessary power cables connected.', 'power'), + ('NVRM: RmInitAdapter failed! (0x24:0x1c:1603)', 'gpu_init'), + ('PCIe Bus Error: severity=Correctable, type=Data Link Layer', 'pcie_correctable'), + ('PCIe Bus Error: severity=Uncorrectable (Fatal)', 'pcie_fatal'), + ('NVRM: Xid (PCI:0000:03:00): 79, GPU has fallen off the bus.', 'gpu_xid'), + ('nvme nvme0: I/O 10 timeout, reset controller', 'storage'), + ('EDAC MC0: 1 CE memory read error', 'memory_cpu')] + for message, expected in cases: + assert classify(message)[0] == expected, message + assert classify('NVRM: loading NVIDIA UNIX x86_64 Kernel Module') is None + assert classify('nvidia: module verification failed: tainting kernel') is None + print('PASS: 7 hardware error cases and 2 benign driver messages') + return + if args.action == 'collect': + BASE.mkdir(parents=True, exist_ok=True) + LOG.mkdir(parents=True, exist_ok=True) + with (BASE / 'lock').open('w') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + collect(load(BASE / 'state.json', {}), load(CONFIG, {})) + elif args.action == 'events': + show_events(args) + else: + show_status() + +if __name__ == '__main__': + main() diff --git a/server.py b/server.py new file mode 100644 index 0000000..d9d6cc5 --- /dev/null +++ b/server.py @@ -0,0 +1,730 @@ +#!/usr/bin/python3 +"""LAN-only, read-only hardware dashboard. Python 3.12 standard library.""" +import argparse +import csv +from contextlib import contextmanager +import datetime as dt +import hashlib +import io +import json +import logging +import math +import os +from pathlib import Path +import re +import signal +import socket +from http.cookies import SimpleCookie, CookieError +import sqlite3 +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + +ROOT = Path(__file__).resolve().parent +DATA = Path(os.environ.get('DASHBOARD_DATA', '/var/lib/hardware-dashboard')) +LOGS = Path(os.environ.get('HARDWARE_LOGS', '/var/log/hardware-watch')) +HWSTATE = Path(os.environ.get('HARDWARE_STATE', '/var/lib/hardware-watch/status.json')) +CONFIG = Path(os.environ.get('HARDWARE_CONFIG', '/etc/hardware-watch.json')) +INTERVAL = 10 +KEEP_DAYS = 30 +TZ = dt.timezone(dt.timedelta(hours=8)) +STOP = threading.Event() +LOCK = threading.Lock() +LATEST = {} +TASK_SOCKET='/run/hardware-taskd/control.sock' +RANGES = {'1h': 3600, '6h': 21600, '24h': 86400, '7d': 604800, '30d': 2592000} +CATEGORY = { + 'power': ('显卡辅助供电异常', '显卡检测不到必要的外接供电。', '正常关机并断电后,检查该卡供电线两端、转接头和退针情况;仍异常时换用匹配的已知正常供电线测试。'), + 'gpu_init': ('显卡初始化失败', '系统发现了显卡,但驱动未能让它进入可用状态。', '先查看同一时段是否有供电或 PCIe 错误,再检查供电与插槽;不要直接反复重装驱动。'), + 'gpu_driver': ('显卡状态读取失败', '管理接口没有返回全部预期显卡。', '对照供电、PCIe 和驱动日志排查;查看下方原始错误。'), + 'gpu_missing': ('显卡未被发现', '实际发现的显卡少于预期,或内核报告未找到 NVIDIA GPU。', '先确认是否主动拆卡;否则正常关机断电后检查供电与插槽。'), + 'gpu_identity': ('预期显卡缺失', '当前显卡身份与监控基准不一致。', '若主动更换了显卡,更新监控基准;否则检查对应显卡是否在线。'), + 'gpu_temp': ('显卡温度偏高', '显卡温度超过配置的告警线。', '检查机箱风道和风扇,必要时降低计算负载;P100 依赖机箱送风。'), + 'pcie_correctable': ('显卡与主板通信需要重试', '显卡与主板传数据时出现了系统可以自动纠正的通信错误。这条告警本身不表示显卡损坏。', '目前显卡在线时可以继续使用并观察。如果持续报错、任务变慢或显卡消失,安排空闲时正常关机并断电,检查显卡供电线和插槽,再做换线或换槽测试。'), + 'pcie_fatal': ('PCIe 严重通信错误', '硬件连接出现不可纠正错误,可能导致设备不可用。', '先保存日志;正常关机断电后检查显卡供电、插槽及转接设备。'), + 'gpu_xid': ('显卡运行错误', 'NVIDIA 驱动记录了 GPU 运行异常。', '结合原始 Xid 编号与同一时间的温度、供电、PCIe 日志排查。'), + 'memory_cpu': ('CPU/内存硬件告警', '内核报告了机器检查或内存纠错事件。', '保留原始槽位与错误信息,由维护人员检查对应内存、CPU 和主板。'), + 'storage': ('磁盘读写/通信异常', '内核记录了存储读写错误、超时或重置。', '先确认重要数据已有备份,再检查 SMART 状态、硬盘连接和供电。'), + 'smart': ('磁盘健康告警', '磁盘 SMART 健康检查异常或无法读取。', '查看具体健康项;若出现介质错误或健康失败,优先备份并安排检修。'), + 'disk_missing': ('预期磁盘缺失', '监控没有发现预期磁盘。', '确认是否主动移除;否则断电检查连接并安排维护。'), + 'thermal': ('硬件温度告警', '内核报告过热或温度相关降速。', '检查散热器、风扇和机箱风道,必要时降低负载。'), + 'sensor_temp': ('传感器温度告警', '硬件传感器温度达到告警阈值。', '检查对应 CPU、磁盘或机箱散热。'), + 'clock': ('系统时间有变化', '历史系统时间曾跳变,原始时间可能不可靠。', '对照采集时间、启动 ID 和启动后经过时间;带“估算”的时刻不能当作精确时间。'), + 'journal': ('日志采集状态', '硬件日志采集器记录了自身的运行状态。', '若持续采集失败或状态超过三分钟未更新,请维护人员检查 hardware-watch 服务。'), + 'boot': ('系统启动记录', '采集器观察到了一个新的系统启动。', '如果不是计划内重启,查看此前是否有断电、温度或硬件异常。'), + 'monitor': ('监控运行记录', '监控服务自身的状态变化。', '查看详细信息。'), + 'web_monitor': ('网页采集异常', '网页无法取得最新硬件数据。', '检查监控服务是否正常,不要把旧数据当成实时状态。'), + 'memory_pressure': ('可用内存偏少', '系统可用内存不足总量的 5%。', '检查正在运行的任务,减少不必要的并发或安排内存扩容。'), +} + +def read(path, default=''): + try: + return Path(path).read_text().strip() + except OSError: + return default + +def read_json(path, default=None): + try: + return json.loads(read(path)) + except (ValueError, TypeError): + return {} if default is None else default + +def iso(ts=None): + return dt.datetime.fromtimestamp(time.time() if ts is None else ts, TZ).isoformat(timespec='seconds') + +def epoch(value): + try: + return dt.datetime.fromisoformat(value).timestamp() + except (TypeError, ValueError): + return None + +def run(args, timeout=6): + try: + p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, + env={**os.environ, 'LC_ALL': 'C'}) + return p.returncode, p.stdout, p.stderr + except (OSError, subprocess.TimeoutExpired) as exc: + return 124, '', str(exc) + +@contextmanager +def connect(): + db = sqlite3.connect(DATA / 'monitor.sqlite3', timeout=8) + db.row_factory = sqlite3.Row + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + +def initialize(): + DATA.mkdir(parents=True, exist_ok=True) + with connect() as db: + db.execute('PRAGMA journal_mode=WAL') + db.executescript(''' + CREATE TABLE IF NOT EXISTS metrics ( + ts INTEGER PRIMARY KEY, cpu REAL, memory REAL, cpu_temp REAL, + disk_read REAL, disk_write REAL, net_rx REAL, net_tx REAL, + gpus TEXT NOT NULL, source TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, occurred REAL NOT NULL, observed REAL NOT NULL, + category TEXT, severity TEXT, raw TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS event_time ON events(occurred DESC); + CREATE INDEX IF NOT EXISTS event_observed ON events(observed DESC); + CREATE INDEX IF NOT EXISTS event_category ON events(category, observed DESC); + CREATE INDEX IF NOT EXISTS metric_source ON metrics(source, ts); + CREATE TABLE IF NOT EXISTS imports (file TEXT PRIMARY KEY, inode TEXT, offset INTEGER); + CREATE TABLE IF NOT EXISTS boot_clock (boot TEXT PRIMARY KEY, anchor REAL, mono REAL); + CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); + ''') + +def event_id(row): + if row.get('source') == 'kernel': + value = [row.get(k) for k in ('boot_id', 'category', 'device', 'first_monotonic_us', 'last_monotonic_us', 'detail')] + else: + value = [row.get(k) for k in ('observed_at', 'category', 'key', 'event', 'detail')] + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + +def import_logs(db): + for path in sorted(LOGS.glob('*.jsonl')): + if not path.name.startswith(('events-', 'samples-')): + continue + stat = path.stat() + checkpoint = db.execute('SELECT inode,offset FROM imports WHERE file=?', (path.name,)).fetchone() + offset = checkpoint['offset'] if checkpoint and checkpoint['inode'] == str(stat.st_ino) and checkpoint['offset'] <= stat.st_size else 0 + if offset == stat.st_size: + continue + with path.open('rb') as f: + f.seek(offset) + for _ in range(5000): + pos = f.tell() + line = f.readline() + if not line or not line.endswith(b'\n'): + f.seek(pos) + break + try: + row = json.loads(line) + except (ValueError, UnicodeDecodeError): + continue + if path.name.startswith('samples-'): + ts = epoch(row.get('checked_at')) + if ts is None or ts < time.time() - KEEP_DAYS * 86400: + continue + gpus = [{'uuid': x.get('uuid'), 'name': x.get('name'), 'pci': x.get('pci'), + 'temperature_c': x.get('temperature_c'), 'power_w': x.get('power_w'), + 'utilization': None, 'memory_used_mib': None, + 'memory_total_mib': x.get('memory_mib')} for x in row.get('gpus', [])] + db.execute('INSERT OR IGNORE INTO metrics VALUES(?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,?,?)', + (int(ts), json.dumps(gpus), 'legacy')) + else: + observed = epoch(row.get('observed_at')) + first = epoch(row.get('first_at')) or observed + if observed is None: + continue + last = epoch(row.get('last_at')) + mono = row.get('last_monotonic_us') + if last and mono and last <= observed + 60 and row.get('boot_id'): + m = int(mono) / 1e6 + db.execute('''INSERT INTO boot_clock VALUES(?,?,?) ON CONFLICT(boot) DO UPDATE + SET anchor=excluded.anchor,mono=excluded.mono WHERE excluded.mono > boot_clock.mono''', + (row['boot_id'], last - m, m)) + db.execute('INSERT OR IGNORE INTO events VALUES(?,?,?,?,?,?)', + (event_id(row), first, observed, row.get('category', 'monitor'), + row.get('severity', 'INFO'), json.dumps(row, ensure_ascii=False))) + offset = f.tell() + db.execute('INSERT OR REPLACE INTO imports VALUES(?,?,?)', (path.name, str(stat.st_ino), offset)) + +def temperatures(): + result = [] + for path in Path('/sys/class/hwmon').glob('hwmon*'): + chip = read(path / 'name') + for inp in path.glob('temp*_input'): + prefix = inp.name[:-6] + try: + result.append({'chip': chip, 'label': read(path / (prefix + '_label'), prefix), + 'value': int(read(inp)) / 1000}) + except ValueError: + pass + return result + +def cpu_counters(): + fields = list(map(int, read('/proc/stat').splitlines()[0].split()[1:9])) + return sum(fields), fields[3] + fields[4] + +def counters(): + net_rx = net_tx = disk_read = disk_write = 0 + for line in read('/proc/net/dev').splitlines()[2:]: + name, values = line.split(':', 1) + if name.strip() == 'lo': + continue + fields = values.split() + net_rx += int(fields[0]); net_tx += int(fields[8]) + disks = [] + for path in Path('/sys/block').glob('*'): + if path.name.startswith(('loop', 'ram', 'zram', 'dm-', 'md')): + continue + fields = read(path / 'stat').split() + if len(fields) >= 7: + disk_read += int(fields[2]) * 512; disk_write += int(fields[6]) * 512 + disks.append(path.name) + return {'net_rx': net_rx, 'net_tx': net_tx, 'disk_read': disk_read, 'disk_write': disk_write}, disks + +def gpu_metrics(): + fields = 'index,uuid,pci.bus_id,name,utilization.gpu,temperature.gpu,power.draw,power.limit,memory.used,memory.total' + code, out, err = run(['nvidia-smi', '--query-gpu=' + fields, '--format=csv,noheader,nounits']) + if code: + return [], (err or out or '无法读取显卡')[:1000] + result = [] + for row in csv.reader(out.splitlines(), skipinitialspace=True): + if len(row) != 10: + continue + item = {'index': row[0], 'uuid': row[1], 'pci': row[2], 'name': row[3]} + for key, value in zip(('utilization', 'temperature_c', 'power_w', 'power_limit_w', 'memory_used_mib', 'memory_total_mib'), row[4:]): + try: + number = float(value) + item[key] = number if math.isfinite(number) else None + except ValueError: + item[key] = None + result.append(item) + return result, None if result else 'NVIDIA 管理接口没有返回可用显卡' + +def issue(category, detail, level='WARN', device=''): + title, meaning, action = CATEGORY.get(category, CATEGORY['monitor']) + return {'category': category, 'title': title, 'meaning': meaning, 'action': action, + 'detail': detail, 'severity': level, 'device': device} + +def pci_address(value): + match = re.search(r'([0-9a-f]{4,8}):([0-9a-f]{2}):([0-9a-f]{2}\.[0-7])$', value or '', re.I) + if match: + return f'{int(match[1],16):04x}:{match[2].lower()}:{match[3].lower()}' + return value or '' + +def device_map(gpus): + """Map current GPU endpoints and their actual sysfs upstream bridges to one identity.""" + result = {} + for index, gpu in enumerate(sorted(gpus,key=lambda x:x['uuid'])): + pci = pci_address(gpu['pci']) + label = f'第 {index+1} 张 P100' if 'P100' in gpu.get('name','') else f'第 {index+1} 张显卡' + entry = {'id':gpu['uuid'], 'label':label, 'pci':pci} + result[pci] = entry + path = Path('/sys/bus/pci/devices') / pci + if path.exists(): + for parent in path.resolve().parents: + if re.fullmatch(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]',parent.name): + result.setdefault(parent.name,entry) + return result + +def event_component(row,current): + address = pci_address(row.get('device','')) + same_boot = row.get('boot_id','').replace('-','') == current.get('boot_id','').replace('-','') + matched = current.get('device_map',{}).get(address) if same_boot else None + if matched: + return matched['id'], matched['label'], address + if row.get('category','').startswith('pcie'): + return address or 'unknown-link', '设备与主板的连接' + (f'(端口 {address[5:]})' if address else ''), address + return address or row.get('key') or 'system', ('设备 '+address[5:]) if address else '这台电脑', address + +def sample(previous): + now = time.time() + mono = time.monotonic() + total, idle = cpu_counters() + cpu = None + if previous and total > previous['cpu_total']: + cpu = max(0, min(100, 100 * (1 - (idle - previous['cpu_idle']) / (total - previous['cpu_total'])))) + mem = {} + for line in read('/proc/meminfo').splitlines(): + key, value = line.split(':', 1) + mem[key] = int(value.strip().split()[0]) * 1024 + ram_total, ram_available = mem['MemTotal'], mem.get('MemAvailable', mem['MemFree']) + count, disks = counters() + rates = {key: None for key in count} + if previous and mono > previous['mono']: + rates = {key: max(0, (value - previous['counters'][key]) / (mono - previous['mono'])) / 1048576 + for key, value in count.items()} + temps = temperatures() + cpu_temp = max([x['value'] for x in temps if x['chip'] == 'coretemp'], default=None) + disk_temp = max([x['value'] for x in temps if x['chip'] == 'nvme'], default=None) + gpu, gpu_error = gpu_metrics() + hw = read_json(HWSTATE) + cfg = read_json(CONFIG) + expected = cfg.get('expected_gpu_count', 2) + faults = [] + if len(gpu) < expected: + faults.append(issue('gpu_missing', f'预期 {expected} 块,当前管理接口可用 {len(gpu)} 块。', 'ERROR')) + if gpu_error: + faults.append(issue('gpu_driver', gpu_error, 'ERROR')) + missing_uuid = set(cfg.get('expected_gpu_uuids', [])) - {x['uuid'] for x in gpu} + if missing_uuid and not gpu_error and len(gpu) >= expected: + faults.append(issue('gpu_identity', '当前显卡 UUID 与预期配置不同。', 'WARN')) + for item in gpu: + if item.get('temperature_c') is not None and item['temperature_c'] >= cfg.get('gpu_warn_c', 85): + faults.append(issue('gpu_temp', f'{item["temperature_c"]}°C,告警线 {cfg.get("gpu_warn_c",85)}°C', 'WARN', item['pci'])) + if cpu_temp is not None and cpu_temp >= 85: + faults.append(issue('sensor_temp', f'CPU 最高温度 {cpu_temp}°C,网页关注线 85°C。')) + if ram_available / ram_total < .05: + faults.append(issue('memory_pressure', f'剩余 {ram_available/1073741824:.1f} GiB。')) + hw_age = now - (epoch(hw.get('checked_at')) or 0) + if hw_age > 180 or hw_age < -60: + faults.append(issue('journal', '硬件日志采集状态超过 3 分钟未更新,或系统时间异常。')) + else: + for value in hw.get('active', {}).values(): + matched = next((x for x in faults if x['category'] == value.get('category')), None) + if matched: + if value.get('severity') == 'ERROR': + matched['severity'] = 'ERROR' + matched['detail'] += ' ' + value.get('detail','') + continue + faults.append(issue(value.get('category', 'monitor'), value.get('detail', ''), value.get('severity', 'WARN'))) + disk_usage = os.statvfs('/var/lib') + used = (disk_usage.f_blocks - disk_usage.f_bfree) * disk_usage.f_frsize + capacity = disk_usage.f_blocks * disk_usage.f_frsize + cpu_name = next((line.split(':', 1)[1].strip() for line in read('/proc/cpuinfo').splitlines() + if line.startswith('model name')), 'CPU') + cpu_sockets = len(set(line.split(':',1)[1].strip() for line in read('/proc/cpuinfo').splitlines() + if line.startswith('physical id'))) + latest = {'ts': now, 'checked_at': iso(now), 'host': socket.gethostname(), 'address': '192.168.1.37', + 'boot_id': read('/proc/sys/kernel/random/boot_id'), 'monotonic_s': mono, + 'uptime_s': float(read('/proc/uptime').split()[0]), 'cpu': cpu, 'cpu_name': cpu_name, + 'cpu_threads': os.cpu_count(), 'cpu_sockets': cpu_sockets, 'cpu_temp': cpu_temp, 'load': list(os.getloadavg()), + 'memory': (1 - ram_available / ram_total) * 100, 'memory_total_gib': ram_total / 1073741824, + 'memory_available_gib': ram_available / 1073741824, 'gpus': gpu, + 'expected_gpu_count': expected, 'gpu_warn_c': cfg.get('gpu_warn_c',85), + 'gpu_error': gpu_error, 'disk_temp': disk_temp, 'disks': disks, 'rates': rates, + 'device_map': device_map(gpu), + 'disk_used_gib': used / 1073741824, 'disk_total_gib': capacity / 1073741824, + 'disk_usage_pct': used / capacity * 100, 'smart': hw.get('smart', {}), + 'hardware_checked_at': hw.get('checked_at'), 'hardware_stale': hw_age > 180 or hw_age < -60, + 'hardware_active': hw.get('active', {}) if 0 <= hw_age <= 180 else {}, + 'issues': faults, 'sensors': temps, 'interval': INTERVAL} + return latest, {'cpu_total': total, 'cpu_idle': idle, 'mono': mono, 'counters': count} + +def sampler(): + previous = None + last_clean = 0 + while not STOP.is_set(): + started = time.monotonic() + try: + current, previous = sample(previous) + with connect() as db: + import_logs(db) + db.execute('INSERT OR REPLACE INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)', + (int(current['ts']), current['cpu'], current['memory'], current['cpu_temp'], + current['rates']['disk_read'], current['rates']['disk_write'], current['rates']['net_rx'], + current['rates']['net_tx'], json.dumps(current['gpus']), 'live')) + db.execute('INSERT OR REPLACE INTO boot_clock VALUES(?,?,?)', + (current['boot_id'].replace('-', ''), current['ts'] - current['monotonic_s'], current['monotonic_s'])) + if time.time() - last_clean >= 3600: + db.execute('DELETE FROM metrics WHERE ts < ?', (time.time() - KEEP_DAYS * 86400,)) + db.execute('DELETE FROM events WHERE observed < ?', (time.time() - 90 * 86400,)) + last_clean = time.time() + current['history_started_at'] = db.execute("SELECT MIN(ts) FROM metrics WHERE source='live'").fetchone()[0] + recent = db.execute('SELECT raw FROM events WHERE observed>? AND category IN (?,?,?)', + (time.time()-180, 'pcie_fatal', 'pcie_correctable', 'power')).fetchall() + for record in recent: + row = json.loads(record['raw']) + source_boot = row.get('boot_id', '').replace('-', '') + source_mono = row.get('last_monotonic_us') + if source_boot != current['boot_id'].replace('-', '') or source_mono is None: + continue + age = current['monotonic_s'] - int(source_mono) / 1e6 + row_component = event_component(row,current)[0] + already_present = any(x['category'] == row['category'] and + event_component({'boot_id':current['boot_id'],'device':x.get('device',''),'category':x['category']},current)[0] == row_component + for x in current['issues']) + if 0 <= age <= 120 and not already_present: + severity = 'WARN' if row['category'] == 'pcie_correctable' else 'ERROR' + report = issue(row['category'], '最近两分钟仍有新的通信报告。' if row['category']=='pcie_correctable' else '最近两分钟仍有新报错。', severity, row.get('device','')) + report['component_label'] = event_component(row,current)[1] + current['issues'].append(report) + with LOCK: + LATEST.clear(); LATEST.update(current) + except Exception: + logging.exception('Sample failed') + with LOCK: + LATEST['collector_error'] = '网页采集失败,请查看服务日志。' + STOP.wait(max(.2, INTERVAL - (time.monotonic() - started))) + +def snapshot(): + with LOCK: + data = json.loads(json.dumps(LATEST)) + data['server_ts'] = time.time() + data['stale'] = not data.get('ts') or time.time() - data.get('ts', 0) > 35 + return data + +def history(range_name): + seconds = RANGES.get(range_name, 86400) + end = int(time.time()); start = end - seconds + step = max(INTERVAL, math.ceil(seconds / 360)) + with connect() as db: + rows = db.execute('''SELECT CAST((ts-?)/? AS INTEGER) AS b,AVG(cpu) cpu,AVG(memory) memory, + AVG(cpu_temp) cpu_temp,AVG(disk_read) disk_read,AVG(disk_write) disk_write, + AVG(net_rx) net_rx,AVG(net_tx) net_tx,SUM(source='live') live + FROM metrics WHERE ts>=? AND ts<=? GROUP BY b ORDER BY b''', (start,step,start,end)).fetchall() + gpu_rows = db.execute('''SELECT CAST((m.ts-?)/? AS INTEGER) AS b, + json_extract(j.value,'$.uuid') uuid, MAX(json_extract(j.value,'$.name')) name, + AVG(json_extract(j.value,'$.utilization')) utilization, + AVG(json_extract(j.value,'$.temperature_c')) temperature_c, + AVG(json_extract(j.value,'$.power_w')) power_w, + AVG(json_extract(j.value,'$.memory_used_mib')) memory_used_mib + FROM metrics m,json_each(m.gpus) j WHERE m.ts>=? AND m.ts<=? + GROUP BY b,uuid ORDER BY b''', (start,step,start,end)).fetchall() + points = {row['b']: {'ts': start + row['b'] * step, **{k: row[k] for k in row.keys() if k != 'b'}, 'gpus': {}} for row in rows} + devices = {} + for row in gpu_rows: + if row['b'] in points and row['uuid']: + points[row['b']]['gpus'][row['uuid']] = {key: row[key] for key in ('utilization','temperature_c','power_w','memory_used_mib')} + devices[row['uuid']] = row['name'] + return {'start': start, 'end': end, 'step': step, 'range': range_name, + 'points': list(points.values()), 'devices': devices} + +def source_times(row, anchors): + observed = epoch(row.get('observed_at')) or time.time() + first = epoch(row.get('first_at')) or observed + last = epoch(row.get('last_at')) or first + adjusted = first > observed + 60 or last < first + note = '' + if adjusted: + boot = row.get('boot_id', '').replace('-', '') + if boot in anchors and row.get('first_monotonic_us') and row.get('last_monotonic_us'): + first = anchors[boot] + int(row['first_monotonic_us']) / 1e6 + last = anchors[boot] + int(row['last_monotonic_us']) / 1e6 + note = '原始系统时间曾异常;发生时间按同次启动的校时记录估算,原始时间保留如下。' + else: + first = last = observed + note = '原始系统时间不可靠,当前按监控读到记录的时间展示,无法确认精确发生时刻。' + return first, last, note + +def merge_incidents(items): + """Group reports, never claim a log line equals an independent hardware failure.""" + buckets = {} + output = [] + for item in sorted(items,key=lambda x:x['first_ts']): + if item['category'] in ('boot','clock','journal','monitor') or item['raw'].get('event') == 'RECOVERED': + item.update(record_count=1, signal_reports=0, raw_examples=[item['raw']]) + output.append(item) + continue + key = (item['category'],item['raw'].get('boot_id','').replace('-',''),item['component_id']) + group = buckets.get(key) + if group is None or item['first_ts'] > group['last_ts'] + 300: + group = dict(item) + group.update(record_count=0, count=0, signal_reports=0, raw_examples=[], + observed_first_at=item['observed_at'], devices=[]) + group['id'] = 'incident-' + hashlib.sha256(json.dumps([*key,item['first_ts']]).encode()).hexdigest()[:24] + buckets[key] = group + output.append(group) + group['first_ts'] = min(group['first_ts'],item['first_ts']) + group['first_at'] = iso(group['first_ts']) + if item['last_ts'] >= group['last_ts']: + for field in ('last_ts','last_at','raw'): + group[field] = item[field] + group['record_count'] += 1 + group['count'] += item['count'] + is_report = item['category'] != 'pcie_correctable' or 'PCIe Bus Error:' in item['raw'].get('detail','') + group['signal_reports'] += item['count'] if is_report else 0 + group['observed_first_at'] = min(group['observed_first_at'],item['observed_at']) + group['observed_at'] = max(group['observed_at'],item['observed_at']) + if item['device'] and item['device'] not in group['devices']: + group['devices'].append(item['device']) + if len(group['raw_examples']) < 8 and not any(x.get('detail') == item['raw'].get('detail') for x in group['raw_examples']): + group['raw_examples'].append(item['raw']) + priority = {'ERROR':3,'WARN':2,'INFO':1} + if priority.get(item['severity'],0)>priority.get(group['severity'],0): + group['severity']=item['severity'] + state_priority = {'unknown':6,'active':5,'related':4,'history':2,'recovered':1,'info':0} + if state_priority.get(item['state'],0)>state_priority.get(group['state'],0): + group['state']=item['state'];group['state_label']=item['state_label'] + if item['time_note']: + group['time_note']=item['time_note'] + return output + +def explain_item(item,current): + online = len(current.get('gpus',[])) + expected = current.get('expected_gpu_count',2) + usable = online >= expected and not current.get('stale') + if item['category']=='pcie_correctable': + known_gpu=item.get('component_id','').lower().startswith('gpu') + device_word='显卡' if known_gpu else '设备' + item['title']=device_word+'连接反复重试' if item.get('signal_reports',0)>1 else device_word+'连接出现一次重试' + if not item.get('signal_reports'): + item['title']='设备连接出现重试记录' + item['meaning']='主板和'+device_word+'传数据时发现通信错误,系统会自动纠正或重发。这条记录不能证明'+device_word+'已经损坏。' + item['impact']=f'当前 {online}/{expected} 块显卡可被系统正常读取,暂未看到掉卡。' if usable else '当前未能确认全部显卡正常,请先看顶部的实时状态。' + item['urgency']='可以继续使用,建议安排空闲时检查连接' if usable else '先确认显卡是否在线,再安排维护' + item['next_steps']=['现在观察计算任务有没有报错、变慢或中断。', + '若通信重试持续出现,在业务空闲时正常关机并彻底断电。', + '检查显卡供电线两端和插槽;仍有问题时由维护人员换线、换槽对照测试。'] + if item['state']=='active': + item['state_label']='仍在重试 · 显卡可用' if usable and known_gpu else '仍有连接重试 · 需要关注' + elif item['category']=='power': + item['meaning']='系统当时检测不到这张显卡需要的外接供电,所以显卡可能无法启动。线接触不良、线材、电源或卡上检测电路都可能引起这一现象。' + item['impact']=f'当前 {online}/{expected} 块显卡可用;这条是以前的供电异常记录。' if usable else '当前显卡状态不完整,请检查顶部告警。' + item['urgency']='历史记录,当前先观察是否再次出现' if usable else '需要检查辅助供电' + item['next_steps']=[item['action']] + else: + item['impact']='最近采样未再发现该问题。' if item['state']=='history' else item['state_label'] + item['urgency']='查看说明,按建议处理' + item['next_steps']=[item['action']] + if item['state']=='history': + item['state_label']='以前的记录 · 当前未再发现' + if item['state']=='unknown': + item['impact']='实时数据已过期,暂时无法判断现在是否正常。' + item['urgency']='先恢复监控连接,再判断是否需要维修' + item['current_checked_at']=current.get('checked_at') + return item + +def events(params): + now = time.time() + days = min(90, max(1, int(params.get('days', ['7'])[0]))) + start = epoch(params.get('from', [''])[0]) or now - days * 86400 + end = epoch(params.get('to', [''])[0]) or now + 60 + if len(params.get('to', [''])[0]) == 10: + end += 86399 + category = params.get('category', [''])[0] + severity = params.get('severity', [''])[0] + query = params.get('q', [''])[0][:120].lower() + page = max(0, min(10000, int(params.get('page', ['0'])[0]))) + where = ['((occurred>=? AND occurred<=?) OR (observed>=? AND observed<=?))'] + values = [start,end,start,end] + if category: + where.append('category=?'); values.append(category) + if severity=='problem': + where.append("severity IN ('ERROR','WARN')") + elif severity: + where.append('severity=?'); values.append(severity) + with connect() as db: + anchors = {x['boot'].replace('-', ''): x['anchor'] for x in db.execute('SELECT * FROM boot_clock')} + records = db.execute('SELECT * FROM events WHERE ' + ' AND '.join(where) + ' ORDER BY observed DESC LIMIT 15000', values).fetchall() + current = snapshot() + items = [] + for record in records: + row = json.loads(record['raw']) + first,last,note = source_times(row,anchors) + if max(first,last) < start or min(first,last) > end: + continue + title,meaning,action = CATEGORY.get(row.get('category'), CATEGORY['monitor']) + ongoing = False + related = False + component_id,component_label,address = event_component(row,current) + for problem in current.get('issues', []): + if problem['category'] == row.get('category'): + other = event_component({'boot_id':current.get('boot_id',''),'device':problem.get('device',''),'category':problem['category']},current)[0] + related = component_id==other or not problem.get('device') + if related: + break + if row.get('key') and row['key'] in current.get('hardware_active', {}): + ongoing = row.get('event') != 'RECOVERED' + if not current.get('stale') and related and row.get('source') == 'kernel': + mono = row.get('last_monotonic_us') + same_boot = row.get('boot_id', '').replace('-','') == current.get('boot_id','').replace('-','') + if same_boot and mono: + ongoing = 0 <= current.get('monotonic_s',0) - int(mono)/1e6 <= 120 + if current.get('stale'): + state,label = 'unknown','当前状态未知' + elif row.get('event') == 'RECOVERED': + state,label = 'recovered','已记录恢复' + elif ongoing: + state,label = 'active','当前仍有同类异常' + elif related: + state,label = 'related','历史记录 · 当前另有同类告警' + else: + state,label = 'history','历史记录 · 当前未复现' + if row.get('severity') == 'INFO' and row.get('event') != 'RECOVERED': + state,label = 'info','运行记录' + items.append({'id': record['id'], 'title': title, 'meaning': meaning, 'action': action, + 'severity': row.get('severity','INFO'), 'category': row.get('category'), + 'device': row.get('device',''), 'first_ts': first,'last_ts': last, + 'first_at': iso(first),'last_at': iso(last),'observed_at': row.get('observed_at'), + 'count': row.get('count',1),'time_note': note,'state': state,'state_label': label,'raw': row, + 'component_id':component_id,'component_label':component_label}) + raw_total=len(items) + view=params.get('view',['issues'])[0] + if view!='raw': + items=merge_incidents(items) + else: + for item in items: + item.update(record_count=1,signal_reports=item['count'] if item['category']!='pcie_correctable' or 'PCIe Bus Error:' in item['raw'].get('detail','') else 0, + raw_examples=[item['raw']]) + for item in items: + explain_item(item,current) + if query: + items=[item for item in items if query in json.dumps(item,ensure_ascii=False).lower()] + raw_total=sum(item.get('record_count',1) for item in items) + items.sort(key=lambda x: x['last_ts'],reverse=True) + offset = page * 30 + return {'items': items[offset:offset+30], 'total': len(items),'raw_total':raw_total,'view':view,'page': page, + 'more': offset+30 < len(items), 'categories': [{'id':key,'name':value[0]} for key,value in CATEGORY.items()]} + +def task_rpc(action,payload=None,token='',csrf='',client=''): + request={'action':action,'payload':payload or {},'token':token,'csrf':csrf,'client':client} + try: + with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as conn: + conn.settimeout(25);conn.connect(TASK_SOCKET) + conn.sendall(json.dumps(request,ensure_ascii=False).encode()+b'\n') + with conn.makefile('rb') as stream: + raw=stream.readline(1024*1024) + return json.loads(raw) + except (OSError,ValueError) as exc: + logging.warning('Task broker unavailable: %s',type(exc).__name__) + return {'ok':False,'status':503,'error':'定时任务管理服务暂不可用;设备监控仍可正常查看。'} + +class Handler(BaseHTTPRequestHandler): + server_version = 'HardwareDashboard/1.0' + + def log_message(self, fmt, *args): + if len(args)>1 and str(args[1]) not in ('200','304'): + logging.info('%s %s',self.client_address[0], fmt % args) + + def send(self, body, content_type, status=200, attachment=None,extra_headers=None): + if isinstance(body,str): + body = body.encode('utf-8') + self.send_response(status) + self.send_header('Content-Type',content_type) + self.send_header('Content-Length',str(len(body))) + self.send_header('Cache-Control','no-store') + self.send_header('X-Content-Type-Options','nosniff') + self.send_header('X-Frame-Options','DENY') + self.send_header('Content-Security-Policy',"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'") + if attachment: + self.send_header('Content-Disposition',f'attachment; filename="{attachment}"') + for key,value in (extra_headers or {}).items():self.send_header(key,value) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + parsed = urlparse(self.path) + params = parse_qs(parsed.query) + try: + if parsed.path in ('/api/tasks/auth','/api/tasks','/api/tasks/item','/api/tasks/runs','/api/tasks/runlog','/api/tasks/history'): + action={'/api/tasks/auth':'auth.status','/api/tasks':'tasks.list','/api/tasks/item':'tasks.get', + '/api/tasks/runs':'tasks.runs','/api/tasks/runlog':'tasks.runlog','/api/tasks/history':'tasks.history'}[parsed.path] + payload={'id':params.get('id',[''])[0],'run_id':params.get('run_id',[''])[0]} + return self.task_response(action,payload) + elif parsed.path == '/api/status': + data = snapshot() + elif parsed.path == '/api/history': + data = history(params.get('range',['24h'])[0]) + elif parsed.path == '/api/events': + data = events(params) + elif parsed.path == '/api/events.csv': + output = io.StringIO(); writer = csv.writer(output) + writer.writerow(['问题开始时间','最近发生时间','监控最近发现时间','原级别','问题','当前判断','设备','合并技术记录数','日志总行数','报错报告数(非独立故障次数)','维护建议','时间说明','原始日志样例']) + for row in events(params)['items']: + values = [row['first_at'],row['last_at'],row['observed_at'],row['severity'],row['title'],row['state_label'],row.get('component_label',row['device']),row.get('record_count',1),row['count'],row.get('signal_reports',0),row['action'],row['time_note'],row['raw'].get('detail','')] + writer.writerow(["'"+v if isinstance(v,str) and v.startswith(('=','+','-','@')) else v for v in values]) + return self.send('\ufeff'+output.getvalue(),'text/csv; charset=utf-8',attachment='hardware-events.csv') + elif parsed.path in ('/','/app.js','/tasks.js','/style.css','/favicon.svg'): + filename = 'index.html' if parsed.path=='/' else parsed.path[1:] + mime = {'index.html':'text/html; charset=utf-8','app.js':'text/javascript; charset=utf-8','tasks.js':'text/javascript; charset=utf-8','style.css':'text/css; charset=utf-8','favicon.svg':'image/svg+xml'} + return self.send((ROOT/'static'/filename).read_bytes(),mime[filename]) + else: + return self.send('Not found','text/plain',404) + self.send(json.dumps(data,ensure_ascii=False,allow_nan=False),'application/json; charset=utf-8') + except (ValueError,TypeError): + self.send('{"error":"参数格式不正确"}','application/json; charset=utf-8',400) + except (BrokenPipeError,ConnectionResetError): + pass + except Exception: + logging.exception('Request failed') + self.send('{"error":"读取失败,请稍后重试"}','application/json; charset=utf-8',500) + + def task_response(self,action,payload): + token='' + try: + cookie=SimpleCookie();cookie.load(self.headers.get('Cookie','')) + if 'hw_task_session' in cookie:token=cookie['hw_task_session'].value + except CookieError:pass + result=task_rpc(action,payload,token,self.headers.get('X-Task-CSRF',''),self.client_address[0]) + headers={} + if result.get('ok'): + data=result['data'] + if action=='auth.login': + secret=data.pop('token') + headers['Set-Cookie']=f'hw_task_session={secret}; Path=/api/tasks; Max-Age={data["max_age"]}; HttpOnly; SameSite=Strict' + elif action in ('auth.logout','auth.password'): + headers['Set-Cookie']='hw_task_session=; Path=/api/tasks; Max-Age=0; HttpOnly; SameSite=Strict' + self.send(json.dumps(data,ensure_ascii=False),'application/json; charset=utf-8',extra_headers=headers) + else: + self.send(json.dumps({'error':result.get('error','任务操作失败。')},ensure_ascii=False), + 'application/json; charset=utf-8',result.get('status',500)) + + def do_POST(self): + parsed=urlparse(self.path) + actions={'/api/tasks/login':'auth.login','/api/tasks/logout':'auth.logout','/api/tasks/password':'auth.password', + '/api/tasks/preview':'tasks.preview','/api/tasks/save':'tasks.save','/api/tasks/enable':'tasks.enable', + '/api/tasks/delete':'tasks.delete','/api/tasks/run':'tasks.run','/api/tasks/stop':'tasks.stop'} + if parsed.path not in actions:return self.send('Not found','text/plain',404) + origin=self.headers.get('Origin','') + host=self.headers.get('Host','') + if urlparse(origin).netloc!=host or urlparse(origin).scheme not in ('http','https') or urlparse(origin).hostname not in ('192.168.1.37','localhost','127.0.0.1'): + return self.send('{"error":"请从本机监控页面发起操作。"}','application/json; charset=utf-8',403) + try: + length=int(self.headers.get('Content-Length','0')) + if not 0 document.getElementById(id); +const colors = {green:'#b3ee80',blue:'#6bbcff',purple:'#b4a1f4',amber:'#f5be6a',red:'#ff8c86'}; +const initialSection=location.hash.slice(1); +let latest = null, selectedRange = '24h', historyData = null, eventPage = 0, eventParams = new URLSearchParams({days:'7',severity:'problem',view:'issues'}); +let eventData = null, statusBusy = false, historySequence = 0, eventsBusy = false, eventSequence = 0, eventController = null; +function el(tag, cls, text) { const e=document.createElement(tag);if(cls)e.className=cls;if(text!==undefined)e.textContent=text;return e; } +function icon(name) {const svg=document.createElementNS('http://www.w3.org/2000/svg','svg');const use=document.createElementNS(svg.namespaceURI,'use');use.setAttribute('href','#'+name);svg.append(use);return svg;} +function valid(v) {return v!==null&&v!==undefined&&Number.isFinite(Number(v));} +function num(v, digits=0) {return valid(v)?Number(v).toFixed(digits):'—';} +function when(v, full=true) {if(!v)return '—';const d=typeof v==='number'?new Date(v*1000):new Date(v);if(Number.isNaN(d.getTime()))return '时间未知';return new Intl.DateTimeFormat('zh-CN',{timeZone:'Asia/Shanghai',...(full?{year:'numeric',month:'2-digit',day:'2-digit'}:{}),hour:'2-digit',minute:'2-digit',second:'2-digit',hour12:false}).format(d);} +function duration(s){if(!valid(s))return '—';s=Math.max(0,Math.round(s));const d=Math.floor(s/86400),h=Math.floor(s%86400/3600),m=Math.floor(s%3600/60);return `${d?d+' 天 ':''}${h} 小时 ${m} 分钟`;} +function toast(text){$('toast').textContent=text;$('toast').hidden=false;clearTimeout(toast.timer);toast.timer=setTimeout(()=>$('toast').hidden=true,4500);} +async function api(path){const response=await fetch(path,{signal:AbortSignal.timeout(9000),cache:'no-store'});if(!response.ok)throw new Error('HTTP '+response.status);return response.json();} +function meter(id,value){$(id).style.width=Math.min(100,Math.max(0,Number(value)||0))+'%';} +function metric(id,value,unit,digits=0){$(id).replaceChildren(document.createTextNode(num(value,digits)),el('small','',unit));} +function connectionLost(message){$('connection').replaceChildren(el('i','dot'),document.createTextNode('连接中断'));$('connection').firstChild.style.background=colors.red;$('health-banner').className='health-banner error';$('health-title').textContent='暂时无法确认设备状态';$('health-description').textContent=message||'网页与设备的连接已中断。下面保留的是上次数据,请勿当作实时状态。';$('health-icon').replaceChildren(icon('alert').firstChild);} +function renderStatus(data){ + if(!data.ts){connectionLost('采集器正在启动,暂未取得实时数据,请稍候。');return;} + const readingAnchor=captureSectionAnchor(); + latest=data; + $('host-name').textContent=data.host||'设备';$('last-update').textContent=when(data.ts,false); + if(data.stale){connectionLost('设备采样超过 35 秒未更新,当前状态未知。');return;} + $('connection').replaceChildren(el('i','dot'),document.createTextNode('实时连接 · 10 秒')); + const issues=data.issues||[],error=issues.some(x=>x.severity==='ERROR'),warn=issues.length>0; + $('health-banner').className='health-banner'+(error?' error':warn?' warning':''); + const onlyRetries=warn&&issues.every(x=>x.category==='pcie_correctable')&&data.gpus.length>=data.expected_gpu_count; + $('health-title').textContent=error?'设备存在故障,需要处理':onlyRetries?'显卡还能用,但连接需要检查':warn?'设备需要关注':'设备运行正常'; + $('health-description').textContent=onlyRetries?'显卡当前在线,主板与显卡之间仍有通信重试。可以继续观察,安排空闲时检查连接。':warn?`${issues.length} 项需要关注,具体原因和处理建议见下方。`:`${data.gpus.length}/${data.expected_gpu_count} 块计算显卡在线 · 当前采样未发现异常`; + $('health-icon').replaceChildren(icon(warn?'alert':'check').firstChild); + const list=$('current-issues');list.hidden=!issues.length;list.replaceChildren(); + issues.forEach(item=>{const card=el('article','issue'+(item.severity==='ERROR'?' error':''));const device=item.component_label?item.component_label+':':'';card.append(el('strong','',device+item.title),el('p','',item.meaning+' '+item.detail),el('small','', '怎么处理:'+item.action));list.append(card);}); + $('nav-alert').hidden=!issues.length;$('nav-alert').textContent=String(issues.length); + metric('cpu-value',data.cpu,'%',1);meter('cpu-bar',data.cpu);$('cpu-threads').textContent=(data.cpu_sockets?data.cpu_sockets+' 路 CPU · ':'')+(data.cpu_threads||'—')+' 线程';$('cpu-temp').textContent=num(data.cpu_temp)+' °C';$('cpu-temp').title=data.cpu_name; + metric('memory-value',data.memory,'%');meter('memory-bar',data.memory);$('memory-used').textContent=`${num(data.memory_total_gib-data.memory_available_gib,1)} / ${num(data.memory_total_gib,1)} GiB`;$('memory-free').textContent=num(data.memory_available_gib,1)+' GiB 可用'; + metric('gpu-count',data.gpus.length,`/ ${data.expected_gpu_count} 在线`);$('gpu-online-text').replaceChildren(el('i','dot'),document.createTextNode(data.gpus.length>=data.expected_gpu_count?'全部预期显卡可用':'显卡数量少于预期'));$('gpu-online-text').firstChild.style.background=data.gpus.length>=data.expected_gpu_count?colors.green:colors.red;$('gpu-type').textContent=data.gpus[0]?.name?.includes('P100')?'Tesla P100':'NVIDIA'; + const disks=Object.values(data.smart||{}),diskBad=!data.hardware_stale&&disks.some(x=>x.passed===false||x.critical_warning),diskKnown=!data.hardware_stale&&disks.length&&disks.every(x=>x.available&&x.passed===true); + $('disk-health').textContent=diskBad?'需要检查':diskKnown?'健康正常':'状态未知';$('disk-health').style.color=diskBad?colors.red:diskKnown?colors.green:colors.amber; + $('disk-health-detail').textContent=diskKnown?'SMART 检查通过':diskBad?'请展开磁盘相关日志':'尚未取得完整 SMART 数据';$('disk-space').textContent=`已用 ${num(data.disk_used_gib,0)} / ${num(data.disk_total_gib,0)} GiB`;$('disk-temp').textContent=num(data.disk_temp)+' °C'; + renderGPUs(data); + $('uptime').textContent=duration(data.uptime_s);['disk-read','disk-write','net-rx','net-tx'].forEach(id=>$(id).textContent=num(data.rates?.[id.replace('-','_')],2)+' MiB/s'); + const start=data.history_started_at?when(data.history_started_at):'本次部署'; + $('history-note').textContent=`CPU、内存和显卡使用率从 ${start} 开始记录。已有的显卡温度、功耗历史已补录(旧记录约 15 分钟一条);空白表示该时段没有采样。曲线为时段均值。`; + restoreSectionAnchor(readingAnchor);queueNavigation(); +} +function renderGPUs(data){ + const holder=$('gpu-grid');holder.replaceChildren(); + const gpus=[...data.gpus].sort((a,b)=>a.uuid.localeCompare(b.uuid)); + gpus.forEach((gpu,index)=>{ + const card=el('article','gpu-card');card.style.setProperty('--accent',index%2?colors.amber:colors.blue); + const head=el('div','gpu-head');const state=el('span','gpu-state');state.append(el('i','dot'),document.createTextNode('在线'));head.append(el('span','gpu-label','GPU '+String(index+1).padStart(2,'0')),el('h3','',gpu.name?.replace('Tesla ','')||'NVIDIA GPU'),state); + const content=el('div','gpu-content'),util=el('div','gpu-util');util.append(document.createTextNode(num(gpu.utilization)),el('small','','%'),el('span','',valid(gpu.utilization)&&gpu.utilization>5?'正在计算':'空闲 / 低负载')); + const numbers=el('div','gpu-numbers');[['温度',gpu.temperature_c,'°C'],['功耗',gpu.power_w,'W']].forEach(([label,value,unit])=>{const cell=el('div','gpu-number'),strong=el('strong','',num(value));strong.append(el('small','',unit));if(unit==='°C'&&value>=data.gpu_warn_c)strong.style.color=colors.red;cell.append(el('span','',label),strong);numbers.append(cell);});content.append(util,numbers); + const vram=el('div','gpu-vram');vram.append(el('span','','显存占用'),el('strong','',`${num(valid(gpu.memory_used_mib)?gpu.memory_used_mib/1024:null,1)} / ${num(valid(gpu.memory_total_mib)?gpu.memory_total_mib/1024:null,0)} GiB`)); + const bar=el('div','meter'),fill=el('i');fill.style.width=(gpu.memory_total_mib?Math.min(100,gpu.memory_used_mib/gpu.memory_total_mib*100):0)+'%';bar.append(fill); + const meta=el('div','gpu-meta');meta.append(el('span','',gpu.pci.replace(/^00000000:/,'0000:')),el('span','',gpu.uuid.slice(0,16)+'…'),el('span','',`功耗上限 ${num(gpu.power_limit_w)}W`));meta.title=gpu.uuid; + card.append(head,content,vram,bar,meta);holder.append(card); + }); + for(let i=gpus.length;ie.setAttribute(k,v));if(text!==undefined)e.textContent=text;return e;} +function chart(id,legendId,series,data,unit,maxValue){ + const container=$(id),legend=$(legendId);container.replaceChildren();legend.replaceChildren(); + series.forEach(s=>{const item=el('span','legend-item'),swatch=el('i','legend-swatch');swatch.style.background=s.color;item.append(swatch,document.createTextNode(s.name));legend.append(item);}); + const width=560,height=190,left=36,right=12,top=10,bottom=27,w=width-left-right,h=height-top-bottom; + const svg=svgNode('svg',{viewBox:`0 0 ${width} ${height}`,preserveAspectRatio:'none','aria-hidden':'true'}); + const values=series.flatMap(s=>data.points.map(p=>s.value(p)).filter(valid)); + const ceiling=Math.max(maxValue||50,Math.ceil(Math.max(0,...values)/25)*25); + const x=t=>left+(t-data.start)/(data.end-data.start)*w,y=v=>top+h-(v/ceiling)*h; + for(let i=0;i<=4;i++){const v=ceiling*i/4,yy=y(v);svg.append(svgNode('line',{x1:left,y1:yy,x2:width-right,y2:yy,class:'grid-line'}),svgNode('text',{x:left-8,y:yy+3,'text-anchor':'end'},num(v)));} + for(let i=0;i<=4;i++){const t=data.start+(data.end-data.start)*i/4;const label=new Intl.DateTimeFormat('zh-CN',{timeZone:'Asia/Shanghai',...(data.end-data.start>86400?{month:'2-digit',day:'2-digit'}:{hour:'2-digit',minute:'2-digit',hour12:false})}).format(new Date(t*1000));svg.append(svgNode('text',{x:x(t),y:height-6,'text-anchor':i===0?'start':i===4?'end':'middle'},label));} + series.forEach(s=>{ + const points=data.points.filter(p=>valid(s.value(p)));let path='',last=null; + points.forEach(p=>{const gap=last? p.ts-last.ts:0;const maxGap=p.live||last?.live?Math.max(data.step*3,45):Math.max(data.step*3,1250);path+=(last&&gap<=maxGap?'L':'M')+x(p.ts).toFixed(2)+','+y(s.value(p)).toFixed(2);last=p;}); + if(path)svg.append(svgNode('path',{d:path,stroke:s.color,class:'series-line'})); + if(points.length<4)points.forEach(p=>svg.append(svgNode('circle',{cx:x(p.ts),cy:y(s.value(p)),r:2.4,fill:s.color}))); + }); + if(unit==='°C'){svg.append(svgNode('line',{x1:left,x2:width-right,y1:y(85),y2:y(85),stroke:colors.amber,'stroke-width':'.7','stroke-dasharray':'5 6',opacity:'.4'}));} + const cursor=svgNode('line',{x1:0,x2:0,y1:top,y2:height-bottom,class:'cursor',visibility:'hidden'}),overlay=svgNode('rect',{x:left,y:top,width:w,height:h,class:'chart-overlay'});svg.append(cursor,overlay);container.append(svg); + const tooltip=el('div','chart-tooltip');tooltip.hidden=true;container.append(tooltip); + if(!values.length){const empty=el('div','chart-empty');empty.append(el('span','','这个时段还没有性能数据'),el('small','','采集运行后,曲线会自动出现'));container.append(empty);} + overlay.addEventListener('pointermove',event=>{ + const bounds=svg.getBoundingClientRect(),px=Math.max(left,Math.min(width-right,(event.clientX-bounds.left)/bounds.width*width)); + const t=data.start+(px-left)/w*(data.end-data.start);let closest=null; + data.points.forEach(p=>{if(!closest||Math.abs(p.ts-t){const row=el('p'),value=s.value(closest);row.append(el('span','',s.name),el('span','',valid(value)?num(value,1)+' '+unit:'未采集'));tooltip.append(row);});}else tooltip.append(el('p','','该时段没有采样')); + tooltip.style.left=Math.max(3,Math.min(container.clientWidth-190,(event.clientX-bounds.left)+12))+'px'; + });overlay.addEventListener('pointerleave',()=>{cursor.setAttribute('visibility','hidden');tooltip.hidden=true;}); +} +function renderHistory(data){ + historyData=data; + const devices=Object.keys(data.devices).sort();const palette=[colors.blue,colors.amber,colors.purple,colors.green]; + const gpuSeries=key=>devices.map((uuid,i)=>({name:'GPU '+String(i+1).padStart(2,'0'),color:palette[i%palette.length],value:p=>p.gpus?.[uuid]?.[key]})); + chart('chart-system','legend-system',[{name:'CPU',color:colors.green,value:p=>p.cpu},{name:'内存',color:colors.purple,value:p=>p.memory}],data,'%',100); + chart('chart-gpu','legend-gpu',gpuSeries('utilization'),data,'%',100); + chart('chart-temp','legend-temp',[{name:'CPU 最高温度',color:colors.green,value:p=>p.cpu_temp},...gpuSeries('temperature_c')],data,'°C',100); + chart('chart-power','legend-power',gpuSeries('power_w'),data,'W',Math.max(250,...data.points.flatMap(p=>Object.values(p.gpus||{}).map(g=>g.power_w||0)))*1.08); + $('history-caption').textContent=`${when(data.start)} — ${when(data.end)} · 曲线为时段均值,移入可查看数值`; +} +async function loadHistory(){const seq=++historySequence;try{const data=await api('/api/history?range='+selectedRange);if(seq===historySequence)renderHistory(data);}catch(error){if(seq===historySequence)$('history-caption').textContent='曲线暂时无法读取,请稍后重试。';}} +function addTime(container,label,value){const cell=el('div');cell.append(el('span','',label),el('strong','',value));container.append(cell);} +function renderEvents(data,preservePosition=true){ + eventData=data;const holder=$('event-list'); + const opened=new Set([...holder.querySelectorAll('details[open]')].map(x=>x.dataset.id)); + const visible=preservePosition?[...holder.querySelectorAll('.event-row')].find(x=>x.getBoundingClientRect().bottom>70&&x.getBoundingClientRect().top{const option=el('option','',item.name);option.value=item.id;$('category').append(option);});} + $('event-count').textContent=data.view==='raw'?`共 ${data.total} 条技术记录`:`合并后 ${data.total} 段问题记录 · 来自 ${data.raw_total} 条技术记录,不是故障次数`; + if(!data.items.length){const empty=el('div','empty-events');empty.append(el('strong','','这个范围内没有匹配记录'),el('span','','可以扩大日期范围,或选择“全部问题”。'));holder.append(empty);} + data.items.forEach(item=>{ + const details=el('details','event-row');details.dataset.id=item.id;details.open=opened.has(item.id); + const summary=el('summary'),severity=item.severity==='ERROR'?'error':item.severity==='INFO'?'info':''; + const main=el('div','event-main');main.append(el('strong','',item.title),el('p','',item.component_label||'这台电脑')); + const time=el('div','event-time',when(item.last_ts));time.append(el('small','',item.time_note?'最近一次 · 时间有估算':'最近一次发生')); + const historical=item.state==='history'||item.state==='related';const badge=item.severity==='ERROR'?(historical?'曾有故障':'需处理'):item.severity==='WARN'?(historical?'曾需关注':'需关注'):'记录'; + summary.append(el('span','event-level '+severity,badge),main,el('span','event-state '+item.state+' '+(item.severity==='WARN'?'warning':''),item.state_label),time,el('span','event-chevron','›')); + const detail=el('div','event-detail'); + [['这是什么问题',item.meaning],['现在还能不能用',item.impact]].forEach(([label,text])=>{const answer=el('div','plain-answer');answer.append(el('strong','',label),el('p','',text));detail.append(answer);}); + const action=el('div','maintenance');action.append(el('strong','','你现在需要做什么'),el('p','maintenance-priority',item.urgency));const steps=el('ol');(item.next_steps||[item.action]).forEach(text=>steps.append(el('li','',text)));action.append(steps);detail.append(action); + const times=el('div','time-grid friendly-times');addTime(times,item.time_note?'这段问题开始(估算)':'这段问题从何时开始',when(item.first_ts));addTime(times,'最近一次发生',when(item.last_ts));addTime(times,'监控最近读到记录',when(item.observed_at));addTime(times,'当前状态检查于',when(item.current_checked_at));detail.append(times); + detail.append(el('p','record-summary',`已合并 ${item.record_count||1} 条技术记录,共 ${item.count} 行日志。${item.signal_reports?'其中有 '+item.signal_reports+' 条报错报告。':''}这些数字不是设备损坏次数。`)); + if(item.time_note)detail.append(el('div','time-warning',item.time_note)); + const technical=el('details','technical-details');technical.dataset.id=item.id+'-technical';technical.open=opened.has(technical.dataset.id);technical.append(el('summary','','查看原始技术日志(维护人员用)')); + const evidence=el('div','technical-body'),rawTimes=el('div','time-grid'); + if(item.raw.first_monotonic_us)addTime(rawTimes,'原始批次:该次启动后经过',num(Number(item.raw.first_monotonic_us)/1e6,3)+' 秒 — '+num(Number(item.raw.last_monotonic_us)/1e6,3)+' 秒'); + if(item.raw.boot_id)addTime(rawTimes,'启动 ID',item.raw.boot_id); + if(item.raw.first_at)addTime(rawTimes,'原始批次首次时间(未改写)',when(item.raw.first_at)); + if(item.raw.last_at)addTime(rawTimes,'原始批次末次时间(未改写)',when(item.raw.last_at)); + if(item.devices?.length)addTime(rawTimes,'涉及的硬件地址',item.devices.join(' / ')); + evidence.append(rawTimes);(item.raw_examples||[item.raw]).forEach(row=>evidence.append(el('pre','raw-log',row.detail||JSON.stringify(row,null,2))));technical.append(evidence);detail.append(technical); + details.append(summary,detail);holder.append(details); + }); + $('page-label').textContent=`第 ${data.page+1} 页 · 每页最多 30 条`;$('previous-page').disabled=data.page===0;$('next-page').disabled=!data.more; + if(anchor&&!navigationLock){const replacement=[...holder.querySelectorAll('.event-row')].find(x=>x.dataset.id===anchor.id);if(replacement){const delta=replacement.getBoundingClientRect().top-anchor.top;if(Math.abs(delta)>1)scrollPage(scrollYValue()+delta,false);}} + queueNavigation(); +} +async function loadEvents(force=false){if(eventsBusy&&!force)return;if(force)eventController?.abort();const seq=++eventSequence;eventsBusy=true;const controller=new AbortController();eventController=controller;const timer=setTimeout(()=>controller.abort(),10000);try{const params=new URLSearchParams(eventParams);params.set('page',eventPage);const response=await fetch('/api/events?'+params.toString(),{signal:controller.signal,cache:'no-store'});if(!response.ok)throw new Error('HTTP '+response.status);const data=await response.json();if(seq===eventSequence)renderEvents(data,!force);}catch(error){if(seq===eventSequence&&error.name!=='AbortError')$('event-count').textContent='暂时无法读取记录,请点击“查询日志”重试。';}finally{clearTimeout(timer);if(seq===eventSequence)eventsBusy=false;}} +function applyEventFilters(){if($('date-from').value&&$('date-to').value&&$('date-from').value>$('date-to').value){toast('开始日期不能晚于结束日期。');return;}eventParams=new URLSearchParams({days:'7'});[['view','event-view'],['category','category'],['severity','severity'],['from','date-from'],['to','date-to'],['q','event-search']].forEach(([key,id])=>{if($(id).value)eventParams.set(key,$(id).value);});eventPage=0;$('event-count').textContent='正在按所选条件查询…';loadEvents(true);} +$('event-filter').addEventListener('submit',event=>{event.preventDefault();applyEventFilters();}); +['event-view','category','severity','date-from','date-to'].forEach(id=>$(id).addEventListener('change',applyEventFilters));let searchTimer;$('event-search').addEventListener('input',()=>{clearTimeout(searchTimer);searchTimer=setTimeout(applyEventFilters,400);}); +$('previous-page').addEventListener('click',()=>{if(!eventsBusy&&eventPage>0){eventPage--;loadEvents(true);}});$('next-page').addEventListener('click',()=>{if(!eventsBusy&&eventData?.more){eventPage++;loadEvents(true);}}); +$('export-events').addEventListener('click',()=>{const params=new URLSearchParams(eventParams);params.set('page',eventPage);window.location.assign('/api/events.csv?'+params.toString());}); +$('ranges').addEventListener('click',event=>{const button=event.target.closest('button[data-range]');if(!button)return;selectedRange=button.dataset.range;document.querySelectorAll('#ranges button').forEach(x=>x.classList.toggle('selected',x===button));$('history-caption').textContent='正在加载所选范围…';loadHistory();}); +$('fullscreen').addEventListener('click',async()=>{try{if(document.fullscreenElement)await document.exitFullscreen();else await document.body.requestFullscreen();}catch(error){toast('此浏览器不支持网页全屏,可按 F11。');}}); +const sections=[{id:'overview',name:'设备总览'},{id:'trends',name:'性能趋势'},{id:'events',name:'问题与维护'},{id:'tasks',name:'定时任务'},{id:'guide',name:'使用说明'}]; +let navigationLock=null,navigationTimer=null,navigationFrame=null,currentSection='overview',userHasScrolled=false; +function scrollRoot(){return document.fullscreenElement===document.body?document.body:document.scrollingElement;} +function scrollYValue(){return scrollRoot()?.scrollTop??window.scrollY;} +function scrollPage(top,smooth=true){const options={top,behavior:smooth&&!matchMedia('(prefers-reduced-motion: reduce)').matches?'smooth':'instant'};if(document.fullscreenElement===document.body)document.body.scrollTo(options);else window.scrollTo(options);} +function captureSectionAnchor(){if(navigationLock||scrollYValue()<200||currentSection==='overview')return null;const target=$(currentSection);return target?{id:currentSection,top:target.getBoundingClientRect().top}:null;} +function restoreSectionAnchor(anchor){if(!anchor||navigationLock)return;const target=$(anchor.id);if(target){const delta=target.getBoundingClientRect().top-anchor.top;if(Math.abs(delta)>1)scrollPage(scrollYValue()+delta,false);}} +function markSection(id,updateHash=true){const index=Math.max(0,sections.findIndex(x=>x.id===id));currentSection=id;document.querySelectorAll('[data-section-link]').forEach(link=>{const active=link.dataset.sectionLink===id;link.classList.toggle('active',active);if(active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current');});$('current-section').textContent=sections[index].name;$('next-section').textContent=index{if($(section.id).getBoundingClientRect().top<=105)active=section.id;});if(y>0&&y>=total-4)active='guide';if(active!==currentSection)markSection(active);} +function queueNavigation(){if(!navigationFrame)navigationFrame=requestAnimationFrame(synchronizeNavigation);} +function navigateSection(id,smooth=true){if(!sections.some(s=>s.id===id))return;userHasScrolled=true;clearTimeout(navigationTimer);navigationLock=id;markSection(id);scrollPage(id==='overview'?0:Math.max(0,$(id).getBoundingClientRect().top+scrollYValue()-76),smooth);navigationTimer=setTimeout(()=>{navigationLock=null;queueNavigation();},smooth?1300:50);} +document.querySelectorAll('[data-section-link]').forEach(link=>link.addEventListener('click',event=>{event.preventDefault();navigateSection(link.dataset.sectionLink);})); +$('back-to-top').addEventListener('click',()=>navigateSection('overview'));$('next-section').addEventListener('click',()=>{const index=sections.findIndex(s=>s.id===currentSection);navigateSection(sections[index+1]?.id||'overview');}); +window.addEventListener('scroll',()=>{queueNavigation();if(navigationLock){clearTimeout(navigationTimer);navigationTimer=setTimeout(()=>{navigationLock=null;queueNavigation();},180);}},{passive:true,capture:true}); +function manualScrollIntent(){userHasScrolled=true;if(navigationLock){scrollPage(scrollYValue(),false);clearTimeout(navigationTimer);navigationLock=null;}queueNavigation();} +['wheel','touchstart'].forEach(name=>window.addEventListener(name,manualScrollIntent,{passive:true})); +window.addEventListener('keydown',event=>{if(['PageDown','PageUp','ArrowDown','ArrowUp','Home','End',' '].includes(event.key))manualScrollIntent();}); +window.addEventListener('hashchange',()=>navigateSection(location.hash.slice(1),false));window.addEventListener('resize',queueNavigation);document.addEventListener('fullscreenchange',queueNavigation); +window.addEventListener('online',()=>{loadStatus();loadHistory();loadEvents();}); +Promise.allSettled([loadStatus(),loadHistory(),loadEvents()]).then(()=>{if(!userHasScrolled&§ions.some(s=>s.id===initialSection))navigateSection(initialSection,false);else queueNavigation();}); +setInterval(loadStatus,10000);setInterval(()=>{if(!document.hidden){loadHistory();loadEvents();}},30000); +document.addEventListener('visibilitychange',()=>{if(!document.hidden){loadStatus();loadHistory();loadEvents();}}); diff --git a/static/favicon.svg b/static/favicon.svg new file mode 100644 index 0000000..a499052 --- /dev/null +++ b/static/favicon.svg @@ -0,0 +1 @@ + diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..b5ebf4d --- /dev/null +++ b/static/index.html @@ -0,0 +1,81 @@ + + + + + 设备监控 · 192.168.1.37 + + + + + + +
+ +
正在查看设备总览
+
正在读取设备状态

首次连接可能需要几秒,请稍候。

最近更新—
+ +
+
CPU 使用率
—%
正在读取— °C
+
内存占用
—%
— / — GiB— 可用
+
计算显卡
—/ — 在线
正在确认显卡状态
按 UUID 识别显卡NVIDIA
+
系统磁盘
—
正在读取健康状态
— / — GiB— °C
+
+

显卡实时状态

高使用率表示正在计算,不等于硬件故障。
每 10 秒采样
+
正在读取显卡…
正在读取显卡…
+
连续运行 —磁盘读取 — MB/s磁盘写入 — MB/s网络接收 — MB/s网络发送 — MB/s
+ +
历史曲线显示真实采样数据;没有采集到的时段留空。
+
+

CPU 与内存

%
+

显卡使用率

%
+

硬件温度

°C
+

显卡功耗

W
+
+
EVENTS & MAINTENANCE

问题与维护

先看设备现在能不能用,再看什么时候出现、下一步怎么处理。

+
重复日志已合并,不代表设备坏了很多次。同一设备、同类问题在 5 分钟内继续报告,会归在同一段记录中。黄色表示需要观察或安排检查,不等于显卡已经损坏。
+
正在读取问题记录…展开看中文说明;技术详情默认收起
+
SCHEDULED TASKS

定时任务

到点自动执行命令或脚本,关闭网页也会继续运行。时间均为北京时间。

+
+
正在连接任务管理服务

监控页面可直接查看,管理任务需要登录。

+

把重复维护交给计划任务

支持命令、Bash/Python 脚本和服务器已有脚本。
可用于定时检查、备份、证书续签或计划重启。

初始管理员登录信息见交付的登录说明文件,账号与 SSH 密码相互独立。
+ +
+

第一次使用?先看状态,再看问题

01

顶部看现在是否正常

绿色表示当前检查未发现异常。黄色表示需要关注,红色表示有故障需要处理。断开连接时会单独提示,不继续显示“正常”。

02

曲线看何时开始变化

选择时间范围,移到曲线上查看具体时刻。使用率高通常说明任务繁忙;温度、掉卡和通信错误才需要结合告警判断。

03

维护操作先看执行内容

问题记录有处理建议。定时任务可以真正执行服务器操作,保存前请检查脚本、周期和启用状态;重启类任务会中断正在进行的计算。

+
设备监控 / NODE 037性能数据保留 30 天 · 故障记录最长保留 90 天 · 北京时间 UTC+8
+
+ + +

登录任务管理

只有管理员可以创建和执行服务器任务。

+

新增定时任务

先填写执行内容,再设置周期。新任务默认停用,可先手动测试。

什么时候执行北京时间 UTC+8
错过执行时间不补跑;同一任务不会重叠执行。
高级设置(可选)
+

确认操作

+

任务执行记录

成功表示命令退出码为 0,具体业务结果仍以输出为准。

选择一次执行,查看输出和错误。
尚未选择执行记录。
+

修改任务管理密码

+ + diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..bc42935 --- /dev/null +++ b/static/style.css @@ -0,0 +1,465 @@ +:root{--bg:#0c1218;--panel:#141d25;--panel2:#19242d;--line:#28333e;--text:#e9eff2;--muted:#8b9ba8;--green:#b3ee80;--blue:#6bbcff;--purple:#b4a1f4;--amber:#f5be6a;--red:#ff8c86;--sans:Inter,"Segoe UI","Microsoft YaHei",system-ui,sans-serif;--mono:"Cascadia Code",Consolas,monospace} +*{box-sizing:border-box} +html{scroll-behavior:smooth;scroll-padding-top:28px} +body{margin:0;background:var(--bg);color:var(--text);font:14px/1.6 var(--sans);letter-spacing:.1px} +button,input,select{font:inherit} +button,a,input,select,summary{-webkit-tap-highlight-color:transparent} +button{cursor:pointer} +button:disabled{cursor:default;opacity:.35} +button:focus-visible,a:focus-visible,input:focus-visible,select:focus-visible,summary:focus-visible{outline:2px solid var(--green);outline-offset:4px} +a{color:inherit;text-decoration:none} +svg{width:20px;height:20px;fill:none;stroke:currentColor;stroke-width:1.6;stroke-linecap:round;stroke-linejoin:round;flex-shrink:0} +.sprite{position:absolute;width:0;height:0;overflow:hidden} +[hidden]{display:none!important} +.sidebar{position:fixed;inset:0 auto 0 0;width:220px;background:#10181f;border-right:1px solid var(--line);display:flex;flex-direction:column;padding:32px 20px;z-index:10} +.brand{display:flex;gap:12px;align-items:center;font-size:19px;font-weight:650;letter-spacing:.5px} +.brand small{display:block;font-size:8px;color:var(--muted);letter-spacing:1.6px;margin-top:1px} +.brand-mark{width:39px;height:39px;border-radius:11px;background:var(--green);color:#18271b;display:grid;place-items:center} +.brand-mark svg{width:26px;height:26px;stroke-width:2} +.nav-caption{font-size:10px;color:#647888;letter-spacing:1.5px;margin:46px 12px 12px} +.sidebar nav{display:grid;gap:6px} +.sidebar nav a{padding:12px;border-radius:7px;display:flex;gap:11px;align-items:center;color:#95a5b1;font-size:13px} +.sidebar nav a:hover{background:#1b2832;color:white} +.sidebar nav a.active{background:#24332b;color:var(--green)} +.nav-dot{width:5px;height:5px;background:var(--green);border-radius:50%;margin-left:auto} +.nav-count{font:10px var(--mono);background:#5a302e;color:var(--red);padding:2px 5px;border-radius:4px;margin-left:auto} +.sidebar-node{margin-top:auto;padding:24px 13px 28px;display:grid;gap:7px} +.tiny-label{font-size:10px;color:#697d8a;letter-spacing:1px} +.sidebar-node strong{font:12px var(--mono);display:flex;align-items:center;gap:8px;margin-top:4px} +.sidebar-node>span:not(.tiny-label){font:12px var(--mono);color:var(--muted);padding-left:14px} +.node-rule{height:1px;background:var(--line);margin:18px 0 4px} +.sidebar-node p{font-size:11px;color:#788c9a;line-height:1.9;margin:0} +.sidebar-footer{border-top:1px solid var(--line);padding:20px 12px 0;font-size:10px;color:#7e929f;display:flex;gap:8px;align-items:center} +.dot{display:inline-block;width:6px;height:6px;border-radius:50%;background:var(--green);flex-shrink:0} +.sidebar-footer .dot{background:#58776b} +main{margin-left:220px;padding:34px 36px 20px;max-width:1920px} +.page-header{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:27px} +.eyebrow{font-size:10px;font-weight:600;letter-spacing:1.2px;color:#8094a3} +h1{font-size:28px;letter-spacing:.3px;font-weight:600;margin:7px 0 5px;line-height:1.4} +.title-dot{color:var(--green);padding-left:1px} +.page-header p{margin:0;color:var(--muted);font-size:11px} +.separator{color:#465865;padding:0 9px} +#host-name{font-family:var(--mono)} +.header-actions{display:flex;align-items:center;gap:17px} +.live-badge{font-size:11px;display:flex;gap:7px;align-items:center;color:#b8c7cf;white-space:nowrap} +.icon-button{background:var(--panel);border:1px solid var(--line);width:36px;height:36px;border-radius:7px;color:#b8c7cf;display:grid;place-items:center} +.icon-button:hover,.text-button:hover{border-color:#61717d;color:white} +.health-banner{display:flex;gap:15px;align-items:center;background:linear-gradient(105deg,#1c3027,#17221f 70%);border:1px solid #36503c;border-radius:9px;padding:18px 21px;margin-bottom:20px} +.health-icon{width:37px;height:37px;border-radius:50%;background:#294c30;color:var(--green);display:grid;place-items:center} +.health-icon svg{width:23px;height:23px;stroke-width:2} +.health-text strong{font-size:16px;font-weight:600;color:#d7edcc} +.health-text p{font-size:11px;color:#92b49d;margin:3px 0 0} +.health-time{margin-left:auto;padding-left:24px;border-left:1px solid #33463d;text-align:right;display:grid;gap:3px;white-space:nowrap} +.health-time span{font-size:10px;color:#88a393} +.health-time strong{font:11px var(--mono);font-weight:400;color:#c8d7cd} +.health-banner.warning{background:linear-gradient(100deg,#332b20,#211f1c);border-color:#5e4a2c} +.warning .health-icon{background:#554025;color:var(--amber)} +.warning .health-text strong{color:#f7d99f} +.warning .health-text p{color:#c0a77f} +.health-banner.error{background:linear-gradient(100deg,#352322,#221c1f);border-color:#60403c} +.error .health-icon{background:#5b302d;color:var(--red)} +.error .health-text strong{color:#ffc2bd} +.error .health-text p{color:#ca9c98} +.health-banner.loading{background:var(--panel);border-color:var(--line)} +.loading .health-icon{background:var(--panel2);color:var(--muted)} +.loading .health-text strong{color:var(--text)} +.loading .health-text p{color:var(--muted)} +.current-issues{display:grid;gap:10px;margin:0 0 20px} +.issue{border:1px solid #67422f;background:#271f1c;border-radius:8px;padding:16px 20px} +.issue strong{color:var(--amber);font-size:14px} +.issue p{margin:5px 0;font-size:12px;color:#c6bbb1} +.issue small{color:#a89583} +.issue.error{border-color:#673b3b;background:#281d20} +.issue.error strong{color:var(--red)} +.overview-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px} +.metric-card{border:1px solid var(--line);background:var(--panel);border-radius:9px;padding:18px 20px 15px;min-width:0} +.card-top{display:flex;align-items:center;justify-content:space-between;color:#a6b6c1;font-size:12px} +.card-top svg{color:#647e8e;width:19px;height:19px} +.metric-value{font:38px/1.25 var(--mono);letter-spacing:-1.8px;margin:17px 0 12px;color:#eef4f4;white-space:nowrap} +.metric-value small{font:12px var(--sans);color:var(--muted);letter-spacing:0;margin-left:7px} +.metric-value.word{font:28px/1.7 var(--sans);letter-spacing:1px;margin-top:12px;margin-bottom:6px;color:var(--green)} +.meter{height:4px;border-radius:4px;background:#26343e;overflow:hidden} +.meter i{display:block;height:100%;width:0;background:var(--green);border-radius:4px;transition:width .6s} +.meter.violet i{background:var(--purple)} +.metric-bottom{display:flex;justify-content:space-between;align-items:center;margin-top:15px;gap:5px;white-space:nowrap;font-size:10px;color:#8195a3} +.metric-bottom strong{color:#b9c6cd;font-size:10px;font-weight:400} +.gpu-online-label,.disk-health-detail{font-size:11px;color:#a4b5bf;display:flex;gap:6px;align-items:center;height:4px;line-height:16px;margin-top:17px;margin-bottom:5px} +.disk-health-detail{height:10px;margin-top:5px} +.section-heading{display:flex;align-items:center;justify-content:space-between;gap:18px;margin:34px 0 16px;scroll-margin-top:24px} +.section-heading h2{font-size:19px;font-weight:600;letter-spacing:.4px;margin:5px 0} +.section-heading p{font-size:11px;color:var(--muted);margin:5px 0 0} +.section-heading.compact{margin:25px 0 13px} +.section-heading.compact>div{display:flex;align-items:center;gap:15px} +.compact h2{font-size:15px;margin:0} +.compact span{font-size:10px;color:var(--muted)} +.subtle-pill{border:1px solid var(--line);padding:3px 9px;border-radius:4px;white-space:nowrap;font:10px var(--mono)!important;color:#91a6b3!important} +.gpu-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px} +.gpu-card{border:1px solid var(--line);border-radius:9px;background:var(--panel);padding:19px 21px;position:relative;overflow:hidden;--accent:var(--blue)} +.gpu-card::before{content:"";position:absolute;top:19px;bottom:19px;left:0;width:3px;background:var(--accent);border-radius:0 4px 4px 0} +.gpu-card .gpu-head{display:flex;gap:10px;align-items:center} +.gpu-label{font:11px var(--mono);color:var(--accent);background:#213443;padding:3px 7px;border-radius:4px} +.gpu-card h3{font-size:13px;margin:0;font-weight:500} +.gpu-state{font-size:10px;color:var(--green);margin-left:auto;display:flex;gap:6px;align-items:center} +.gpu-card .gpu-content{display:flex;gap:28px;align-items:center;margin:19px 0 11px} +.gpu-util{font:43px/1 var(--mono);letter-spacing:-1.8px;min-width:115px} +.gpu-util small{font:12px var(--sans);color:var(--muted);margin-left:3px;letter-spacing:0} +.gpu-util span{display:block;font:10px var(--sans);color:var(--muted);letter-spacing:0;margin-top:8px} +.gpu-numbers{flex:1;display:grid;grid-template-columns:1fr 1fr;gap:14px} +.gpu-number>span{display:block;font-size:10px;color:var(--muted);margin-bottom:4px} +.gpu-number strong{font:20px var(--mono)} +.gpu-number small{font:10px var(--sans);color:#899daa;margin-left:5px} +.gpu-vram{display:flex;justify-content:space-between;gap:10px;font-size:10px;color:var(--muted);margin:16px 0 7px} +.gpu-vram strong{font:10px var(--mono);color:#b3c2cb} +.gpu-card .meter{height:3px} +.gpu-card .meter i{background:var(--accent)} +.gpu-meta{display:flex;justify-content:space-between;gap:9px;border-top:1px solid #24313b;padding-top:11px;margin-top:13px;font:9px var(--mono);color:#738c9d;white-space:nowrap} +.skeleton{display:grid;place-items:center;color:var(--muted);min-height:230px;animation:fade 2s infinite} +@keyframes fade{50%{opacity:.45} +} +.system-strip{display:flex;justify-content:space-between;flex-wrap:wrap;gap:12px 20px;margin-top:14px;border:1px solid #202c35;border-radius:7px;background:#111a21;padding:13px 17px;color:#7d939f;font-size:10px} +.system-strip span{display:flex;gap:8px;align-items:center} +.system-strip svg{width:13px;height:13px} +.system-strip strong{font:10px var(--mono);color:#afc1cd} +.segmented{display:flex;gap:2px;padding:4px;border:1px solid var(--line);background:#121c23;border-radius:7px;white-space:nowrap} +.segmented button{border:none;color:var(--muted);background:none;padding:6px 11px;border-radius:4px;font-size:11px} +.segmented button.selected{background:#304131;color:#cff5b3} +.segmented button:hover{color:white} +.history-note{font-size:10px;color:#8398a6;border-left:2px solid #3d505e;padding-left:10px;margin:-4px 0 15px;line-height:1.8} +.chart-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px} +.chart-card{border:1px solid var(--line);border-radius:9px;background:var(--panel);padding:18px 20px 12px;min-width:0} +.chart-heading{display:flex;justify-content:space-between;gap:10px;align-items:center} +.chart-heading h3{margin:0;font-size:13px;font-weight:500} +.chart-heading>span{font:10px var(--mono);color:#728c9e} +.chart-legend{display:flex;gap:17px;flex-wrap:wrap;margin:13px 0 4px;font-size:9px;color:#a6b8c5;min-height:17px} +.legend-item{display:flex;align-items:center;gap:6px} +.legend-swatch{width:13px;height:3px;border-radius:3px} +.chart{height:190px;position:relative} +.chart svg{width:100%;height:100%;stroke:none;overflow:visible;display:block} +.chart text{font:9px var(--mono);fill:#6e8798} +.chart .grid-line{stroke:#27343e;stroke-dasharray:3 5;stroke-width:.6} +.chart .series-line{fill:none;stroke-width:1.7;stroke-linejoin:round;stroke-linecap:round} +.chart .cursor{stroke:#93a6b1;stroke-width:.7;stroke-dasharray:4 3} +.chart .chart-overlay{fill:transparent;cursor:crosshair} +.chart-tooltip{position:absolute;top:18px;background:#202d37;border:1px solid #455866;border-radius:7px;padding:9px 12px;z-index:2;box-shadow:0 8px 24px #0007;font-size:10px;pointer-events:none;min-width:140px} +.chart-tooltip strong{display:block;color:#dce8ef;font:10px var(--mono);margin-bottom:7px} +.chart-tooltip p{margin:3px 0;display:flex;justify-content:space-between;gap:15px;color:#b5c5d0} +.chart-empty{position:absolute;inset:0;display:grid;place-content:center;text-align:center;color:#75909e;font-size:11px;pointer-events:none;gap:3px} +.chart-empty small{font-size:9px;color:#5c7482} +.text-button{display:inline-flex;align-items:center;justify-content:center;gap:7px;background:#18242d;border:1px solid #34444f;border-radius:5px;padding:7px 12px;color:#afc2cc;font-size:11px;white-space:nowrap} +.text-button svg{width:14px;height:14px} +.event-panel{border:1px solid var(--line);background:var(--panel);border-radius:9px;overflow:hidden} +.event-toolbar{padding:17px 20px;display:flex;align-items:flex-end;gap:10px;border-bottom:1px solid var(--line);flex-wrap:wrap} +.event-toolbar label{display:grid;gap:5px;font-size:10px;color:#8ea3b1} +.event-toolbar select,.event-toolbar input{border:1px solid #34434e;border-radius:5px;background:#101a22;color:#cedae2;height:33px;padding:5px 9px;font-size:11px;max-width:200px;min-width:110px} +.event-toolbar input[type=date]{max-width:135px;font-family:var(--mono);font-size:10px} +.event-toolbar .search-label{flex:1;min-width:150px} +.event-toolbar input[type=search]{width:100%;max-width:none} +.primary-button{border:1px solid #b3e98a;background:var(--green);color:#233423;padding:7px 14px;border-radius:5px;font-size:11px;font-weight:600;height:33px;white-space:nowrap} +.primary-button:hover{background:#c7fa9d} +.event-caption{display:flex;justify-content:space-between;gap:15px;font-size:10px;color:#8298a8;padding:13px 20px;background:#121c24} +.event-caption>span:first-child{color:#b1c3ce} +.event-row{border-top:1px solid #26343f} +.event-row>summary{display:flex;gap:13px;align-items:center;padding:15px 20px;list-style:none;cursor:pointer} +.event-row>summary::-webkit-details-marker{display:none} +.event-row>summary:hover{background:#1a2630} +.event-level{font-size:10px;border:1px solid #4c3d2c;background:#332b20;color:var(--amber);border-radius:4px;padding:2px 6px;white-space:nowrap} +.event-level.error{border-color:#663c3b;background:#392526;color:var(--red)} +.event-level.info{border-color:#354b5b;background:#23323f;color:#9ab7cf} +.event-main{flex:1;min-width:0} +.event-main strong{display:block;font-size:12px;font-weight:500} +.event-main p{margin:3px 0 0;color:#7891a2;font-size:9px;font-family:var(--mono);overflow-wrap:anywhere} +.event-state{border:1px solid #344651;background:#1a2932;color:#93acbb;padding:3px 7px;border-radius:4px;font-size:9px;white-space:nowrap} +.event-state.active{color:var(--red);background:#382628;border-color:#654044} +.event-state.recovered{color:var(--green);background:#24372c;border-color:#3c5b40} +.event-state.related{color:var(--amber);background:#372c20;border-color:#665036} +.event-time{text-align:right;font:10px var(--mono);color:#afc1cd;white-space:nowrap;min-width:135px} +.event-time small{display:block;font:9px var(--sans);color:#748d9e;margin-top:2px} +.event-chevron{font-size:14px;color:#738b99;transform:rotate(0deg);transition:transform .2s} +.event-row[open] .event-chevron{transform:rotate(90deg)} +.event-detail{padding:3px 20px 19px 64px;font-size:11px;color:#9cb2c1} +.event-meaning{margin:0 0 10px;color:#c2d0d9} +.maintenance{background:#1d2b32;border-left:2px solid #6a879a;padding:10px 13px;margin:10px 0;color:#c5d4dc;line-height:1.9} +.maintenance strong{color:var(--green);font-size:10px;display:block} +.time-grid{display:grid;grid-template-columns:1fr 1fr;gap:6px 18px;font-size:10px;margin:12px 0} +.time-grid span{color:#738d9f;display:block;font-size:9px} +.time-grid strong{font-family:var(--mono);font-weight:400;overflow-wrap:anywhere} +.time-warning{padding:9px 12px;border:1px solid #5d4a32;border-radius:4px;background:#2b251d;font-size:10px;color:#d9b67d;margin:10px 0} +.raw-log{margin:10px 0 0;background:#0c151d;border:1px solid #293a47;border-radius:5px;padding:10px 12px;color:#9bb4c6;font:10px/1.8 var(--mono);white-space:pre-wrap;word-break:break-word} +.pagination{display:flex;align-items:center;justify-content:space-between;padding:15px 20px;border-top:1px solid var(--line);font-size:10px;color:#869eac} +.pagination>div{display:flex;gap:7px} +.empty-events{padding:35px;text-align:center;color:var(--muted);font-size:12px} +.empty-events strong{display:block;color:#c9d9e2;font-weight:500;margin-bottom:5px} +.guide{margin-top:34px;border:1px solid #293944;border-radius:9px;padding:21px;background:#121d25;scroll-margin-top:25px} +.guide>div:first-child{display:flex;align-items:center;gap:9px} +.guide>div:first-child svg{color:#91b0c4;width:18px;height:18px} +.guide h2{font-size:14px;font-weight:500;margin:0} +.guide-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:28px;margin-top:20px} +.guide-number{font:11px var(--mono);color:#8dad9b} +.guide h3{font-size:12px;font-weight:500;margin:6px 0} +.guide p{font-size:10px;color:#839baa;line-height:1.9;margin:0} +footer{display:flex;justify-content:space-between;gap:20px;padding-top:25px;color:#607b8d;font-size:9px} +.toast{position:fixed;bottom:24px;left:50%;transform:translateX(-50%);background:#304438;border:1px solid #6b886c;color:#e0f0de;padding:12px 20px;border-radius:7px;z-index:50;box-shadow:0 8px 20px #0006;font-size:12px} +body:fullscreen main{margin-left:0} +body:fullscreen .sidebar{display:none} +@media(min-width:1700px){main{padding-left:50px;padding-right:50px} +.chart{height:230px} +.metric-card{padding:23px} +.gpu-card{padding:23px} +} +@media(max-width:1200px){.sidebar{width:185px;padding-left:13px;padding-right:13px} +.brand{font-size:17px;gap:9px} +.brand-mark{width:33px;height:33px} +.brand small{font-size:7px} +main{margin-left:185px;padding:27px 23px} +.metric-card{padding:15px} +.metric-value{font-size:33px} +.metric-bottom{font-size:9px;flex-wrap:wrap} +.gpu-card{padding:17px} +.gpu-content{gap:14px!important} +.gpu-util{font-size:37px;min-width:95px} +.gpu-meta{font-size:8px} +.event-toolbar{gap:8px} +.event-state{max-width:150px;white-space:normal} +.compact>div>span{display:none} +} +@media(max-width:950px){.sidebar{width:66px;align-items:center;padding:23px 8px} +.brand>span:not(.brand-mark),.nav-caption,.sidebar nav a:not(.active) .nav-dot,.sidebar-node,.sidebar-footer{display:none} +.sidebar nav{margin-top:37px;width:100%} +.sidebar nav a{font-size:0;padding:12px;gap:0;justify-content:center} +.sidebar nav a svg{width:20px} +.nav-dot{display:none} +.nav-count{font-size:8px;position:absolute;margin:0 0 0 31px} +.sidebar nav a{position:relative} +main{margin-left:66px;padding:25px 22px} +.overview-grid{gap:11px} +.gpu-meta{flex-wrap:wrap;gap:4px} +.gpu-numbers{gap:12px} +.gpu-card h3{font-size:11px} +.gpu-state{font-size:9px} +.event-time{min-width:100px;font-size:9px} +.event-state{font-size:8px} +.event-row>summary{gap:9px;padding:14px} +.guide-grid{gap:17px} +.segmented button{padding:6px 8px} +} +@media(max-width:720px){main{padding:21px 15px} +.page-header{margin-bottom:20px} +h1{font-size:23px} +.eyebrow{font-size:8px} +.header-actions{gap:10px} +.live-badge{font-size:9px} +.page-header p{font-size:9px} +.overview-grid{grid-template-columns:repeat(2,minmax(0,1fr))} +.metric-value{font-size:34px;margin-top:12px} +.metric-card{padding:15px 17px} +.health-banner{padding:14px;gap:10px} +.health-icon{width:30px;height:30px} +.health-icon svg{width:20px} +.health-text strong{font-size:13px} +.health-text p{font-size:10px} +.health-time{padding-left:10px;font-size:9px} +.health-time strong{font-size:9px} +.gpu-grid{grid-template-columns:1fr} +.gpu-card h3{font-size:13px} +.gpu-content{gap:25px!important} +.gpu-meta{font-size:9px} +.chart-grid{grid-template-columns:1fr} +.section-heading{flex-wrap:wrap;margin-top:28px} +.section-heading h2{font-size:18px} +.section-heading p{font-size:10px} +.segmented{width:100%;justify-content:space-around} +.segmented button{flex:1} +.section-heading.compact{flex-wrap:nowrap} +.event-caption{font-size:9px;padding:12px} +.event-caption span:last-child{display:none} +.event-row>summary{flex-wrap:wrap;gap:8px} +.event-main{min-width:150px} +.event-state{order:5;margin-left:42px;font-size:9px} +.event-time{margin-left:auto;min-width:130px;font-size:9px} +.event-chevron{order:6;margin-left:auto} +.event-detail{padding:5px 14px 17px} +.time-grid{grid-template-columns:1fr} +.event-toolbar{padding:14px} +.event-toolbar label{flex:1;min-width:110px} +.event-toolbar input,.event-toolbar select{max-width:none;width:100%;min-width:0} +.event-toolbar input[type=date]{max-width:none} +.event-toolbar .search-label{min-width:100%} +.primary-button{width:100%} +.guide-grid{grid-template-columns:1fr;gap:20px} +.guide-number{float:left;margin-right:13px;margin-top:5px} +.guide p{font-size:11px} +.guide h3{font-size:13px} +.guide h2{font-size:12px} +footer{flex-direction:column;gap:4px;padding-bottom:10px} +} +@media(max-width:420px){.sidebar{display:none} +main{margin-left:0;padding:20px 13px} +h1{font-size:22px} +.health-time{display:none} +.gpu-numbers{gap:9px} +.gpu-util{min-width:100px} +.metric-card{padding:14px} +.page-header{display:block;position:relative} +.header-actions{position:absolute;right:0;top:0;gap:8px} +.page-header .eyebrow{padding-right:130px;min-height:24px} +.page-header h1{font-size:23px;margin-top:10px;white-space:nowrap} +} +@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto} +*{animation:none!important;transition:none!important} +} + +/* Readable incidents and scroll-linked navigation, revision 1.1. */ +main{max-width:none} +.section-context{position:sticky;top:0;z-index:12;display:flex;align-items:center;justify-content:space-between;gap:14px;padding:11px 14px;margin:-8px 0 20px;background:#101b24f2;border:1px solid #2c404d;border-radius:7px;backdrop-filter:blur(12px)} +.section-context>div:first-child{display:flex;gap:11px;align-items:center} +.context-label{font-size:11px;color:#87a0b0} +.section-context strong{font-size:13px;font-weight:500;color:#e3ecef} +.context-actions{display:flex;gap:8px} +.context-button{background:transparent;border:1px solid #3c505d;border-radius:5px;padding:4px 10px;color:#b9cdd8;font-size:11px} +.context-button:hover{border-color:var(--green);color:var(--green)} +.reading-progress{height:2px;position:absolute;bottom:-1px;left:0;right:0;overflow:hidden;border-radius:0 0 7px 7px} +.reading-progress i{display:block;height:100%;width:0;background:var(--green);transition:width .1s linear} +.sidebar nav a.active:after{content:'';width:5px;height:5px;border-radius:50%;background:var(--green);margin-left:auto} +.sidebar nav a:not(.active):after{display:none} +.event-explainer{margin:0 0 14px;padding:14px 17px;border:1px solid #425344;border-radius:7px;background:#1a2823;display:grid;gap:4px} +.event-explainer strong{font-size:13px;color:#d0e3c8;font-weight:500} +.event-explainer span{font-size:12px;color:#99b1a5} +.event-row>summary{padding:18px 20px;gap:15px} +.event-main strong{font-size:15px;line-height:1.7} +.event-main p{font-size:12px;font-family:var(--sans);color:#91a9b9;margin-top:4px} +.event-level{font-size:11px;padding:3px 7px} +.event-state{font-size:11px;padding:4px 8px} +.event-state.active.warning{color:var(--amber);background:#3a2c1a;border-color:#785a30} +.event-time{font-size:12px;min-width:160px} +.event-time small{font-size:10px} +.event-detail{padding:4px 22px 23px 22px;font-size:13px;line-height:1.85} +.plain-answer{padding:10px 13px;margin-bottom:5px;border-left:2px solid #3a5262} +.plain-answer>strong{display:block;color:#cfdee7;font-weight:500;font-size:12px} +.plain-answer p{margin:3px 0 0;color:#a5becd;font-size:13px} +.maintenance{margin:12px 0 15px;padding:14px 17px} +.maintenance>strong{font-size:12px} +.maintenance-priority{margin:3px 0 7px;font-size:14px;color:#e6f1e3} +.maintenance ol{margin:5px 0 0;padding-left:21px;font-size:13px} +.maintenance li{margin:4px 0} +.friendly-times{background:#111d25;padding:12px 15px;border:1px solid #2a3b47;border-radius:6px;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px} +.friendly-times span{font-size:11px;margin-bottom:3px} +.friendly-times strong{font-size:12px;color:#bdd0dd} +.record-summary{font-size:12px;color:#91a9b8;line-height:1.8} +.time-warning{font-size:12px} +.technical-details{border:1px solid #2d414f;border-radius:6px;margin-top:15px} +.technical-details>summary{cursor:pointer;padding:10px 13px;font-size:12px;color:#91adbf;background:#121e27} +.technical-details>summary:hover{color:var(--green)} +.technical-body{padding:2px 13px 14px} +.technical-body .time-grid{font-size:11px;grid-template-columns:repeat(2,minmax(0,1fr))} +.technical-body .time-grid span{font-size:10px} +.technical-body .raw-log{font-size:11px} +.event-toolbar label{font-size:11px} +.event-toolbar select,.event-toolbar input{font-size:12px;height:36px} +.primary-button{height:36px;font-size:12px} +.event-caption{font-size:11px;line-height:1.8} +.mobile-section-nav{display:none} +@media(min-width:1700px){.card-top{font-size:14px}.metric-bottom,.metric-bottom strong{font-size:12px}.gpu-card h3{font-size:15px}.gpu-number>span,.gpu-vram,.gpu-vram strong{font-size:12px}.system-strip,.system-strip strong{font-size:12px}.gpu-meta{font-size:11px}.chart-heading h3{font-size:15px}.chart-legend{font-size:11px}.chart{height:235px}} +@media(max-width:1100px){.friendly-times{grid-template-columns:repeat(2,minmax(0,1fr))}.event-row>summary{gap:10px}.event-state{font-size:10px}.event-time{min-width:140px;font-size:10px}} +@media(max-width:950px){.mobile-section-nav{position:fixed;display:grid;grid-template-columns:repeat(4,1fr);bottom:0;left:66px;right:0;background:#111d27f5;border-top:1px solid #3a4c57;z-index:25;padding:8px 6px calc(8px + env(safe-area-inset-bottom));backdrop-filter:blur(12px)}.mobile-section-nav a{display:flex;gap:7px;align-items:center;justify-content:center;font-size:12px;color:#8da8b9;padding:7px 4px;border-radius:5px}.mobile-section-nav a.active{color:var(--green);background:#293a2a}.mobile-section-nav svg{width:17px;height:17px}main{padding-bottom:90px}.sidebar nav a.active:after{position:absolute;right:3px;width:4px;height:4px}.context-button{font-size:10px}.section-context{margin-bottom:16px}} +@media(max-width:720px){.event-main{min-width:130px}.event-main strong{font-size:14px}.event-row>summary{padding:15px}.event-state{margin-left:0}.event-time{margin-left:auto;min-width:140px}.event-detail{padding:3px 14px 17px}.event-caption{padding:12px 14px;font-size:11px}.plain-answer p,.maintenance ol{font-size:12px}.friendly-times strong{font-size:11px}.technical-body .time-grid{grid-template-columns:1fr}.context-label{display:none}.section-context{padding:9px 10px}.section-context strong{font-size:12px}.context-button{padding:4px 7px}.event-explainer strong{font-size:12px}.event-explainer span{font-size:11px}} +@media(max-width:420px){.mobile-section-nav{left:0}.context-button{font-size:9px}.section-context strong{font-size:11px}.event-state{font-size:9px}.event-time{min-width:125px;font-size:9px}.friendly-times{grid-template-columns:1fr}.technical-body{padding-left:9px;padding-right:9px}} + +/* Scheduled tasks */ +.task-panel{border:1px solid var(--line);border-radius:9px;background:var(--panel);overflow:hidden;min-height:max(380px,calc(100vh - 300px));display:flex;flex-direction:column} +#task-manager,#task-list{display:flex;flex-direction:column;flex:1} +#task-locked,#task-list>.task-empty{flex:1;display:flex;flex-direction:column;align-items:center;justify-content:center} +.task-authbar{display:flex;justify-content:space-between;gap:18px;align-items:center;padding:18px 21px;background:#17252c;border-bottom:1px solid #30424d} +.task-authbar strong{font-size:14px;font-weight:500;color:#d5e5ed} +.task-authbar p{font-size:12px;color:#91aab9;margin:4px 0 0} +.task-auth-actions{display:flex;gap:8px;flex-shrink:0} +.task-empty{text-align:center;padding:44px 20px;color:#a5bdca} +.task-empty>svg{width:31px;height:31px;color:var(--green);margin-bottom:7px} +.task-empty h3{font-size:17px;font-weight:500;color:#d6e4ec;margin:6px 0} +.task-empty p{font-size:13px;line-height:1.9;margin:7px 0} +.task-empty>span{font-size:11px;color:#7993a4} +.task-toolbar{display:flex;justify-content:space-between;align-items:center;padding:15px 20px;gap:15px} +.task-toolbar>div{display:flex;align-items:center;gap:12px;flex-wrap:wrap} +.task-toolbar strong{font-size:13px;font-weight:500;color:#d5e3eb} +.task-toolbar span{font-size:10px;color:#7f99aa} +.task-table-heading,.task-row{display:grid;grid-template-columns:minmax(150px,1.1fr) minmax(180px,1.2fr) minmax(145px,.9fr) minmax(200px,1.2fr);gap:18px;padding:13px 20px;align-items:center} +.task-table-heading{background:#101b23;color:#7e9aaa;font-size:11px} +.task-row{border-top:1px solid #283b47;padding-top:20px;padding-bottom:20px} +.task-name-cell,.task-schedule-cell,.task-status-cell{min-width:0} +.task-name-cell>strong{font-size:14px;font-weight:500;overflow-wrap:anywhere} +.task-name-cell>p{font-size:11px;color:#829daa;line-height:1.8;margin:5px 0} +.task-row-meta{display:flex;gap:7px;flex-wrap:wrap;margin-top:6px} +.task-row-meta span{font-size:10px;color:#8eacbe;padding:1px 6px;background:#1d303c;border-radius:4px} +.task-row-meta .task-danger-label{color:var(--amber);background:#3a2e1f} +.task-schedule-cell>strong{font-size:12px;font-weight:400;overflow-wrap:anywhere;display:block;color:#c4d6e0} +.task-schedule-cell>span{display:block;font:10px/1.8 var(--mono);color:#89a4b5;margin-top:6px} +.task-status-cell{display:flex;align-items:flex-start;gap:6px;flex-wrap:wrap} +.task-status-cell>small{font-size:10px;color:#809dac;flex-basis:100%;margin-top:3px} +.task-state,.task-enabled{font-size:10px;border-radius:4px;padding:2px 7px;display:inline-flex;align-items:center;gap:4px;border:1px solid #3d5261;color:#a4bac9;background:#1a2a34;white-space:nowrap} +.task-state.success{background:#253c2b;border-color:#466341;color:#b7ed97} +.task-state.failed{background:#39282b;border-color:#734a49;color:#ffaaa1} +.task-state.running{background:#22384a;border-color:#42657c;color:#96d0ff} +.task-state.warning{background:#3a3020;border-color:#6c5734;color:#ebc181} +.task-enabled.enabled{border-color:#3e6451;color:#a4dca8;background:#21382b} +.task-row-actions{display:flex;gap:4px 9px;flex-wrap:wrap} +.task-action{background:none;border:none;padding:4px 0;font-size:12px;color:#a9d88f;white-space:nowrap} +.task-action:hover{color:#d2fabc;text-decoration:underline} +.task-action.danger{color:#d9958d} +.task-footnote{font-size:11px;color:#7996a7;background:#101b23;padding:13px 20px;border-top:1px solid #2b3b47;line-height:1.9} +.task-dialog{width:min(810px,calc(100% - 26px));max-height:90vh;background:#14212a;color:var(--text);border:1px solid #4b6272;border-radius:12px;padding:0;box-shadow:0 30px 100px #000b;overflow:hidden} +.task-dialog::backdrop{background:#030b12bf;backdrop-filter:blur(3px)} +.task-dialog.small-dialog{max-width:510px} +.task-dialog.log-dialog{width:min(1150px,calc(100% - 26px))} +.task-dialog>form{display:flex;flex-direction:column;max-height:90vh} +.dialog-heading{padding:21px 24px 17px;display:flex;justify-content:space-between;gap:16px;align-items:flex-start;border-bottom:1px solid #30414f;background:#192a34} +.dialog-heading h2{font-size:19px;font-weight:500;margin:0;color:#e6eff4} +.dialog-heading p{font-size:12px;color:#8eacbd;margin:5px 0 0;line-height:1.8} +.dialog-close{border:1px solid #3b5160;border-radius:5px;background:#1a2a36;color:#9eb8c9;line-height:1;width:29px;height:29px;font-size:22px;flex-shrink:0} +.dialog-close:hover{color:white;border-color:#7695aa} +.dialog-content{padding:22px 24px;overflow:auto;line-height:1.9;min-height:0} +.task-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:17px 20px} +.task-form-grid label{display:grid;gap:7px;color:#b3c9d7;font-size:12px;align-content:start;min-width:0} +.task-form-grid input:not([type=checkbox]):not([type=file]),.task-form-grid select,.task-form-grid textarea{width:100%;border:1px solid #3a5264;border-radius:5px;background:#0e1a23;color:#d8e6f0;padding:9px 11px;font:12px/1.65 var(--sans);height:38px;min-width:0} +.task-form-grid textarea{height:auto;resize:vertical;font-family:var(--mono);line-height:1.75;font-size:12px;min-height:110px;tab-size:2} +.task-form-grid input[type=checkbox]{accent-color:var(--green);width:15px;height:15px} +.task-form-grid small{font-size:11px;color:#7e9aab;line-height:1.7} +.full-field{grid-column:1/-1} +.task-form-divider{display:flex;align-items:center;justify-content:space-between;border-top:1px solid #314651;padding-top:15px;margin-top:2px} +.task-form-divider strong{font-size:13px;font-weight:500;color:#d3e4ee} +.task-form-divider span{font:10px var(--mono);color:#7f9eaf} +.field-label-row{display:flex;justify-content:space-between;align-items:center;gap:10px} +.inline-button{border:none;background:none;color:var(--green);font-size:11px;padding:0} +.weekday-field>span{font-size:12px;color:#b1c9d8} +#task-weekdays{display:flex;gap:10px;margin-top:9px;flex-wrap:wrap} +#task-weekdays label{display:flex;gap:7px;align-items:center;background:#1b303d;border:1px solid #385164;border-radius:5px;padding:7px 10px;min-width:54px} +.schedule-preview{display:flex;gap:12px;align-items:center;background:#152a29;border:1px solid #365347;border-radius:5px;padding:11px 13px} +.schedule-preview span{font-size:11px;color:#a5c2b0;line-height:1.9} +.schedule-preview button{flex-shrink:0} +.template-hint{padding:12px 14px;border:1px solid #705938;color:#e3c493;background:#332d21;border-radius:5px;font-size:12px;line-height:1.8} +.task-advanced{border:1px solid #30495b;border-radius:5px} +.task-advanced>summary{padding:10px 13px;color:#9ebacb;cursor:pointer;font-size:12px} +.task-advanced .task-form-grid{padding:4px 13px 15px} +.task-form-grid .task-enabled-field{display:flex;align-items:flex-start;gap:10px;background:#1f3028;padding:13px 14px;border:1px solid #3b5141;border-radius:5px} +.task-enabled-field small{display:block;margin-top:3px;color:#8eae97} +.task-enabled-field input{margin-top:4px} +.dialog-footer{padding:15px 24px;display:flex;justify-content:space-between;gap:15px;align-items:center;border-top:1px solid #354a58;background:#182932} +.dialog-footer>span{font-size:11px;color:#8ba8ba} +.dialog-footer>div{display:flex;gap:8px} +.form-error{background:#3b2529;border:1px solid #75474b;color:#ffb3ad;border-radius:5px;font-size:12px;line-height:1.8;padding:10px 12px;margin:0} +#task-confirm-message{font-size:13px;color:#c4d8e5;margin:0;white-space:pre-wrap} +#task-confirm-code{max-height:230px;overflow:auto;font-size:11px} +.run-layout{display:grid;grid-template-columns:290px minmax(0,1fr);height:min(560px,70vh);min-height:300px} +.task-runs-list{overflow:auto;background:#101d27;border-right:1px solid #304756;padding:10px} +.run-item{background:none;border:1px solid transparent;display:grid;gap:5px;text-align:left;width:100%;color:#a5bdcc;border-radius:6px;padding:12px;margin-bottom:7px;font-size:11px} +.run-item:hover,.run-item.selected{background:#203a45;border-color:#3f6271} +.run-item>.task-state{justify-self:start} +.run-item strong{font:11px var(--mono);color:#c7dce9} +.run-item>span:not(.task-state){font-size:12px;overflow-wrap:anywhere} +.run-item small{font-size:10px;color:#87a5b9} +.task-no-runs{font-size:12px;color:#90aebe;line-height:1.9;padding:12px} +.task-run-output{display:flex;flex-direction:column;min-width:0;background:#0b1620} +.task-run-meta{padding:13px 17px;border-bottom:1px solid #2b4150;display:flex;gap:10px;align-items:center;flex-wrap:wrap;font-size:11px;color:#9fb9c9} +#task-run-output{flex:1;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;font:12px/1.9 var(--mono);color:#c0d6e4;padding:17px;margin:0;tab-size:2} +@media(max-width:1100px){.task-table-heading,.task-row{grid-template-columns:1fr 1.2fr 1fr}.task-table-heading>span:last-child{display:none}.task-row-actions{grid-column:1/-1;justify-content:flex-end;border-top:1px solid #293f4c;padding-top:10px}.task-row{gap:13px}.task-authbar{align-items:flex-start}.task-auth-actions{flex-wrap:wrap;justify-content:flex-end}} +@media(max-width:950px){.mobile-section-nav{grid-template-columns:repeat(5,1fr)}.mobile-section-nav a{gap:5px;font-size:11px}} +@media(max-width:720px){.task-authbar{flex-direction:column;padding:16px}.task-auth-actions{justify-content:flex-start}.task-toolbar{align-items:flex-start;padding:15px;flex-wrap:wrap}.task-table-heading{display:none}.task-row{grid-template-columns:1fr;padding:17px}.task-name-cell>strong{font-size:15px}.task-status-cell{align-items:center}.task-schedule-cell>strong{font-size:12px}.task-row-actions{justify-content:flex-start;gap:6px 17px}.task-action{font-size:13px}.task-status-cell>small{flex-basis:auto}.task-dialog{max-height:92vh}.task-dialog>form{max-height:92vh}.dialog-heading{padding:16px}.dialog-heading h2{font-size:17px}.dialog-heading p{font-size:11px}.dialog-content{padding:16px}.dialog-footer{padding:13px 16px;gap:9px}.dialog-footer>span{font-size:10px}.task-form-grid{grid-template-columns:1fr;gap:15px}.schedule-preview{flex-direction:column;align-items:flex-start}.task-form-divider{gap:10px}.run-layout{grid-template-columns:1fr;grid-template-rows:160px minmax(0,1fr);height:70vh}.task-runs-list{border-right:none;border-bottom:1px solid #2e4656}.run-item{grid-template-columns:auto 1fr;gap:6px 12px}.run-item small{grid-column:2}.task-empty{padding:30px 16px}.task-footnote{padding:12px 16px}} + diff --git a/static/tasks.js b/static/tasks.js new file mode 100644 index 0000000..992e212 --- /dev/null +++ b/static/tasks.js @@ -0,0 +1,176 @@ +/* Authenticated task management. Credentials are never saved in localStorage. */ +(()=>{ +'use strict'; +let auth={logged_in:false},tasks=[],listBusy=false,listRefreshPending=false,selectedRun=null,runsTask=null,confirmResolve=null,runReadSequence=0; +const states={running:['执行中','running'],success:['执行成功','success'],failed:['执行失败','failed'],timeout:['超时已停止','failed'],cancelled:['已停止','neutral'],interrupted:['执行被中断','warning']}; +function taskError(id,message){$(id).textContent=message;$(id).hidden=!message;} +async function request(path,body){ + const options={cache:'no-store',signal:AbortSignal.timeout(28000),credentials:'same-origin'}; + if(body!==undefined){options.method='POST';options.headers={'Content-Type':'application/json','X-Task-CSRF':auth.csrf||''};options.body=JSON.stringify(body);} + const response=await fetch('/api/tasks'+path,options); + const data=await response.json(); + if(!response.ok){ + if(response.status===401&&path!=='/login'){setAuth({logged_in:false});} + const error=new Error(data.error||'任务操作未完成。');error.status=response.status;throw error; + } + return data; +} +function setAuth(value){ + auth=value;$('task-manager').hidden=!value.logged_in;$('task-locked').hidden=!!value.logged_in; + $('task-create').disabled=!value.logged_in;$('task-login-open').hidden=!!value.logged_in; + $('task-change-password').hidden=!value.logged_in;$('task-logout').hidden=!value.logged_in; + $('task-auth-title').textContent=value.logged_in?'管理员已登录':'定时任务管理未登录'; + $('task-auth-description').textContent=value.logged_in?'可新增、编辑和执行任务。登录最长有效 8 小时;执行身份为 root。':'设备监控无需登录;创建、修改或执行服务器任务需要管理员登录。'; + if(!value.logged_in){tasks=[];$('task-list').replaceChildren();} + queueNavigation(); +} +async function refreshAuth(){ + try{const value=await request('/auth');setAuth(value);if(value.logged_in)await refreshList();} + catch(error){$('task-auth-title').textContent='任务管理服务暂不可用';$('task-auth-description').textContent=error.message;$('task-create').disabled=true;} +} +function openLogin(){taskError('task-login-error','');$('task-login-password').value='';$('task-login-dialog').showModal();$('task-login-password').focus();} +$('task-login-open').addEventListener('click',openLogin); +$('task-login-form').addEventListener('submit',async event=>{ + event.preventDefault();const button=event.submitter;button.disabled=true;taskError('task-login-error',''); + try{const result=await request('/login',{username:$('task-login-user').value.trim(),password:$('task-login-password').value});setAuth(result);$('task-login-password').value='';$('task-login-dialog').close();await refreshList();toast('已登录定时任务管理。');} + catch(error){taskError('task-login-error',error.message);}finally{button.disabled=false;} +}); +$('task-logout').addEventListener('click',async()=>{try{await request('/logout',{});setAuth({logged_in:false});document.querySelectorAll('.task-dialog[open]').forEach(dialog=>dialog.close());$('task-body').value='';$('task-run-output').textContent='';toast('已退出任务管理。');}catch(error){toast(error.message);}}); +document.querySelectorAll('[data-close-dialog]').forEach(button=>button.addEventListener('click',()=>$(button.dataset.closeDialog).close())); +function confirmAction(title,message,code=''){ + return new Promise(resolve=>{confirmResolve=resolve;$('task-confirm-heading').textContent=title;$('task-confirm-message').textContent=message;$('task-confirm-code').textContent=code;$('task-confirm-code').hidden=!code;$('task-confirm-dialog').showModal();}); +} +function finishConfirm(value){$('task-confirm-dialog').close();const resolve=confirmResolve;confirmResolve=null;resolve?.(value);} +$('task-confirm-no').addEventListener('click',()=>finishConfirm(false));$('task-confirm-yes').addEventListener('click',()=>finishConfirm(true));$('task-confirm-dialog').addEventListener('cancel',event=>{event.preventDefault();finishConfirm(false);}); +function stateBadge(state){const [text,cls]=states[state]||['尚未执行','neutral'];return el('span','task-state '+cls,text);} +function actionButton(label,action,task,cls=''){ + const button=el('button','task-action '+cls,label);button.type='button';button.addEventListener('click',()=>taskAction(action,task,button));return button; +} +function renderList(data){ + tasks=data.tasks||[];const holder=$('task-list');holder.replaceChildren(); + $('task-count').textContent=`共 ${tasks.length} 个任务 · ${tasks.filter(x=>x.enabled).length} 个已启用 · ${tasks.filter(x=>x.running).length} 个执行中`; + $('task-server-time').textContent='服务器时间 '+when(data.server_time); + if(!tasks.length){const empty=el('div','task-empty');empty.append(icon('clock'),el('h3','','还没有计划任务'),el('p','','点击“新增任务”,选择模板或填写自己的命令。任务默认停用,可先测试再启用。'));holder.append(empty);return;} + tasks.forEach(task=>{ + const row=el('article','task-row');row.dataset.taskId=task.id; + const name=el('div','task-name-cell');name.append(el('strong','',task.name));const meta=el('div','task-row-meta');meta.append(el('span','',({command:'命令',script:'脚本内容',file:'服务器脚本'})[task.mode]||task.mode),el('span','','root'));if(task.dangerous)meta.append(el('span','task-danger-label','重启/关机类'));name.append(meta);if(task.notes)name.append(el('p','',task.notes)); + const schedule=el('div','task-schedule-cell');schedule.append(el('strong','',task.schedule_label),el('span','',task.enabled?(task.next_run?'下次:'+when(task.next_run):'没有下次执行时间/单次任务已到期'):'计划已停用,不会自动执行')); + const status=el('div','task-status-cell');status.append(el('span','task-enabled '+(task.enabled?'enabled':''),task.enabled?'计划已启用':'计划已停用'),stateBadge(task.running?'running':task.last_run?.state));if(task.last_run)status.append(el('small','','最近:'+when(task.last_run.started))); + const actions=el('div','task-row-actions'); + if(task.running)actions.append(actionButton('停止本次','stop',task,'danger')); + else actions.append(actionButton('执行一次','run',task)); + actions.append(actionButton('日志','logs',task),actionButton('编辑','edit',task),actionButton(task.enabled?'停用':'启用','enable',task)); + if(!task.running)actions.append(actionButton('删除','delete',task,'danger')); + row.append(name,schedule,status,actions);holder.append(row); + });queueNavigation(); +} +async function refreshList(showMessage=false){if(!auth.logged_in)return;if(listBusy){listRefreshPending=true;return;}listBusy=true;try{renderList(await request(''));if(showMessage)toast('任务状态已更新。');}catch(error){if(error.status!==401)toast(error.message);}finally{listBusy=false;if(listRefreshPending){listRefreshPending=false;refreshList();}}} +$('task-refresh').addEventListener('click',()=>refreshList(true)); +async function taskAction(action,task,button){ + button.disabled=true; + try{ + if(action==='edit'){await openEditor(task.id);return;} + if(action==='logs'){await openRuns(task);return;} + if(action==='run'){ + const detail=await request('/item?id='+encodeURIComponent(task.id)); + const code=detail.definition.mode==='file'?detail.definition.path+' '+detail.definition.arguments:detail.definition.body; + if(!await confirmAction('立即执行“'+task.name+'”?',task.dangerous?'这可能会立刻重启或关闭服务器,中断计算任务和网页连接。只有确认现在可以中断业务时才继续。':'会立刻以 root 身份执行以下内容。手动执行不会自动启用这个任务的计划。',code.slice(0,1200)))return; + const result=await request('/run',{id:task.id});toast(result.message);await refreshList();await openRuns(task);return; + } + if(action==='enable'){ + if(!await confirmAction(task.enabled?'停用这项计划?':'启用这项计划?',task.enabled?'只停止以后的定时触发,已经开始的执行不会被强制中断。':`执行周期:${task.schedule_label}。${task.dangerous?'此任务可能重启/关闭服务器,请确认执行时段允许中断业务。':'到计划时间会以 root 身份执行。'}`))return; + toast((await request('/enable',{id:task.id,enabled:!task.enabled})).message); + } + if(action==='delete'){ + if(!await confirmAction('删除“'+task.name+'”?','将停止后续调度并删除任务定义,已有执行记录仍保留。'))return; + toast((await request('/delete',{id:task.id})).message); + } + if(action==='stop'){ + if(!await confirmAction('停止本次执行?','将终止本任务的执行进程。脚本可能尚未完成修改,请先确认可以中断。'))return; + toast((await request('/stop',{id:task.id})).message); + } + await refreshList(); + }catch(error){toast(error.message);}finally{button.disabled=false;} +} +function localDateValue(timestamp){const parts=new Intl.DateTimeFormat('sv-SE',{timeZone:'Asia/Shanghai',year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',hour12:false}).format(new Date(timestamp*1000));return parts.replace(' ','T');} +function updateEditorFields(){ + const mode=$('task-mode').value,cycle=$('task-cycle').value; + $('task-body-field').hidden=mode==='file';$('task-body').required=mode!=='file';$('task-body').disabled=mode==='file'; + $('task-path-field').hidden=mode!=='file';$('task-path').required=mode==='file';$('task-path').disabled=mode!=='file';$('task-arguments-field').hidden=mode!=='file'; + $('task-language-field').hidden=mode==='command';$('task-body-label').textContent=mode==='command'?'执行命令':'脚本内容';$('task-body').rows=mode==='command'?5:10; + for(const option of $('task-interpreter').options)option.disabled=option.value==='auto'&&mode!=='file'; + if(mode==='script'&&$('task-interpreter').value==='auto')$('task-interpreter').value='bash'; + document.querySelectorAll('[data-task-cycle]').forEach(wrapper=>{const shown=wrapper.dataset.taskCycle.split(' ').includes(cycle);wrapper.hidden=!shown;wrapper.querySelectorAll('input,select').forEach(input=>input.disabled=!shown);}); + $('task-every').max=cycle==='hours'?'168':'1440'; + $('task-save').textContent=$('task-enabled').checked?'保存并启用':'保存任务'; + $('task-preview-text').textContent='修改后可检查执行周期;错过时间不补跑。'; +} +function formDefinition(){ + const kind=$('task-cycle').value;const schedule={kind}; + if(['daily','weekly','monthly'].includes(kind))schedule.time=$('task-time').value; + if(kind==='weekly')schedule.weekdays=[...$('task-weekdays').querySelectorAll('input:checked')].map(x=>Number(x.value)); + if(kind==='monthly')schedule.day=Number($('task-day').value); + if(kind==='minutes'||kind==='hours')schedule.every=Number($('task-every').value); + if(kind==='once')schedule.at=$('task-once').value; + if(kind==='cron')schedule.expression=$('task-cron').value; + return {id:$('task-id').value||undefined,name:$('task-name').value.trim(),template:$('task-template').value, + mode:$('task-mode').value,interpreter:$('task-interpreter').value,body:$('task-body').value, + path:$('task-path').value,arguments:$('task-arguments').value,cwd:$('task-cwd').value, + timeout:Number($('task-timeout').value),notes:$('task-notes').value,enabled:$('task-enabled').checked,schedule}; +} +async function openEditor(id=null){ + $('task-editor-form').reset();$('task-id').value='';$('task-template-hint').hidden=true;taskError('task-editor-error',''); + $('task-once').value=localDateValue(Date.now()/1000+3600);$('task-enabled').checked=false; + $('task-editor-heading').textContent=id?'编辑定时任务':'新增定时任务'; + if(id){const task=await request('/item?id='+encodeURIComponent(id)),d=task.definition,s=d.schedule; + $('task-id').value=id;$('task-name').value=d.name;$('task-template').value=d.template||'custom';$('task-mode').value=d.mode; + $('task-interpreter').value=d.interpreter;$('task-body').value=d.body;$('task-path').value=d.path;$('task-arguments').value=d.arguments; + $('task-cwd').value=d.cwd;$('task-timeout').value=d.timeout;$('task-notes').value=d.notes;$('task-enabled').checked=!!task.enabled; + $('task-cycle').value=s.kind;$('task-time').value=s.time||'03:00';$('task-day').value=s.day||1;$('task-every').value=s.every||60;$('task-cron').value=s.expression||'0 3 * * *';if(s.at)$('task-once').value=s.at.slice(0,16); + $('task-weekdays').querySelectorAll('input').forEach(input=>input.checked=(s.weekdays||[0]).includes(Number(input.value))); + } + updateEditorFields();$('task-editor-dialog').showModal();$('task-name').focus(); +} +$('task-create').addEventListener('click',()=>openEditor().catch(error=>toast(error.message))); +['task-mode','task-cycle','task-enabled'].forEach(id=>$(id).addEventListener('change',updateEditorFields)); +$('task-preview').addEventListener('click',async()=>{if(!$('task-editor-form').reportValidity())return;const button=$('task-preview');button.disabled=true;try{const result=await request('/preview',formDefinition());$('task-preview-text').textContent=result.schedule_label+';预计下次:'+(result.next_run?when(result.next_run):'没有可计算的将来时间');}catch(error){$('task-preview-text').textContent=error.message;}finally{button.disabled=false;}}); +$('task-editor-form').addEventListener('submit',async event=>{ + event.preventDefault();taskError('task-editor-error','');const definition=formDefinition(); + const dangerous=/\b(reboot|shutdown|poweroff|halt)\b|\binit\s+[06]\b/i.test(definition.body+' '+definition.path); + if(definition.enabled&&!await confirmAction(dangerous?'保存并启用重启/关机类任务?':'保存并启用任务?',dangerous?'此任务将在计划时间中断服务器业务。请确认脚本和执行时段已核对。':'保存后会按设置的周期执行,不会立即运行。',definition.mode==='file'?definition.path:definition.body.slice(0,800)))return; + $('task-save').disabled=true; + try{const result=await request('/save',definition);$('task-editor-dialog').close();toast(result.message);await refreshList();} + catch(error){taskError('task-editor-error',error.message);}finally{$('task-save').disabled=false;} +}); +$('task-template').addEventListener('change',()=>{ + const type=$('task-template').value;$('task-enabled').checked=false;$('task-template-hint').hidden=type==='custom'; + if(type==='hardware'){$('task-name').value='检查显卡状态';$('task-mode').value='command';$('task-body').value='/usr/bin/nvidia-smi';$('task-cycle').value='daily';$('task-time').value='08:00';$('task-template-hint').textContent='只读取显卡状态,结果会出现在执行日志。可先保存为停用,再执行一次测试。';} + if(type==='reboot'){$('task-name').value='服务器定时重启';$('task-mode').value='command';$('task-body').value='/usr/bin/systemctl reboot';$('task-cycle').value='weekly';$('task-time').value='03:00';$('task-weekdays').querySelectorAll('input').forEach(input=>input.checked=input.value==='6');$('task-template-hint').textContent='这条命令会重启整台服务器,正在运行的 GPU 任务会中断。模板默认停用,请先确认允许重启的时间。';} + if(type==='certificate'){$('task-name').value='SSL 证书续签';$('task-mode').value='script';$('task-interpreter').value='bash';$('task-cycle').value='daily';$('task-time').value='03:10';$('task-body').value='#!/usr/bin/env bash\nset -euo pipefail\n# 先确认已有域名和续签配置,且没有其他重复的自动续签计划。\nif command -v certbot >/dev/null 2>&1; then\n certbot renew --quiet\nelif [ -x /root/.acme.sh/acme.sh ]; then\n /root/.acme.sh/acme.sh --cron --home /root/.acme.sh\nelse\n echo "未检测到 Certbot 或 acme.sh,请先配置证书续签工具。" >&2\n exit 1\nfi\n# 如需重新加载 Web 服务,请按实际服务另行配置 deploy-hook。';$('task-template-hint').textContent='这是续签示例,不会安装或配置域名。请先确认 Certbot/acme.sh 的现有配置,并检查是否已有续签任务。';} + updateEditorFields(); +}); +$('task-import-file').addEventListener('click',()=>$('task-upload').click()); +$('task-upload').addEventListener('change',async()=>{const file=$('task-upload').files[0];if(!file)return;if(file.size>65536){toast('脚本文件不能超过 64 KiB。');return;}$('task-mode').value='script';$('task-template').value='custom';$('task-interpreter').value=file.name.endsWith('.py')?'python3':'bash';$('task-body').value=await file.text();if(!$('task-name').value)$('task-name').value=file.name;updateEditorFields();$('task-upload').value='';}); +async function openRuns(task=null){runsTask=task;selectedRun=null;$('task-runs-heading').textContent=task?'执行记录 · '+task.name:'全部任务执行记录';$('task-run-meta').textContent='选择一次执行,查看输出和错误。';$('task-run-output').textContent='尚未选择执行记录。';$('task-runs-dialog').showModal();await refreshRuns();} +async function refreshRuns(){ + if(!auth.logged_in||!$('task-runs-dialog').open)return; + try{const data=await request(runsTask?'/runs?id='+encodeURIComponent(runsTask.id):'/history');const holder=$('task-runs-list');holder.replaceChildren(); + if(!data.runs.length){holder.append(el('p','task-no-runs','还没有执行记录。可以等待计划时间,或手动执行一次。'));return;} + data.runs.forEach(run=>{const button=el('button','run-item'+(selectedRun===run.id?' selected':''));button.type='button';button.dataset.runId=run.id;button.append(stateBadge(run.state),el('strong','',when(run.started)),el('span','',run.task_name),el('small','',(run.trigger==='manual'?'手动执行':'定时执行')+' · '+(run.duration!=null?num(run.duration,1)+' 秒':run.state==='running'?'执行中':'耗时未记录')));button.addEventListener('click',()=>loadRun(run.id));holder.append(button);}); + if(!selectedRun)selectedRun=data.runs[0].id;await loadRun(selectedRun); + }catch(error){if(error.status!==401)toast(error.message);} +} +async function loadRun(id){ + selectedRun=id;const sequence=++runReadSequence; + try{const data=await request('/runlog?run_id='+encodeURIComponent(id));if(sequence!==runReadSequence)return; + $('task-run-meta').replaceChildren(stateBadge(data.run.state),el('span','',`开始 ${when(data.run.started)} · 退出码 ${data.run.exit_code??'尚未取得'} · 修订 ${data.run.revision}`)); + const output=$('task-run-output'),stick=output.scrollHeight-output.scrollTop-output.clientHeight<50;output.textContent=(data.tail_only?'[仅显示最后 128 KiB]\n':'')+data.output;if(stick)output.scrollTop=output.scrollHeight; + $('task-runs-list').querySelectorAll('.run-item').forEach(button=>button.classList.toggle('selected',button.dataset.runId===id)); + }catch(error){if(error.status!==401)$('task-run-output').textContent=error.message;} +} +$('task-all-runs').addEventListener('click',()=>openRuns()); +$('task-change-password').addEventListener('click',()=>{$('task-password-form').reset();taskError('task-password-error','');$('task-password-dialog').showModal();}); +$('task-password-form').addEventListener('submit',async event=>{event.preventDefault();taskError('task-password-error','');if($('task-new-password').value!==$('task-repeat-password').value){taskError('task-password-error','两次新密码输入不一致。');return;}const button=event.submitter;button.disabled=true;try{const result=await request('/password',{old_password:$('task-old-password').value,new_password:$('task-new-password').value});$('task-password-dialog').close();setAuth({logged_in:false});toast(result.message);}catch(error){taskError('task-password-error',error.message);}finally{button.disabled=false;}}); +refreshAuth();setInterval(()=>{if(!document.hidden&&auth.logged_in)refreshList();},15000);setInterval(()=>{if(!document.hidden&&$('task-runs-dialog').open)refreshRuns();},3000); +document.addEventListener('visibilitychange',()=>{if(!document.hidden)refreshAuth();}); +})(); diff --git a/task_core.py b/task_core.py new file mode 100644 index 0000000..da95201 --- /dev/null +++ b/task_core.py @@ -0,0 +1,386 @@ +"""Root-owned task store and native systemd schedules; no third-party modules.""" +from contextlib import contextmanager +import datetime as dt +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import sqlite3 +import subprocess +import time +import uuid + +BASE=Path(os.environ.get('HARDWARE_TASK_DATA','/var/lib/hardware-taskd')) +LOGS=Path(os.environ.get('HARDWARE_TASK_LOGS','/var/log/hardware-taskd')) +UNITS=Path(os.environ.get('HARDWARE_TASK_UNITS','/etc/systemd/system')) +APP=Path(__file__).resolve().parent +TZ=dt.timezone(dt.timedelta(hours=8)) +WEEKDAYS=['Mon','Tue','Wed','Thu','Fri','Sat','Sun'] + +class TaskError(Exception): + def __init__(self,message,status=400): + self.status=status + super().__init__(message) + +def iso(value=None): + return dt.datetime.fromtimestamp(time.time() if value is None else value,TZ).isoformat(timespec='seconds') + +@contextmanager +def db(): + con=sqlite3.connect(BASE/'tasks.sqlite3',timeout=10) + con.row_factory=sqlite3.Row + try: + yield con;con.commit() + except Exception: + con.rollback();raise + finally: + con.close() + +def init_store(): + BASE.mkdir(parents=True,exist_ok=True,mode=0o700) + LOGS.mkdir(parents=True,exist_ok=True,mode=0o700) + (BASE/'scripts').mkdir(exist_ok=True,mode=0o700) + with db() as con: + con.execute('PRAGMA journal_mode=WAL') + con.executescript(''' + CREATE TABLE IF NOT EXISTS tasks(id TEXT PRIMARY KEY,name TEXT,definition TEXT, + enabled INTEGER,deleted INTEGER DEFAULT 0,created REAL,updated REAL,revision INTEGER); + CREATE TABLE IF NOT EXISTS runs(id TEXT PRIMARY KEY,task_id TEXT,task_name TEXT, + started REAL,finished REAL,state TEXT,exit_code INTEGER,trigger TEXT,boot TEXT, + script_hash TEXT,log_bytes INTEGER DEFAULT 0,truncated INTEGER DEFAULT 0,revision INTEGER,duration REAL, + log_purged INTEGER DEFAULT 0); + CREATE INDEX IF NOT EXISTS run_task ON runs(task_id,started DESC); + CREATE TABLE IF NOT EXISTS audit(id INTEGER PRIMARY KEY AUTOINCREMENT,at REAL, + action TEXT,task_id TEXT,actor TEXT,client TEXT,detail TEXT); + CREATE TABLE IF NOT EXISTS requests(task_id TEXT PRIMARY KEY,trigger TEXT,expires REAL); + ''') + +def atom_write(path,text,mode=0o600): + temporary=Path(str(path)+'.tmp') + temporary.write_text(text,encoding='utf-8');os.chmod(temporary,mode);os.replace(temporary,path) + +def validate_id(value): + if not isinstance(value,str) or not re.fullmatch('[0-9a-f]{32}',value): + raise TaskError('任务编号不正确。') + return value + +def execute(args,timeout=15,check=True): + try: + p=subprocess.run(args,capture_output=True,text=True,timeout=timeout,env={**os.environ,'LC_ALL':'C','TZ':'Asia/Shanghai'}) + except (OSError,subprocess.TimeoutExpired) as exc: + raise TaskError('系统调度操作未完成:'+str(exc),503) + if check and p.returncode: + raise TaskError('系统调度操作失败:'+(p.stderr or p.stdout)[-1200:],503) + return p + +def number(value,minimum,maximum,label): + try: + if isinstance(value,bool):raise ValueError() + n=int(str(value)) + except (ValueError,TypeError): + raise TaskError(label+'必须是整数。') + if not minimum<=n<=maximum: + raise TaskError(f'{label}必须在 {minimum} 至 {maximum} 之间。') + return n + +def clock(value): + if not isinstance(value,str) or not re.fullmatch(r'\d{2}:\d{2}',value): + raise TaskError('执行时间格式应为 03:00。') + h,m=map(int,value.split(':')) + if h>23 or m>59:raise TaskError('执行时间不正确。') + return value+':00' + +def cron_field(value,low,high,names=None): + value=value.upper() + if names: + value=re.sub('[A-Z]{3}',lambda m:str(names[m[0]]) if m[0] in names else m[0],value) + result=set() + for part in value.split(','): + pieces=part.split('/') + if len(pieces)>2:raise TaskError('Cron 的步长格式不正确。') + step=number(pieces[1],1,high-low+1,'Cron 步长') if len(pieces)==2 else 1 + base=pieces[0] + if base=='*':start,end=low,high + elif re.fullmatch(r'\d+-\d+',base):start,end=map(int,base.split('-')) + elif base.isdigit():start=int(base);end=high if len(pieces)==2 else start + else:raise TaskError('Cron 仅支持数字、*、范围、逗号和步长;不支持 L、W、# 或 ?。') + if not low<=start<=end<=high:raise TaskError('Cron 字段数值超出允许范围。') + result.update(range(start,end+1,step)) + return sorted(result) + +def cron_calendars(expression): + parts=expression.split() + if len(parts)!=5:raise TaskError('Cron 需要五段:分 时 日 月 周,例如 0 3 * * *。') + minute=cron_field(parts[0],0,59);hour=cron_field(parts[1],0,23) + day=cron_field(parts[2],1,31) + month=cron_field(parts[3],1,12,{x:i+1 for i,x in enumerate(['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC'])}) + dow=cron_field(parts[4],0,7,{'SUN':0,'MON':1,'TUE':2,'WED':3,'THU':4,'FRI':5,'SAT':6}) + join=lambda values:','.join(map(str,values)) + months='*' if parts[3]=='*' else join(month) + hours='*' if parts[1]=='*' else join(hour) + minutes='*' if parts[0]=='*' else join(minute) + dates='*' if parts[2]=='*' else join(day) + days=','.join(['Sun','Mon','Tue','Wed','Thu','Fri','Sat'][v] for v in sorted({v%7 for v in dow})) + timepart=f'{hours}:{minutes}:00 Asia/Shanghai' + if parts[4]=='*':return [f'*-{months}-{dates} {timepart}'] + if parts[2]=='*':return [f'{days} *-{months}-* {timepart}'] + # Traditional cron: restricted day-of-month and weekday are joined with OR. + return [f'*-{months}-{dates} {timepart}',f'{days} *-{months}-* {timepart}'] + +def schedule_definition(schedule): + if not isinstance(schedule,dict):raise TaskError('请设置执行周期。') + kind=schedule.get('kind','daily') + if kind=='daily': + value=clock(schedule.get('time','03:00')) + return [f'OnCalendar=*-*-* {value} Asia/Shanghai'],f'每天 {value[:5]}' + if kind=='weekly': + chosen=schedule.get('weekdays',[0]) + if not isinstance(chosen,list) or not chosen:raise TaskError('至少选择一个星期。') + selected=sorted({number(v,0,6,'星期') for v in chosen}) + value=clock(schedule.get('time','03:00')) + days=','.join(WEEKDAYS[v] for v in selected) + label='、'.join('周'+['一','二','三','四','五','六','日'][v] for v in selected) + return [f'OnCalendar={days} *-*-* {value} Asia/Shanghai'],f'每{label} {value[:5]}' + if kind=='monthly': + day=number(schedule.get('day',1),1,31,'每月日期');value=clock(schedule.get('time','03:00')) + return [f'OnCalendar=*-*-{day} {value} Asia/Shanghai'],f'每月 {day} 日 {value[:5]}'+('(当月没有该日则跳过)' if day>28 else '') + if kind in ('minutes','hours'): + every=number(schedule.get('every',1),1,1440 if kind=='minutes' else 168,'间隔') + seconds=every*(60 if kind=='minutes' else 3600) + return [f'OnActiveSec={seconds}s',f'OnUnitInactiveSec={seconds}s'],f'每 {every} '+('分钟' if kind=='minutes' else '小时')+'(上次执行结束后重新计时)' + if kind=='once': + try: + at=dt.datetime.fromisoformat(schedule.get('at','')) + if at.tzinfo is None:at=at.replace(tzinfo=TZ) + at=at.replace(second=0,microsecond=0) + except (ValueError,TypeError):raise TaskError('请填写正确的指定日期与时间。') + if at.timestamp()150:raise TaskError('Cron 表达式过长。') + return ['OnCalendar='+x for x in cron_calendars(expression)],'Cron:'+expression + raise TaskError('不支持的执行周期。') + +def validate_definition(payload,check_paths=True): + name=str(payload.get('name','')).strip() + if not 1<=len(name)<=80 or any(ord(c)<32 for c in name):raise TaskError('任务名称需要 1 至 80 个字符,不能包含换行。') + mode=payload.get('mode','command');interpreter=payload.get('interpreter','bash') + if mode not in ('command','script','file'):raise TaskError('请选择命令、脚本内容或服务器脚本文件。') + if interpreter not in ('bash','python3','auto'):raise TaskError('脚本语言不正确。') + body=str(payload.get('body','')).replace('\r\n','\n').replace('\r','\n').lstrip('\ufeff').strip() + path=str(payload.get('path','')).strip() + arguments=str(payload.get('arguments','')).strip() + if mode in ('command','script') and not body:raise TaskError('请输入要执行的命令或脚本。') + if len(body.encode())>65536:raise TaskError('脚本内容不能超过 64 KiB。') + if mode=='script' and interpreter=='auto':raise TaskError('脚本内容请选择 Bash 或 Python 3。') + cwd=str(payload.get('cwd','/root')).strip() or '/root' + if not PurePosixPath(cwd).is_absolute() or '\x00' in cwd:raise TaskError('工作目录必须是绝对路径。') + if check_paths and not Path(cwd).is_dir():raise TaskError('工作目录不存在。') + if mode=='file': + if not PurePosixPath(path).is_absolute() or '\x00' in path:raise TaskError('脚本文件需要填写服务器上的绝对路径。') + if check_paths and not Path(path).is_file():raise TaskError('服务器上找不到这个脚本文件。') + if check_paths and interpreter=='auto' and not os.access(path,os.X_OK):raise TaskError('该文件没有执行权限,请选择 Bash/Python 3 解释器,或先设置文件执行权限。') + try:shlex.split(arguments) + except ValueError:raise TaskError('脚本参数的引号不匹配。') + lines,label=schedule_definition(payload.get('schedule',{})) + notes=str(payload.get('notes','')).strip()[:400] + timeout=number(payload.get('timeout',1800),10,86400,'最长执行秒数') + if not isinstance(payload.get('enabled',False),bool):raise TaskError('启用状态必须为是或否。') + dangerous=bool(re.search(r'\b(reboot|shutdown|poweroff|halt)\b|\binit\s+[06]\b',body+' '+path,re.I)) + return {'name':name,'mode':mode,'interpreter':interpreter,'body':body,'path':path,'arguments':arguments, + 'cwd':cwd,'schedule':payload.get('schedule',{}),'schedule_lines':lines,'schedule_label':label, + 'timeout':timeout,'notes':notes,'enabled':bool(payload.get('enabled',False)), + 'dangerous':dangerous,'template':str(payload.get('template','custom'))[:40]} + +def calendar_preview(definition): + result=[] + for line in definition['schedule_lines']: + if line.startswith('OnCalendar='): + p=execute(['systemd-analyze','calendar','--iterations=1',line.split('=',1)[1]],timeout=5,check=False) + if p.returncode:raise TaskError('执行周期无效:'+(p.stderr or p.stdout)[-600:]) + result.append(p.stdout.strip()) + if result and all(re.search(r'Next elapse:\s*n/a',output) for output in result): + raise TaskError('这个周期没有可以执行的将来时间,请检查日期或 Cron 表达式。') + return result + +def get_task(task_id,include_deleted=False): + validate_id(task_id) + with db() as con: + row=con.execute('SELECT * FROM tasks WHERE id=?'+('' if include_deleted else ' AND deleted=0'),(task_id,)).fetchone() + if not row:raise TaskError('找不到这个任务。',404) + item=dict(row);item['definition']=json.loads(item['definition']);return item + +def unit_name(task_id):return 'hwjob-'+validate_id(task_id) + +def running(task_id): + p=execute(['systemctl','is-active',unit_name(task_id)+'.service'],check=False,timeout=5) + return p.stdout.strip() in ('active','activating','deactivating') + +def render_units(task_id,definition): + name=unit_name(task_id) + service=f'''[Unit] +Description=Dashboard scheduled task {task_id} +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +ExecStart=/usr/bin/python3 {APP}/task_exec.py {task_id} +RuntimeMaxSec={definition['timeout']+10} +TimeoutStopSec=15 +KillMode=mixed +Restart=no +UMask=0077 +Environment=TZ=Asia/Shanghai +''' + timer=f'''[Unit] +Description=Dashboard task schedule {task_id} + +[Timer] +{chr(10).join(definition['schedule_lines'])} +AccuracySec=1s +Persistent=false +Unit={name}.service + +[Install] +WantedBy=timers.target +''' + return service,timer + +def install_units(task_id,definition): + service,timer=render_units(task_id,definition) + name=unit_name(task_id) + atom_write(UNITS/(name+'.service'),service,0o644) + atom_write(UNITS/(name+'.timer'),timer,0o644) + execute(['systemctl','daemon-reload']) + +def audit(action,task_id='',actor='admin',client='',detail=''): + with db() as con: + con.execute('INSERT INTO audit(at,action,task_id,actor,client,detail) VALUES(?,?,?,?,?,?)', + (time.time(),action,task_id,actor,client,detail[:2000])) + +def save_task(payload,client=''): + task_id=payload.get('id') + old=get_task(task_id) if task_id else None + if old and running(task_id):raise TaskError('任务正在执行,请等待结束后再修改。',409) + definition=validate_definition(payload);calendar_preview(definition) + if not old: + with db() as con: + if con.execute('SELECT COUNT(*) FROM tasks WHERE deleted=0').fetchone()[0]>=100:raise TaskError('最多管理 100 个任务。') + task_id=uuid.uuid4().hex + if old: + execute(['systemctl','disable','--now',unit_name(task_id)+'.timer'],check=False) + if running(task_id): + if old['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer'],check=False) + raise TaskError('任务刚刚开始执行,请等待完成后再修改。',409) + now=time.time();revision=old['revision']+1 if old else 1 + try: + install_units(task_id,definition) + with db() as con: + con.execute('''INSERT INTO tasks VALUES(?,?,?,?,0,?,?,?) ON CONFLICT(id) DO UPDATE SET + name=excluded.name,definition=excluded.definition,enabled=excluded.enabled, + updated=excluded.updated,revision=excluded.revision''', + (task_id,definition['name'],json.dumps(definition,ensure_ascii=False),int(definition['enabled']), + old['created'] if old else now,now,revision)) + if definition['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer']) + except Exception: + if old: + install_units(task_id,old['definition']) + with db() as con: + con.execute('UPDATE tasks SET name=?,definition=?,enabled=?,revision=? WHERE id=?', + (old['name'],json.dumps(old['definition'],ensure_ascii=False),old['enabled'],old['revision'],task_id)) + if old['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer'],check=False) + else: + execute(['systemctl','disable','--now',unit_name(task_id)+'.timer'],check=False) + with db() as con:con.execute('DELETE FROM tasks WHERE id=?',(task_id,)) + for suffix in ('.service','.timer'):(UNITS/(unit_name(task_id)+suffix)).unlink(missing_ok=True) + execute(['systemctl','daemon-reload'],check=False) + raise + audit('update' if old else 'create',task_id,client=client,detail='revision '+str(revision)) + return {'id':task_id,'message':'任务已保存。'+('已启用,将按计划执行。' if definition['enabled'] else '当前停用,不会自动执行。')} + +def set_enabled(task_id,enabled,client=''): + task=get_task(task_id) + if enabled: + schedule_definition(task['definition']['schedule']) + calendar_preview(task['definition']) + execute(['systemctl','enable' if enabled else 'disable','--now',unit_name(task_id)+'.timer']) + with db() as con: + definition=task['definition'];definition['enabled']=bool(enabled) + con.execute('UPDATE tasks SET enabled=?,definition=?,updated=? WHERE id=?', + (int(bool(enabled)),json.dumps(definition,ensure_ascii=False),time.time(),task_id)) + audit('enable' if enabled else 'disable',task_id,client=client) + return {'message':'已启用。' if enabled else '已停用后续调度;已经开始的执行不会被强制中断。'} + +def delete_task(task_id,client=''): + get_task(task_id) + if running(task_id):raise TaskError('任务正在执行,请先等待完成或停止本次执行。',409) + execute(['systemctl','disable','--now',unit_name(task_id)+'.timer']) + for suffix in ('.service','.timer'): + (UNITS/(unit_name(task_id)+suffix)).unlink(missing_ok=True) + execute(['systemctl','daemon-reload']) + with db() as con:con.execute('UPDATE tasks SET deleted=1,enabled=0,updated=? WHERE id=?',(time.time(),task_id)) + audit('delete',task_id,client=client) + return {'message':'任务已删除,已有执行记录仍保留在服务器。'} + +def run_now(task_id,client=''): + task=get_task(task_id) + if running(task_id):raise TaskError('这个任务正在执行,不会重复启动。',409) + with db() as con: + con.execute('INSERT OR REPLACE INTO requests VALUES(?,?,?)',(task_id,'manual',time.time()+30)) + execute(['systemctl','reset-failed',unit_name(task_id)+'.service'],check=False) + execute(['systemctl','start','--no-block',unit_name(task_id)+'.service']) + audit('run_now',task_id,client=client,detail='revision '+str(task['revision'])) + return {'message':'已提交执行,请在执行日志中查看结果。'} + +def stop_task(task_id,client=''): + get_task(task_id) + execute(['systemctl','stop',unit_name(task_id)+'.service']) + audit('stop_run',task_id,client=client) + return {'message':'已请求停止本次执行。'} + +def list_tasks(): + p=execute(['systemctl','list-timers','--all','--output=json','--no-pager','hwjob-*.timer'],check=False) + try:timers={x['unit']:x for x in json.loads(p.stdout or '[]')} + except (ValueError,TypeError):timers={} + p=execute(['systemctl','list-units','--all','--type=service','--output=json','--no-pager','hwjob-*.service'],check=False) + try:active={x['unit']:x for x in json.loads(p.stdout or '[]')} + except (ValueError,TypeError):active={} + with db() as con: + tasks=con.execute('SELECT * FROM tasks WHERE deleted=0 ORDER BY created DESC').fetchall() + result=[] + for row in tasks: + item=dict(row);definition=json.loads(item.pop('definition')) + timer=timers.get(unit_name(item['id'])+'.timer',{}) + next_us=timer.get('next') + item['next_run']=next_us/1e6 if isinstance(next_us,(int,float)) and 0131072} diff --git a/task_exec.py b/task_exec.py new file mode 100644 index 0000000..ba298e5 --- /dev/null +++ b/task_exec.py @@ -0,0 +1,104 @@ +#!/usr/bin/python3 +"""Run one approved, root-owned task definition; record bounded output and result.""" +import fcntl +import hashlib +import json +import os +import selectors +import shlex +import signal +import subprocess +import sys +import time +import uuid +import task_core as core + +LIMIT=512*1024 + +def main(task_id): + task=core.get_task(task_id) + definition=task['definition'] + lock=(core.BASE/(task_id+'.lock')).open('a') + try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError:return 0 + run_id=uuid.uuid4().hex;started=time.time();start_mono=time.monotonic() + boot=core.read_boot() if hasattr(core,'read_boot') else open('/proc/sys/kernel/random/boot_id').read().strip() + with core.db() as con: + requested=con.execute('SELECT * FROM requests WHERE task_id=?',(task_id,)).fetchone() + trigger='manual' if requested and requested['expires']>=started else 'scheduled' + con.execute('DELETE FROM requests WHERE task_id=?',(task_id,)) + body=definition['body'] + fingerprint=hashlib.sha256((body if definition['mode']!='file' else definition['path']+' '+definition['arguments']).encode()).hexdigest() + con.execute('INSERT INTO runs(id,task_id,task_name,started,state,trigger,boot,script_hash,revision) VALUES(?,?,?,?,?,?,?,?,?)', + (run_id,task_id,task['name'],started,'running',trigger,boot,fingerprint,task['revision'])) + cancelled=[False] + def stop(*_):cancelled[0]=True + signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop) + state='failed';exit_code=None;proc=None;written=0;truncated=False + log_path=core.LOGS/(run_id+'.log') + with log_path.open('wb',buffering=0) as output: + os.chmod(log_path,0o600) + def write(data): + nonlocal written,truncated + if isinstance(data,str):data=data.encode('utf-8') + available=max(0,LIMIT-written) + if available: + piece=data[:available];output.write(piece);written+=len(piece) + if len(data)>available:truncated=True + write(f'开始:{core.iso(started)}\n任务:{task["name"]}\n触发:{"手动" if trigger=="manual" else "定时"}\n执行身份:root\n修订:{task["revision"]}\n'+'─'*32+'\n') + try: + if definition['mode']=='command': + argv=['/bin/bash','-e','-o','pipefail','-c',body] + elif definition['mode']=='script': + script=core.BASE/'scripts'/(run_id+('.py' if definition['interpreter']=='python3' else '.sh')) + core.atom_write(script,body+'\n') + argv=['/usr/bin/python3',str(script)] if definition['interpreter']=='python3' else ['/bin/bash','-e','-o','pipefail',str(script)] + else: + argv=[definition['path'],*shlex.split(definition['arguments'])] + if definition['interpreter']=='bash':argv=['/bin/bash','-e','-o','pipefail',*argv] + elif definition['interpreter']=='python3':argv=['/usr/bin/python3',*argv] + env={'HOME':'/root','USER':'root','LOGNAME':'root','LANG':'C.UTF-8','TZ':'Asia/Shanghai', + 'PATH':'/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin', + 'HARDWARE_TASK_ID':task_id,'HARDWARE_TASK_RUN_ID':run_id} + proc=subprocess.Popen(argv,cwd=definition['cwd'],env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,start_new_session=True) + selector=selectors.DefaultSelector();selector.register(proc.stdout,selectors.EVENT_READ) + deadline=start_mono+definition['timeout'];exited_at=None + while selector.get_map() or proc.poll() is None: + if cancelled[0] or time.monotonic()>=deadline: + state='cancelled' if cancelled[0] else 'timeout' + try:os.killpg(proc.pid,signal.SIGTERM) + except ProcessLookupError:pass + try:proc.wait(timeout=3) + except subprocess.TimeoutExpired: + try:os.killpg(proc.pid,signal.SIGKILL) + except ProcessLookupError:pass + proc.wait(timeout=3) + break + for key,_ in selector.select(.25): + chunk=os.read(key.fileobj.fileno(),32768) + if chunk:write(chunk) + else:selector.unregister(key.fileobj) + if proc.poll() is not None: + exited_at=exited_at or time.monotonic() + if time.monotonic()-exited_at>1:break + selector.close();proc.stdout.close() + exit_code=proc.wait(timeout=3) + if state not in ('cancelled','timeout'):state='success' if exit_code==0 else 'failed' + except Exception as exc: + write('\n执行错误:'+str(exc)+'\n');state='failed' + if proc and proc.poll() is None: + try:os.killpg(proc.pid,signal.SIGKILL);proc.wait(timeout=3) + except (ProcessLookupError,subprocess.TimeoutExpired):pass + elapsed=time.monotonic()-start_mono + tail=f'\n'+('─'*32)+f'\n结束:{core.iso()}\n结果:{state}\n退出码:{exit_code}\n耗时:{elapsed:.2f} 秒\n' + if truncated:output.write('\n[输出超过 512 KiB,后续内容已省略。]\n'.encode()) + output.write(tail.encode()) + os.fsync(output.fileno()) + with core.db() as con: + con.execute('UPDATE runs SET finished=?,state=?,exit_code=?,log_bytes=?,truncated=?,duration=? WHERE id=?', + (time.time(),state,exit_code,log_path.stat().st_size,int(truncated),elapsed,run_id)) + lock.close() + return 0 if state=='success' else 1 + +if __name__=='__main__': + raise SystemExit(main(core.validate_id(sys.argv[1]))) diff --git a/taskd.py b/taskd.py new file mode 100644 index 0000000..783052f --- /dev/null +++ b/taskd.py @@ -0,0 +1,193 @@ +#!/usr/bin/python3 +"""Authenticated local task broker. Only the web service UID may connect to its socket.""" +import argparse +import getpass +import hashlib +import hmac +import json +import os +from pathlib import Path +try:import pwd +except ImportError:pwd=None +import secrets +import socket +import socketserver +import struct +import re +import datetime as dt +import threading +import time +import task_core as core + +ADMIN=Path('/etc/hardware-task-admin.json') +SOCKET=Path('/run/hardware-taskd/control.sock') +SESSIONS={};ATTEMPTS={};GUARD=threading.RLock();MUTATE=threading.RLock() +ALLOWED_UID=None + +def password_record(password): + salt=secrets.token_hex(16) + return {'username':'admin','salt':salt,'hash':hashlib.pbkdf2_hmac('sha256',password.encode(),bytes.fromhex(salt),240000).hex()} + +def verify(password): + try: + record=json.loads(ADMIN.read_text()) + hashed=hashlib.pbkdf2_hmac('sha256',password.encode(),bytes.fromhex(record['salt']),240000).hex() + return hmac.compare_digest(record['hash'],hashed) + except (OSError,ValueError,KeyError):return False + +def create_admin(): + if ADMIN.exists():return + password=secrets.token_urlsafe(18) + core.atom_write(ADMIN,json.dumps(password_record(password))) + core.atom_write(Path('/root/hardware-task-admin-login.txt'), + '定时任务管理员登录信息\n\n网页:http://192.168.1.37:8090/#tasks\n账号:admin\n初始密码:'+password+ + '\n\n此账号可以管理以 root 身份执行的服务器任务,请妥善保存。登录后可修改管理密码。\nSSH root 密码未被修改,也没有作为网页管理密码保存。\n') + print('初始登录信息已保存到 /root/hardware-task-admin-login.txt(仅 root 可读)。') + +def session(token,csrf=None,write=False): + with GUARD: + record=SESSIONS.get(token) + if not record or record['expires']=6:raise core.TaskError('登录失败次数较多,请五分钟后重试。',429) + if payload.get('username')!='admin' or not verify(str(payload.get('password',''))): + ATTEMPTS[client]=recent+[now] + core.audit('login_failed',client=client) + raise core.TaskError('账号或密码不正确。',401) + ATTEMPTS.pop(client,None) + if len(SESSIONS)>200: + for key in list(SESSIONS): + if SESSIONS[key]['expires']131072:raise core.TaskError('请求内容过大。',413) + request=json.loads(raw) + response={'ok':True,'data':dispatch(request)} + except core.TaskError as exc:response={'ok':False,'error':str(exc),'status':exc.status} + except Exception as exc: + print('task broker error:',type(exc).__name__,str(exc),flush=True) + response={'ok':False,'error':'任务操作未完成,请查看任务管理服务日志。','status':500} + try:self.wfile.write(json.dumps(response,ensure_ascii=False).encode()+b'\n') + except (BrokenPipeError,ConnectionResetError):pass + +class Server(getattr(socketserver,'ThreadingUnixStreamServer',socketserver.ThreadingTCPServer)): + daemon_threads=True + +def maintenance(): + while True: + try: + with core.db() as con: + rows=con.execute("SELECT * FROM runs WHERE state='running'").fetchall() + boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip() + for row in rows: + if row['boot']!=boot or (time.time()-row['started']>30 and not core.running(row['task_id'])): + with core.db() as con: + con.execute("UPDATE runs SET state='interrupted',finished=? WHERE id=? AND state='running'",(time.time(),row['id'])) + with (core.LOGS/(row['id']+'.log')).open('ab') as f: + f.write(('\n[执行被系统重启或进程中断,没有取得完整退出结果。]\n').encode()) + cutoff=time.time()-90*86400 + with core.db() as con: + old=con.execute('SELECT id FROM runs WHERE started