Initial hardware dashboard and scheduled task service
This commit is contained in:
commit
cedae7ab92
12
.gitattributes
vendored
Normal file
12
.gitattributes
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
* text=auto
|
||||
*.py text eol=lf
|
||||
*.js text eol=lf
|
||||
*.css text eol=lf
|
||||
*.html text eol=lf
|
||||
*.md text eol=lf
|
||||
*.json text eol=lf
|
||||
*.service text eol=lf
|
||||
*.timer text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.svg text eol=lf
|
||||
scripts/hwlog text eol=lf
|
||||
24
.gitignore
vendored
Normal file
24
.gitignore
vendored
Normal file
@ -0,0 +1,24 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Runtime state and local deployment secrets
|
||||
*.sqlite3
|
||||
*.sqlite3-shm
|
||||
*.sqlite3-wal
|
||||
*.log
|
||||
*.pid
|
||||
config/hardware-watch.json
|
||||
config/hardware-task-admin.json
|
||||
hardware-task-admin-login*.txt
|
||||
|
||||
# Local archives and editor state
|
||||
*.tar.gz
|
||||
.idea/
|
||||
.vscode/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
58
README.md
Normal file
58
README.md
Normal file
@ -0,0 +1,58 @@
|
||||
# Hardware Dashboard
|
||||
|
||||
这是运行在 192.168.1.37 上的局域网设备监控与维护面板。页面提供 CPU、内存、双 P100 显卡、磁盘 SMART、性能历史曲线、硬件问题记录和定时任务管理。后端只使用 Python 标准库；前端使用原生 HTML、CSS 和 JavaScript，不依赖外部 CDN。
|
||||
|
||||
当前访问地址是 http://192.168.1.37:8090。定时任务可以执行服务器命令，必须使用独立的管理员账号登录。该账号与 SSH root 密码相互独立。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
.
|
||||
├── server.py # 监控网页、实时指标和历史数据 API
|
||||
├── task_core.py # 定时任务定义、校验和 systemd 管理
|
||||
├── taskd.py # root 侧任务管理服务
|
||||
├── task_exec.py # 单个任务的执行器和输出记录
|
||||
├── static/ # 前端页面、样式和脚本
|
||||
├── scripts/hwlog # 硬件事件采集与命令行查看工具
|
||||
├── config/
|
||||
| └── hardware-watch.example.json
|
||||
├── deploy/systemd/ # dashboard、taskd 和硬件采集服务文件
|
||||
├── deploy/journald/ # 原始内核日志持久化配置
|
||||
├── docs/DEPLOYMENT.zh-CN.md # 服务器部署与技术说明
|
||||
└── tests/ # 自动化测试
|
||||
```
|
||||
|
||||
## 本地检查
|
||||
|
||||
项目只需要 Python 3.12 或更新版本，没有第三方 Python 依赖。
|
||||
|
||||
```
|
||||
python -m unittest discover -s tests -p "test_*.py" -v
|
||||
python -m py_compile server.py task_core.py taskd.py task_exec.py scripts/hwlog
|
||||
```
|
||||
|
||||
如需本地启动网页：
|
||||
|
||||
```
|
||||
python server.py --host 127.0.0.1 --port 8090
|
||||
```
|
||||
|
||||
服务器上的完整部署方式、文件路径、权限、任务管理和维护命令见 docs/DEPLOYMENT.zh-CN.md。
|
||||
|
||||
## 服务器配置
|
||||
|
||||
仓库中不保存真实硬件基准配置和任务管理凭据。部署前复制示例配置：
|
||||
|
||||
```
|
||||
sudo cp config/hardware-watch.example.json /etc/hardware-watch.json
|
||||
sudo nano /etc/hardware-watch.json
|
||||
```
|
||||
|
||||
把 expected_gpu_count、GPU UUID 和磁盘名改成当前服务器的实际值。修改后重启监控采集服务。定时任务管理服务首次初始化会生成独立管理员密码文件；不要把生成的密码、任务数据库、日志或系统配置哈希提交到仓库。
|
||||
|
||||
## 安全边界
|
||||
|
||||
- 监控页面本身是只读的，可查看状态、曲线和日志。
|
||||
- 定时任务能以 root 身份执行命令，新增、编辑、启用和立即执行均需要任务管理员登录。
|
||||
- 不要把这个服务直接暴露到公网。当前设计只适合可信局域网。
|
||||
- 远程访问应通过可信 VPN 或带 HTTPS 与额外认证的反向代理，而不是直接端口映射。
|
||||
11
config/hardware-watch.example.json
Normal file
11
config/hardware-watch.example.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"expected_gpu_count": 2,
|
||||
"expected_gpu_uuids": [
|
||||
"GPU-00000000-0000-0000-0000-000000000001",
|
||||
"GPU-00000000-0000-0000-0000-000000000002"
|
||||
],
|
||||
"expected_disks": ["nvme0n1"],
|
||||
"gpu_warn_c": 85,
|
||||
"history_since": "7 days ago",
|
||||
"retention_days": 90
|
||||
}
|
||||
44
deploy/install.sh
Executable file
44
deploy/install.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
INSTALL_ROOT="${INSTALL_ROOT:-/opt/hardware-dashboard}"
|
||||
CONFIG_FILE="${CONFIG_FILE:-/etc/hardware-watch.json}"
|
||||
SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}"
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "必须使用 root 运行部署脚本。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${CONFIG_FILE}" ]]; then
|
||||
echo "缺少 ${CONFIG_FILE}。请先复制 config/hardware-watch.example.json 并填入当前服务器实际值。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
getent group hardware-watch >/dev/null || groupadd --system hardware-watch
|
||||
getent passwd hardware-dashboard >/dev/null || useradd --system --gid hardware-watch --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin hardware-dashboard
|
||||
|
||||
install -d -m 0755 "${INSTALL_ROOT}/static"
|
||||
install -d -m 0750 /var/lib/hardware-dashboard /var/lib/hardware-watch /var/log/hardware-watch
|
||||
install -d -m 0755 /etc/systemd/journald.conf.d
|
||||
|
||||
install -m 0644 server.py task_core.py taskd.py task_exec.py "${INSTALL_ROOT}/"
|
||||
install -m 0644 static/index.html static/app.js static/tasks.js static/style.css static/favicon.svg "${INSTALL_ROOT}/static/"
|
||||
install -m 0755 scripts/hwlog /usr/local/sbin/hwlog
|
||||
install -m 0640 config/hardware-watch.json "${CONFIG_FILE}"
|
||||
install -m 0644 deploy/systemd/*.service deploy/systemd/*.timer "${SYSTEMD_DIR}/"
|
||||
install -m 0644 deploy/journald/60-hardware-watch.conf /etc/systemd/journald.conf.d/60-hardware-watch.conf
|
||||
|
||||
setfacl -m u:hardware-dashboard:rx /var/lib/hardware-watch /var/log/hardware-watch
|
||||
setfacl -d -m u:hardware-dashboard:rx /var/lib/hardware-watch /var/log/hardware-watch
|
||||
find /var/lib/hardware-watch /var/log/hardware-watch -maxdepth 1 -type f -exec setfacl -m u:hardware-dashboard:r {} +
|
||||
setfacl -m u:hardware-dashboard:r "${CONFIG_FILE}"
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now hardware-watch.timer
|
||||
systemctl restart hardware-watch.service
|
||||
systemctl enable --now hardware-taskd.service
|
||||
systemctl restart hardware-dashboard.service
|
||||
|
||||
echo "部署完成。首次初始化定时任务管理员："
|
||||
echo "python3 ${INSTALL_ROOT}/taskd.py init-admin"
|
||||
5
deploy/journald/60-hardware-watch.conf
Normal file
5
deploy/journald/60-hardware-watch.conf
Normal file
@ -0,0 +1,5 @@
|
||||
[Journal]
|
||||
Storage=persistent
|
||||
SystemMaxUse=1G
|
||||
SystemKeepFree=2G
|
||||
MaxRetentionSec=90day
|
||||
29
deploy/systemd/hardware-dashboard.service
Normal file
29
deploy/systemd/hardware-dashboard.service
Normal file
@ -0,0 +1,29 @@
|
||||
[Unit]
|
||||
Description=Hardware monitoring dashboard on LAN port 8090
|
||||
After=network.target hardware-watch.timer
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=hardware-dashboard
|
||||
Group=hardware-watch
|
||||
WorkingDirectory=/opt/hardware-dashboard
|
||||
ExecStart=/usr/bin/python3 /opt/hardware-dashboard/server.py --port 8090
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
Environment=TZ=Asia/Shanghai
|
||||
StateDirectory=hardware-dashboard
|
||||
StateDirectoryMode=0750
|
||||
UMask=0027
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectControlGroups=true
|
||||
ReadWritePaths=/var/lib/hardware-dashboard
|
||||
MemoryMax=384M
|
||||
Nice=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
23
deploy/systemd/hardware-taskd.service
Normal file
23
deploy/systemd/hardware-taskd.service
Normal file
@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Authenticated local scheduled-task broker
|
||||
After=local-fs.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
Group=hardware-watch
|
||||
ExecStart=/usr/bin/python3 /opt/hardware-dashboard/taskd.py serve
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
RuntimeDirectory=hardware-taskd
|
||||
RuntimeDirectoryMode=0750
|
||||
StateDirectory=hardware-taskd
|
||||
StateDirectoryMode=0700
|
||||
LogsDirectory=hardware-taskd
|
||||
LogsDirectoryMode=0700
|
||||
UMask=0077
|
||||
Environment=TZ=Asia/Shanghai
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
23
deploy/systemd/hardware-watch.service
Normal file
23
deploy/systemd/hardware-watch.service
Normal file
@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Hardware event journal and GPU / disk health sampling
|
||||
After=systemd-journald.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/hwlog collect
|
||||
TimeoutStartSec=55
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
UMask=0027
|
||||
StateDirectory=hardware-watch
|
||||
StateDirectoryMode=0750
|
||||
LogsDirectory=hardware-watch
|
||||
LogsDirectoryMode=0750
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
NoNewPrivileges=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectControlGroups=true
|
||||
MemoryMax=384M
|
||||
ReadWritePaths=/var/lib/hardware-watch /var/log/hardware-watch
|
||||
11
deploy/systemd/hardware-watch.timer
Normal file
11
deploy/systemd/hardware-watch.timer
Normal file
@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Check hardware health every minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=45s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
Unit=hardware-watch.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
263
docs/DEPLOYMENT.zh-CN.md
Normal file
263
docs/DEPLOYMENT.zh-CN.md
Normal file
@ -0,0 +1,263 @@
|
||||
# 192.168.1.37 监控与定时任务技术说明
|
||||
|
||||
版本：v1.2，2026 年 11 月 4 日。
|
||||
|
||||
## 1. 访问与登录
|
||||
|
||||
设备监控页面：**[http://192.168.1.37:8090](http://192.168.1.37:8090)**
|
||||
|
||||
定时任务入口在“问题与维护”下方，也可通过左侧“定时任务”菜单定位。手机底部有同名入口。
|
||||
|
||||
硬件监控仍可直接查看。定时任务能够实际执行服务器操作，因此使用独立的管理员账号登录：
|
||||
|
||||
- 账号：`admin`。
|
||||
- 初始密码：见交付的 `定时任务管理员登录信息.txt`，本说明不包含密码。
|
||||
- 登录最长有效 8 小时；修改密码后所有任务管理登录均失效。
|
||||
- 此密码与 SSH root 密码相互独立，本次没有修改 SSH 密码。
|
||||
|
||||
页面只开放在原有局域网访问范围内。当前使用局域网 HTTP，适用于可信局域网；不应将这个具备 root 任务执行能力的入口直接映射到公网。将来如需远程公开访问，应先配置 HTTPS 和适当的访问控制。
|
||||
|
||||
## 2. 本次新增功能
|
||||
|
||||
“定时任务”支持：
|
||||
|
||||
- 新增、编辑、启用、停用和删除任务。
|
||||
- 立即执行一次；这不会自动启用任务的定时计划。
|
||||
- 停止本次执行；与停用后续调度分开处理。
|
||||
- 查看下次计划时间、最近执行结果和是否正在执行。
|
||||
- 查看单任务执行记录，以及包含已删除任务的全局执行记录。
|
||||
- 查看开始／结束时间、耗时、退出码、标准输出和错误输出。
|
||||
- 管理员登录、退出及修改密码。
|
||||
|
||||
执行方式有三种：
|
||||
|
||||
| 执行方式 | 如何填写 | 说明 |
|
||||
|---|---|---|
|
||||
| 直接输入命令 | 例如 `/usr/bin/nvidia-smi` | 使用 Bash 执行，支持管道和多行命令。 |
|
||||
| 粘贴脚本内容 | 选择 Bash 或 Python 3，粘贴代码 | 可以导入本地 `.sh`、`.py` 或文本脚本到编辑框；保存后才会传给服务器。 |
|
||||
| 服务器上的脚本文件 | 填写绝对路径和可选参数 | 可选择 Bash、Python 3 或按文件自身方式执行；最后一种要求文件具有执行权限。 |
|
||||
|
||||
执行身份固定为 **root**，因为重启服务器和维护证书通常需要管理员权限。它不是隔离的脚本沙箱：应只运行自己理解并信任的命令。
|
||||
|
||||
工作目录默认 `/root`，使用非交互执行环境，不继承某个 SSH 窗口的虚拟环境。PATH 包含常用系统目录和 `/snap/bin`；如需 Conda、venv 或自定义环境变量，应在脚本内明确设置。
|
||||
|
||||
## 3. 第一次添加任务
|
||||
|
||||
1. 刷新页面，打开“定时任务”，点击“登录管理”。
|
||||
2. 使用交付文件中的 `admin` 与初始密码登录。
|
||||
3. 点击“新增任务”，填写任务名称。
|
||||
4. 选择执行方式，输入命令、脚本或服务器脚本路径。
|
||||
5. 设置执行周期，点击“检查执行周期”核对下次时间。
|
||||
6. 保持“保存后启用”不勾选，先保存为停用状态。
|
||||
7. 如适合测试，点击“执行一次”，确认执行内容，查看执行日志。
|
||||
8. 核对结果后，再启用计划。
|
||||
|
||||
保存任务本身不会立即执行脚本。新任务默认停用。退出登录、关闭网页或重启网页服务不会停止已经启用的系统计划。
|
||||
|
||||
### 常用模板
|
||||
|
||||
新增窗口提供：
|
||||
|
||||
- **显卡状态检查**：读取 `nvidia-smi`，结果保存在执行日志中。
|
||||
- **服务器定时重启**：示例命令为 `systemctl reboot`，默认周期是每周日 03:00，默认停用。启用或立即执行前都会提示可能中断计算任务。
|
||||
- **SSL 证书续签示例**：检查已有的 Certbot／acme.sh 工具并调用其续签入口。它不会安装工具、申请新域名或替用户配置证书。先确认现有续签配置，以及是否已经有其他自动续签任务；Web 服务的重新加载应按实际服务配置。
|
||||
|
||||
本次部署没有创建或启用实际的服务器重启任务和证书续签任务，也没有执行这两类操作。
|
||||
|
||||
## 4. 执行周期
|
||||
|
||||
所有日历时间按 **Asia/Shanghai，北京时间 UTC+8** 处理，页面显示服务器当前时间。
|
||||
|
||||
| 周期 | 规则 |
|
||||
|---|---|
|
||||
| 每天 | 指定时、分。 |
|
||||
| 每周 | 可选择一个或多个星期及时间。 |
|
||||
| 每月 | 选择 1 至 31 日和时间；当月不存在该日期时跳过。 |
|
||||
| 每隔若干分钟／小时 | 启用后先等待指定间隔；每次执行结束后重新计时，避免长任务造成重叠。 |
|
||||
| 指定日期，仅执行一次 | 选择未来的日期和时间，精确到分钟；至少晚于保存时刻 30 秒。 |
|
||||
| 自定义 Cron | 标准五段：分、时、日、月、周。支持 `*`、逗号、范围、步长及常见英文月／星期名。 |
|
||||
|
||||
Cron 示例：
|
||||
|
||||
```text
|
||||
0 3 * * * 每天 03:00
|
||||
0 3 * * 0 每周日 03:00
|
||||
0 3 1 * * 每月 1 日 03:00
|
||||
*/30 * * * * 每小时的 00、30 分
|
||||
```
|
||||
|
||||
不支持 `L`、`W`、`#`、`?` 等扩展语法，也不接受带秒的六段 Cron。日和星期同时指定时，按传统 Cron 的“任一满足”语义处理。
|
||||
|
||||
Cron 是日历触发，和“上次结束后再间隔 N 分钟”不同。例如长任务执行 10 分钟，再间隔 5 分钟，下一次大约在本次开始 15 分钟后。
|
||||
|
||||
### 开机、错过与长任务
|
||||
|
||||
- 日历任务设置 `Persistent=false`，服务器关机期间错过的执行时间不补跑，避免开机后补执行重启类任务。
|
||||
- 同一个任务由固定 systemd 服务承载，并有文件锁，不会同时重叠执行两份。
|
||||
- 一个任务执行超时后会被停止。默认最长 1800 秒，可在高级设置中改为 10 至 86400 秒。
|
||||
- 关闭网页只会关闭查看界面，不会停止系统定时器。
|
||||
- 本机历史上出现过时钟回拨。启用重启等任务前，应核对页面显示的服务器时间；定时调度依赖服务器时钟和系统校时状态。
|
||||
|
||||
部署验证时，服务器时区为 `Asia/Shanghai`，系统报告 `NTPSynchronized=yes`。
|
||||
|
||||
## 5. 看懂执行结果
|
||||
|
||||
| 显示 | 意义 |
|
||||
|---|---|
|
||||
| 尚未执行 | 没有记录到执行。 |
|
||||
| 执行中 | 本次进程仍在运行。 |
|
||||
| 执行成功 | 命令正常退出，退出码为 0。 |
|
||||
| 执行失败 | 命令退出码非 0，或启动过程中发生错误。 |
|
||||
| 超时已停止 | 达到设置的最长执行时间，已停止本次进程。 |
|
||||
| 已停止 | 本次进程被停止，可能是手动操作或系统停止服务。 |
|
||||
| 执行被中断 | 执行期间系统重启或进程中断，未取得完整结果。 |
|
||||
|
||||
成功只表示脚本返回了 0，不保证某个证书一定已更新、备份一定可恢复或某个业务结果一定正确；这些应结合脚本输出核对。
|
||||
|
||||
“停用”只取消将来的调度，不会强制中止已经开始的执行。“停止本次”会终止执行进程，脚本可能尚未完成修改，操作前会提示确认。
|
||||
|
||||
目前单任务窗口展示最近 40 次执行，全局窗口展示最近 80 次。服务器执行记录按 90 天清理；每次输出上限约 512 KiB，网页每次读取最后 128 KiB。输出与脚本快照还有合计 256 MiB 的空间限制，较旧文件可能提前清理，执行状态元数据仍保留至期限。
|
||||
|
||||
删除任务会停用计划并移除模块生成的定时器，保留已有执行记录。历史记录可在“全部执行记录”查看。
|
||||
|
||||
## 6. 参考与技术栈
|
||||
|
||||
界面参考 [宝塔 Linux 面板](https://www.bt.cn/) 计划任务的组织方式：任务列表、执行周期、Shell 脚本、启停、手动执行和执行日志。本次功能自行编写，沿用已有监控页面的深色风格，没有安装宝塔，也没有复制其面板源码。
|
||||
|
||||
原监控布局仍参考：
|
||||
|
||||
- [Zabbix](https://github.com/zabbix/zabbix)：设备总览和告警分级。
|
||||
- [Netdata](https://github.com/netdata/netdata)：实时指标展示。
|
||||
- [Grafana](https://github.com/grafana/grafana)：历史曲线、图例和时间范围。
|
||||
|
||||
| 部分 | 使用技术 |
|
||||
|---|---|
|
||||
| 监控与任务前端 | HTML、CSS、原生 JavaScript、原生 `<dialog>` 对话框。 |
|
||||
| 曲线 | 原生 SVG，保留原有 CPU、内存、GPU 温度、功耗和利用率历史。 |
|
||||
| 网页后端 | Python 3.12 标准库，`ThreadingHTTPServer`。 |
|
||||
| 数据库 | SQLite，WAL 模式；监控数据与任务数据分库存储。 |
|
||||
| 系统调度 | systemd `.timer` 和 `.service`，不是浏览器内的倒计时。 |
|
||||
| 网页与特权服务通信 | 本机 Unix socket，限制连接进程身份。 |
|
||||
| 管理密码 | 随机盐、PBKDF2-HMAC-SHA256，240000 次迭代。 |
|
||||
| 登录态 | 内存会话、HttpOnly／SameSite=Strict Cookie；写操作同时检查 Origin 与 CSRF。 |
|
||||
| 执行控制 | 独立进程组、文件锁、超时、停止信号、限量输出、执行审计。 |
|
||||
|
||||
没有新增 npm、pip 或外部 CDN 依赖。服务器不需要 Node.js；开发阶段使用 Node.js 检查 JavaScript 语法。
|
||||
|
||||
## 7. 为什么要拆成两个服务
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A[浏览器：监控与任务界面] --> B[hardware-dashboard\n普通账号\nHTTP 8090]
|
||||
B --> C[(原监控数据库)]
|
||||
B -->|本机 Unix socket\n验证会话与 CSRF| D[hardware-taskd\nroot 任务管理服务]
|
||||
D --> E[(独立任务数据库\nroot 专用目录)]
|
||||
D --> F[systemd 定时器]
|
||||
F --> G[task_exec\n执行指定任务]
|
||||
G --> H[执行日志与结果]
|
||||
H --> D
|
||||
```
|
||||
|
||||
硬件网页继续使用原来的 `hardware-dashboard` 普通账号。新增 root 服务只通过本机 Unix socket 接收经验证的任务操作，不额外开放网络端口。
|
||||
|
||||
任务内容不插入 systemd 的 `ExecStart`。生成的服务只启动固定的执行器和经过验证的任务 ID；执行器从 root 专用数据库读取任务内容。这避免任务名称、换行或参数变成 systemd 单元配置。
|
||||
|
||||
上传或粘贴的脚本文本会统一换行为 Linux 格式。服务器脚本路径和参数以独立参数列表传递；命令内容模式则按管理员输入交给 Bash 执行。
|
||||
|
||||
root 服务不是恶意脚本隔离环境。管理员既然有权输入 root 命令，就应当视为具有服务器管理员权限。
|
||||
|
||||
## 8. 文件与数据库位置
|
||||
|
||||
以下均在 `192.168.1.37`：
|
||||
|
||||
| 路径 | 内容 |
|
||||
|---|---|
|
||||
| `/opt/hardware-dashboard/server.py` | 网页 HTTP 和监控 API；新增任务 API 代理。 |
|
||||
| `/opt/hardware-dashboard/static/tasks.js` | 任务表单、登录、启停、执行和日志交互。 |
|
||||
| `/opt/hardware-dashboard/task_core.py` | 校验、周期转换、任务数据库和 systemd 管理。 |
|
||||
| `/opt/hardware-dashboard/taskd.py` | 本机任务服务、会话验证、审计和清理。 |
|
||||
| `/opt/hardware-dashboard/task_exec.py` | 单次执行、输出、超时与退出结果。 |
|
||||
| `/etc/systemd/system/hardware-taskd.service` | 新增任务管理常驻服务。 |
|
||||
| `/etc/systemd/system/hwjob-任务ID.timer` | 本模块为已创建任务生成的定时器。 |
|
||||
| `/etc/systemd/system/hwjob-任务ID.service` | 对应的执行服务。 |
|
||||
| `/run/hardware-taskd/control.sock` | 普通网页与 root 服务之间的本机通信入口。 |
|
||||
| `/etc/hardware-task-admin.json` | 管理密码的盐与散列，仅 root 可读。 |
|
||||
| `/var/lib/hardware-taskd/tasks.sqlite3` | 任务定义、执行元数据和审计。 |
|
||||
| `/var/lib/hardware-taskd/scripts/` | 脚本内容模式的执行快照。 |
|
||||
| `/var/log/hardware-taskd/` | 每次执行的输出日志，仅经管理登录读取。 |
|
||||
| `/var/lib/hardware-dashboard/monitor.sqlite3` | 原来的性能和硬件日志数据库，继续保留。 |
|
||||
| `/root/hardware-task-admin-login.txt` | 初始登录信息的服务器副本，仅 root 可读。 |
|
||||
|
||||
原有 `hwlog` 命令、硬件采集器及性能历史继续运行。本模块只管理自己创建的 `hwjob-*` 单元，不接管现有 crontab、系统 timer 或其他面板任务。
|
||||
|
||||
## 9. 常用维护命令
|
||||
|
||||
```bash
|
||||
# 查看网页服务与任务服务
|
||||
systemctl status hardware-dashboard.service hardware-taskd.service
|
||||
|
||||
# 查看任务管理服务自身报错
|
||||
journalctl -u hardware-taskd.service -n 50 --no-pager
|
||||
|
||||
# 查看本模块的全部系统定时器
|
||||
systemctl list-timers --all 'hwjob-*.timer'
|
||||
|
||||
# 查看某个任务的系统层日志，任务 ID 可从页面或数据库查出
|
||||
# 将下方 32 位 ID 替换成实际值，不要把说明文字直接输入命令
|
||||
systemctl status hwjob-0123456789abcdef0123456789abcdef.service
|
||||
|
||||
# 重启网页服务：不会取消已启用的任务
|
||||
systemctl restart hardware-dashboard.service
|
||||
|
||||
# 重启任务管理服务：已有网页登录会失效，系统定时器仍独立运行
|
||||
systemctl restart hardware-taskd.service
|
||||
```
|
||||
|
||||
忘记任务管理密码时，可通过 SSH 以 root 执行以下命令。新密码在交互输入，不放进命令行参数：
|
||||
|
||||
```bash
|
||||
python3 /opt/hardware-dashboard/taskd.py password
|
||||
systemctl restart hardware-taskd.service
|
||||
```
|
||||
|
||||
只停止网页或任务管理进程，不等于停用计划。要停止后续执行，应先在页面将对应任务停用；需要立即中断时再使用“停止本次”。
|
||||
|
||||
## 10. 备份与回退
|
||||
|
||||
本次更新前的网页代码备份：`/opt/hardware-dashboard-backup.vNWXhLKD/`。
|
||||
|
||||
任务数据库与监控数据库分开。备份正在运行的 SQLite 应使用 SQLite 的备份接口，不能只复制主文件而忽略 WAL。备份任务数据库、管理密码散列和脚本／日志时，应继续保持 root 专用权限。
|
||||
|
||||
如需回退界面，先在页面停用所有不再需要的任务，再停止任务管理服务并恢复代码。只恢复旧网页代码不会自动删除或停用已经生成的系统定时器。
|
||||
|
||||
## 11. 已完成的验证
|
||||
|
||||
### 自动化测试
|
||||
|
||||
16 项本地和服务器测试通过，覆盖：
|
||||
|
||||
- 原监控数据、问题合并、时钟回拨及过期状态。
|
||||
- 每日／每周／每月／间隔周期与 Cron 字段解析。
|
||||
- Cron 日期和星期的 OR 语义、非法范围和步长。
|
||||
- 任务名称与脚本内容不会成为 systemd 单元配置指令。
|
||||
- 未登录和错误 CSRF 不会调用特权任务操作。
|
||||
- 修改密码使所有旧会话失效。
|
||||
- 日志读取量上限与路径编号验证。
|
||||
|
||||
### 服务器真实执行验证
|
||||
|
||||
使用明确命名的“部署验证”任务，确认了：
|
||||
|
||||
| 验证内容 | 实际结果 |
|
||||
|---|---|
|
||||
| 手动打印命令 | 成功，退出码 0。 |
|
||||
| 指定日期自动触发 | 在计划时间由 systemd 自动执行，记录为 `scheduled`，退出码 0。 |
|
||||
| Python 脚本内容 | 成功，输出 `PYTHON_TASK_OK`。 |
|
||||
| 服务器已有文件执行 | 成功，输出 `EXISTING_FILE_OK`。 |
|
||||
| 主动返回退出码 7 | 正确显示失败，保留错误输出。 |
|
||||
| 10 秒超时 | 约 10 秒后停止，显示超时。 |
|
||||
| 手动停止等待任务 | 正确停止并记录状态。 |
|
||||
| 编辑、启用和停用 | 页面与系统定时器状态一致。 |
|
||||
| 未登录、错误来源、错误 CSRF | 分别被 HTTP 401／403 拒绝。 |
|
||||
|
||||
这些测试仅打印信息或短暂等待，没有执行重启、关机、证书续签或修改现有业务配置。测试计划在交付前移除，历史中保留有“部署验证”字样的执行记录。
|
||||
26
docs/REPOSITORY.md
Normal file
26
docs/REPOSITORY.md
Normal file
@ -0,0 +1,26 @@
|
||||
# Git 提交说明
|
||||
|
||||
仓库已按可发布项目结构整理。源代码和部署模板可提交；运行状态、日志、数据库和管理凭据不应提交。
|
||||
|
||||
## 提交前检查
|
||||
|
||||
```
|
||||
python -m unittest discover -s tests -p "test_*.py" -v
|
||||
python -m py_compile server.py task_core.py taskd.py task_exec.py scripts/hwlog
|
||||
```
|
||||
|
||||
确认没有新增以下内容：
|
||||
|
||||
- /var/lib/hardware-dashboard/monitor.sqlite3
|
||||
- /var/log/hardware-watch/*.jsonl
|
||||
- /etc/hardware-task-admin.json
|
||||
- 管理员密码、SSH 密码、私钥或真实硬件 UUID
|
||||
|
||||
真实配置文件 config/hardware-watch.json 已被 .gitignore 忽略。仓库只保留 config/hardware-watch.example.json。
|
||||
|
||||
## 初始化远端
|
||||
|
||||
```
|
||||
git remote add origin <你的仓库地址>
|
||||
git push -u origin main
|
||||
```
|
||||
10
pyproject.toml
Normal file
10
pyproject.toml
Normal file
@ -0,0 +1,10 @@
|
||||
[project]
|
||||
name = "hardware-dashboard"
|
||||
version = "1.2.0"
|
||||
description = "LAN hardware monitoring dashboard with authenticated scheduled tasks"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["."]
|
||||
438
scripts/hwlog
Executable file
438
scripts/hwlog
Executable file
@ -0,0 +1,438 @@
|
||||
#!/usr/bin/python3
|
||||
"""Read-only hardware sampling and bounded, persistent event summaries."""
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
BASE = Path('/var/lib/hardware-watch')
|
||||
LOG = Path('/var/log/hardware-watch')
|
||||
CONFIG = Path('/etc/hardware-watch.json')
|
||||
VERSION = '1.0'
|
||||
PATTERN = (r'NVRM|NVIDIA|AER:|PCIe Bus Error|BadTLP|BadDLLP|'
|
||||
r'EDAC|Machine check|Hardware Error|mce:|'
|
||||
r'I/O error|Buffer I/O|EXT4-fs error|BTRFS.*error|'
|
||||
r'nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail|reset)|'
|
||||
r'critical temperature|temperature above threshold|thermal.*(thrott|critical)')
|
||||
NAMES = {
|
||||
'power': '显卡辅助供电检测失败', 'gpu_init': '显卡驱动初始化失败',
|
||||
'pcie_fatal': 'PCIe 不可纠正错误', 'pcie_correctable': 'PCIe 可纠正通信错误',
|
||||
'gpu_xid': 'NVIDIA Xid 错误', 'memory_cpu': 'CPU／内存硬件报错',
|
||||
'storage': '磁盘或存储通信错误', 'thermal': '硬件温度告警',
|
||||
'gpu_missing': 'PCIe 显卡数量不足', 'gpu_driver': '显卡管理接口异常',
|
||||
'gpu_identity': '预期显卡 UUID 缺失', 'gpu_temp': '显卡温度超过设定阈值',
|
||||
'disk_missing': '预期磁盘缺失', 'smart': '磁盘 SMART 健康告警',
|
||||
'sensor_temp': '传感器温度告警', 'journal': '系统日志采集异常',
|
||||
'monitor': '硬件日志采集器', 'boot': '系统启动',
|
||||
'clock': '系统时钟发生明显变化',
|
||||
}
|
||||
|
||||
def stamp():
|
||||
return dt.datetime.now().astimezone().isoformat(timespec='seconds')
|
||||
|
||||
def read(path, default=''):
|
||||
try:
|
||||
return Path(path).read_text().strip()
|
||||
except OSError:
|
||||
return default
|
||||
|
||||
def load(path, default):
|
||||
try:
|
||||
return json.loads(Path(path).read_text())
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
def save(path, obj):
|
||||
tmp = Path(str(path) + '.tmp')
|
||||
tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2) + '\n')
|
||||
os.chmod(tmp, 0o640)
|
||||
os.replace(tmp, path)
|
||||
|
||||
def command(args, timeout=12):
|
||||
try:
|
||||
p = subprocess.run(args, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=timeout, env={**os.environ, 'LC_ALL': 'C'})
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return 124, '', str(exc)
|
||||
|
||||
def emit(kind, severity='INFO', **values):
|
||||
row = {'observed_at': stamp(), 'severity': severity, 'category': kind,
|
||||
'summary': NAMES.get(kind, kind), **values}
|
||||
path = LOG / ('events-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl')
|
||||
with path.open('a', encoding='utf-8') as f:
|
||||
f.write(json.dumps(row, ensure_ascii=False) + '\n')
|
||||
os.chmod(path, 0o640)
|
||||
|
||||
def classify(message):
|
||||
low = message.lower()
|
||||
if 'necessary power cables' in low:
|
||||
return 'power', 'ERROR'
|
||||
if 'rminitadapter failed' in low or 'rm_init_adapter failed' in low:
|
||||
return 'gpu_init', 'ERROR'
|
||||
if 'no nvidia gpu found' in low:
|
||||
return 'gpu_missing', 'WARN'
|
||||
if 'fallen off' in low or re.search(r'\bXid\b', message):
|
||||
return 'gpu_xid', 'ERROR'
|
||||
if any(x in low for x in ('severity=uncorrectable', 'aer: device recovery failed', 'non-functional downstream link')):
|
||||
return 'pcie_fatal', 'ERROR'
|
||||
if 'severity=correctable' in low or 'badtlp' in low or 'baddllp' in low or re.search(r'\[\s*12\]\s+Timeout', message):
|
||||
return 'pcie_correctable', 'WARN'
|
||||
if re.search(r'machine check|hardware error|edac.*(\b[cu]e\b|error)|mce:.*(error|bank)', low):
|
||||
return 'memory_cpu', 'ERROR'
|
||||
if re.search(r'i/o error|buffer i/o|ext4-fs error|btrfs.*error|nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail)', low):
|
||||
return 'storage', 'ERROR'
|
||||
if re.search(r'critical temperature|temperature above threshold|thermal.*(thrott|critical)', low):
|
||||
return 'thermal', 'WARN'
|
||||
return None
|
||||
|
||||
def kernel_events(state, cfg):
|
||||
args = ['journalctl', '--no-pager', '-o', 'json', '_TRANSPORT=kernel',
|
||||
'--grep=' + PATTERN, '--case-sensitive=no', '--lines=+20000']
|
||||
cursor = state.get('cursor')
|
||||
args.append('--after-cursor=' + cursor if cursor else '--since=' + cfg.get('history_since', '7 days ago'))
|
||||
code, out, err = command(args, 22)
|
||||
if code not in (0, 1):
|
||||
state['journal_error'] = err[:1000] or f'journalctl exit {code}'
|
||||
emit('journal', 'ERROR', detail=err[:1000], action='下次继续重试；原始 journal 未删除。')
|
||||
if 'cursor' in err.lower() and 'cursor' in state:
|
||||
state.pop('cursor')
|
||||
return
|
||||
state.pop('journal_error', None)
|
||||
state['journal_last_success'] = stamp()
|
||||
entries = []
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
entries.append(json.loads(line))
|
||||
except ValueError:
|
||||
continue
|
||||
# A journal with a backwards clock step can replay records before its cursor.
|
||||
recent = state.get('recent_source_ids', [])
|
||||
seen = set(recent)
|
||||
def source_id(entry):
|
||||
return hashlib.sha256(entry.get('__CURSOR', '').encode()).hexdigest()[:24]
|
||||
if not recent and cursor and entries and entries[-1].get('__CURSOR') == cursor:
|
||||
recent = [source_id(entry) for entry in entries]
|
||||
seen = set(recent)
|
||||
emit('clock', 'WARN', detail='检测到历史校时造成 journal 游标重放；已跳过此前处理过的记录，避免误计为新故障。')
|
||||
groups = {}
|
||||
rows = 0
|
||||
for entry in entries:
|
||||
rows += 1
|
||||
state['cursor'] = entry.get('__CURSOR', state.get('cursor'))
|
||||
token = source_id(entry)
|
||||
if token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
recent.append(token)
|
||||
message = entry.get('MESSAGE', '')
|
||||
if not isinstance(message, str):
|
||||
continue
|
||||
result = classify(message)
|
||||
if not result:
|
||||
continue
|
||||
category, severity = result
|
||||
match = re.search(r'(?:[0-9a-f]{4}:)?[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]', message, re.I)
|
||||
device = match.group(0) if match else ''
|
||||
normalized = re.sub(r'0x[0-9a-f]+|\b[0-9]+\b', '#', message, flags=re.I)
|
||||
key = (entry.get('_BOOT_ID', ''), category, device, normalized)
|
||||
realtime = int(entry.get('__REALTIME_TIMESTAMP', 0)) / 1000000
|
||||
when = dt.datetime.fromtimestamp(realtime).astimezone().isoformat(timespec='seconds')
|
||||
observed = stamp()
|
||||
monotonic = entry.get('__MONOTONIC_TIMESTAMP')
|
||||
group = groups.setdefault(key, {'category': category, 'severity': severity,
|
||||
'source': 'kernel', 'device': device, 'boot_id': entry.get('_BOOT_ID', ''),
|
||||
'first_at': when, 'last_at': when, 'first_monotonic_us': entry.get('__MONOTONIC_TIMESTAMP'),
|
||||
'last_monotonic_us': monotonic, 'first_observed_at': observed,
|
||||
'last_observed_at': observed, 'count': 0, 'detail': message[:1600]})
|
||||
group['count'] += 1
|
||||
group['last_at'] = when
|
||||
group['last_monotonic_us'] = monotonic
|
||||
group['last_observed_at'] = observed
|
||||
for row in groups.values():
|
||||
emit(row.pop('category'), row.pop('severity'), **row)
|
||||
state['recent_source_ids'] = recent[-4096:]
|
||||
pending = rows >= 20000
|
||||
if pending and not state.get('journal_backlog'):
|
||||
emit('journal', 'WARN', detail='本轮读取 20000 条匹配记录；下轮从游标继续处理积压，未跳过中间记录。')
|
||||
elif not pending and state.get('journal_backlog'):
|
||||
emit('journal', 'INFO', detail='历史日志积压已处理完成，后续按游标增量采集。')
|
||||
state['journal_backlog'] = pending
|
||||
|
||||
def pci_gpus():
|
||||
items = []
|
||||
for path in sorted(Path('/sys/bus/pci/devices').glob('*')):
|
||||
if read(path / 'vendor') != '0x10de' or not read(path / 'class').startswith('0x03'):
|
||||
continue
|
||||
driver = path / 'driver'
|
||||
items.append({'pci': path.name, 'device_id': read(path / 'device'),
|
||||
'driver': driver.resolve().name if driver.exists() else None,
|
||||
'link_speed': read(path / 'current_link_speed'), 'link_width': read(path / 'current_link_width')})
|
||||
return items
|
||||
|
||||
def gpu_query():
|
||||
args = ['nvidia-smi', '--query-gpu=pci.bus_id,name,uuid,temperature.gpu,power.draw,memory.total',
|
||||
'--format=csv,noheader,nounits']
|
||||
code, out, err = command(args)
|
||||
result = []
|
||||
if code == 0:
|
||||
for row in csv.reader(out.splitlines(), skipinitialspace=True):
|
||||
if len(row) != 6:
|
||||
continue
|
||||
try:
|
||||
result.append({'pci': row[0], 'name': row[1], 'uuid': row[2],
|
||||
'temperature_c': float(row[3]), 'power_w': float(row[4]), 'memory_mib': float(row[5])})
|
||||
except ValueError:
|
||||
result.append({'pci': row[0], 'name': row[1], 'uuid': row[2], 'raw': row})
|
||||
return result, None if code == 0 and result else (err or out or 'nvidia-smi 未返回设备')[:1200]
|
||||
|
||||
def disk_names():
|
||||
return sorted(p.name for p in Path('/sys/block').glob('*')
|
||||
if not p.name.startswith(('loop', 'ram', 'zram', 'dm-', 'md')))
|
||||
|
||||
def sensors():
|
||||
result = []
|
||||
for path in sorted(Path('/sys/class/hwmon').glob('hwmon*')):
|
||||
for inp in path.glob('temp*_input'):
|
||||
try:
|
||||
prefix = inp.name[:-6]
|
||||
result.append({'chip': read(path / 'name'), 'label': read(path / (prefix + '_label'), prefix),
|
||||
'celsius': int(read(inp)) / 1000,
|
||||
'critical_c': int(read(path / (prefix + '_crit'), '0')) / 1000})
|
||||
except ValueError:
|
||||
pass
|
||||
return result
|
||||
|
||||
def active_problem(problems, key, kind, detail, severity='ERROR'):
|
||||
problems[key] = {'category': kind, 'severity': severity, 'detail': detail}
|
||||
|
||||
def smart_checks(state, problems, disks):
|
||||
# Hourly, read-only SMART query. Never starts self-tests or changes disk settings.
|
||||
cached = state.get('smart', {})
|
||||
if time.time() - state.get('smart_at', 0) >= 3600:
|
||||
cached = {}
|
||||
for name in disks:
|
||||
code, out, err = command(['smartctl', '-a', '-j', '/dev/' + name], 7)
|
||||
try:
|
||||
data = json.loads(out)
|
||||
except ValueError:
|
||||
cached[name] = {'available': False, 'detail': (err or out)[:500]}
|
||||
continue
|
||||
nv = data.get('nvme_smart_health_information_log', {})
|
||||
cached[name] = {'available': 'smart_status' in data,
|
||||
'passed': data.get('smart_status', {}).get('passed'),
|
||||
'critical_warning': nv.get('critical_warning'), 'media_errors': nv.get('media_errors'),
|
||||
'temperature_c': data.get('temperature', {}).get('current'),
|
||||
'percentage_used': nv.get('percentage_used'), 'exit_status': code}
|
||||
state['smart'], state['smart_at'] = cached, time.time()
|
||||
for name, item in cached.items():
|
||||
if not item.get('available'):
|
||||
active_problem(problems, 'smart-unavailable:' + name, 'smart',
|
||||
f'{name}: SMART 读取不可用，当前仅监控内核存储报错。{item.get("detail", "")}', 'WARN')
|
||||
elif item.get('passed') is False or item.get('critical_warning', 0):
|
||||
active_problem(problems, 'smart:' + name, 'smart', f'{name}: {json.dumps(item, ensure_ascii=False)}')
|
||||
previous = state.get('smart_media', {}).get(name)
|
||||
media = item.get('media_errors')
|
||||
if media is not None and previous is not None and media > previous:
|
||||
emit('smart', 'ERROR', detail=f'{name}: NVMe 介质错误计数从 {previous} 增加到 {media}')
|
||||
if media is not None:
|
||||
state.setdefault('smart_media', {})[name] = media
|
||||
return cached
|
||||
|
||||
def collect(state, cfg):
|
||||
now = stamp()
|
||||
boot = read('/proc/sys/kernel/random/boot_id')
|
||||
clock = {'wall': time.time(), 'monotonic': time.monotonic(), 'boot_id': boot}
|
||||
old_clock = state.get('clock', {})
|
||||
if old_clock.get('boot_id') == boot:
|
||||
delta = (clock['wall'] - old_clock['wall']) - (clock['monotonic'] - old_clock['monotonic'])
|
||||
if abs(delta) > 60:
|
||||
emit('clock', 'WARN', detail=f'同一次启动中墙上时钟相对单调时钟变化 {delta:.1f} 秒；核对事件时间时应结合启动 ID 和单调时钟。')
|
||||
state['clock'] = clock
|
||||
if boot != state.get('boot_id'):
|
||||
emit('boot', boot_id=boot, previous_boot_id=state.get('boot_id'),
|
||||
detail='监控首次观察到本次启动；准确启动信息可通过 journalctl 查看。')
|
||||
state['boot_id'] = boot
|
||||
state['smart_at'] = 0
|
||||
kernel_events(state, cfg)
|
||||
pci = pci_gpus()
|
||||
gpu, nv_error = gpu_query() if pci else ([], 'PCIe 总线没有 NVIDIA 显卡')
|
||||
disks = disk_names()
|
||||
temps = sensors()
|
||||
problems = {}
|
||||
if state.get('journal_error'):
|
||||
active_problem(problems, 'journal-read', 'journal', state['journal_error'], 'WARN')
|
||||
if len(pci) < cfg['expected_gpu_count']:
|
||||
active_problem(problems, 'gpu-count', 'gpu_missing',
|
||||
f'预期 {cfg["expected_gpu_count"]} 块 NVIDIA 显卡，PCIe 实际 {len(pci)} 块。')
|
||||
if nv_error or len(gpu) < len(pci):
|
||||
active_problem(problems, 'gpu-driver', 'gpu_driver', nv_error or f'PCIe {len(pci)} 块，nvidia-smi 仅 {len(gpu)} 块。')
|
||||
if not nv_error:
|
||||
missing = sorted(set(cfg.get('expected_gpu_uuids', [])) - {x['uuid'] for x in gpu})
|
||||
if missing:
|
||||
active_problem(problems, 'gpu-uuid', 'gpu_identity', '缺少预期 GPU：' + ', '.join(missing))
|
||||
for item in gpu:
|
||||
if item.get('temperature_c', 0) >= cfg.get('gpu_warn_c', 85):
|
||||
active_problem(problems, 'gpu-temp:' + item['uuid'], 'gpu_temp',
|
||||
f'{item["name"]} {item["pci"]}: {item["temperature_c"]}°C', 'WARN')
|
||||
for name in set(cfg.get('expected_disks', [])) - set(disks):
|
||||
active_problem(problems, 'disk:' + name, 'disk_missing', name)
|
||||
for item in temps:
|
||||
if item['critical_c'] > 0 and item['celsius'] >= item['critical_c']:
|
||||
active_problem(problems, 'sensor:' + item['chip'] + item['label'], 'sensor_temp',
|
||||
json.dumps(item, ensure_ascii=False), 'ERROR')
|
||||
smart = smart_checks(state, problems, disks)
|
||||
previous = state.get('active', {})
|
||||
for key, value in problems.items():
|
||||
old = previous.get(key)
|
||||
value['since'] = old.get('since', now) if old else now
|
||||
if not old or old.get('detail') != value['detail']:
|
||||
emit(value['category'], value['severity'], event='FAULT', key=key, detail=value['detail'], since=value['since'])
|
||||
for key, value in previous.items():
|
||||
if key not in problems:
|
||||
emit(value['category'], 'INFO', event='RECOVERED', key=key, detail='此前异常在本轮检查中已消失。', since=value.get('since'))
|
||||
state['active'] = problems
|
||||
status = {'checked_at': now, 'boot_id': boot, 'pci_gpus': pci, 'gpus': gpu,
|
||||
'nvidia_error': nv_error, 'disks': disks, 'smart': smart, 'sensors': temps, 'active': problems,
|
||||
'journal_last_success': state.get('journal_last_success'), 'journal_backlog': state.get('journal_backlog', False)}
|
||||
save(BASE / 'status.json', status)
|
||||
if time.time() - state.get('snapshot_at', 0) >= 900:
|
||||
path = LOG / ('samples-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl')
|
||||
with path.open('a', encoding='utf-8') as f:
|
||||
f.write(json.dumps(status, ensure_ascii=False) + '\n')
|
||||
os.chmod(path, 0o640)
|
||||
state['snapshot_at'] = time.time()
|
||||
save(BASE / 'state.json', state)
|
||||
# Age and total-size limits apply only to this monitor's own log files.
|
||||
files = sorted(LOG.glob('*.jsonl'), key=lambda p: p.stat().st_mtime)
|
||||
cutoff = time.time() - cfg.get('retention_days', 90) * 86400
|
||||
total = sum(p.stat().st_size for p in files)
|
||||
for path in files:
|
||||
if path.stat().st_mtime < cutoff or total > 256 * 1024 * 1024:
|
||||
if path.name.endswith(dt.datetime.now().strftime('%Y-%m-%d') + '.jsonl'):
|
||||
continue
|
||||
total -= path.stat().st_size
|
||||
path.unlink()
|
||||
|
||||
def show_status():
|
||||
status = load(BASE / 'status.json', {})
|
||||
print('硬件管理日志 | 最近检查：' + status.get('checked_at', '尚未运行'))
|
||||
age = (dt.datetime.now().astimezone() - dt.datetime.fromisoformat(status['checked_at'])).total_seconds() if status else 9999
|
||||
if age > 180:
|
||||
print('警告：最近检查超过 3 分钟，请检查 systemctl status hardware-watch.timer hardware-watch.service')
|
||||
for item in status.get('pci_gpus', []):
|
||||
print(f'PCIe {item["pci"]} | {item["link_speed"]} x{item["link_width"]} | {item["driver"]}')
|
||||
for item in status.get('gpus', []):
|
||||
print(f'{item["name"]} | {item["pci"]} | {item.get("temperature_c", "?")}°C | {item.get("power_w", "?")}W | {item["uuid"]}')
|
||||
for name, item in status.get('smart', {}).items():
|
||||
print(f'磁盘 {name} | SMART={item.get("passed")} | 温度={item.get("temperature_c")}°C | 介质错误={item.get("media_errors")}')
|
||||
print('当前采样告警：' + str(len(status.get('active', {}))))
|
||||
for value in status.get('active', {}).values():
|
||||
print(f' [{value["severity"]}] {NAMES[value["category"]]}：{value["detail"]}')
|
||||
if status.get('journal_backlog'):
|
||||
print('历史内核日志正在补录，实时 GPU／磁盘采样已启用。')
|
||||
print('PCIe 等内核历史事件另见 hwlog events；当前在线不代表此前没有通信错误。')
|
||||
|
||||
def show_events(args):
|
||||
rows = []
|
||||
seen = set()
|
||||
status = load(BASE / 'status.json', {})
|
||||
active = status.get('active', {})
|
||||
since = dt.datetime.fromisoformat(args.since).astimezone() if args.since else None
|
||||
for path in sorted(LOG.glob('events-*.jsonl')):
|
||||
for line in path.read_text().splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
if row.get('source') == 'kernel':
|
||||
key = (row.get('boot_id'), row.get('category'), row.get('device'),
|
||||
row.get('first_monotonic_us'), row.get('last_monotonic_us'), row.get('detail'))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
event_time = row.get('first_at', row['observed_at'])
|
||||
source_last = row.get('last_at', event_time)
|
||||
observed = row.get('observed_at', event_time)
|
||||
if since and max(source_last, observed) < since.isoformat():
|
||||
continue
|
||||
if args.category and row['category'] != args.category:
|
||||
continue
|
||||
rows.append(row)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
# Recently collected records remain visible even if old source clocks were ahead.
|
||||
rows.sort(key=lambda x: x['observed_at'])
|
||||
for row in rows[-max(1, args.limit):]:
|
||||
if args.json:
|
||||
print(json.dumps(row, ensure_ascii=False))
|
||||
else:
|
||||
print(f'{row.get("first_at", row["observed_at"])} [{row["severity"]}] {row["summary"]} {row.get("device", "")} {row.get("event", "")}')
|
||||
if 'count' in row:
|
||||
print(f' 原始故障时间：{row.get("first_at", "未知")} 至 {row.get("last_at", "未知")}')
|
||||
print(f' 监控发现时间：{row.get("first_observed_at", row["observed_at"])} 至 {row.get("last_observed_at", row["observed_at"])}')
|
||||
first_mono = row.get('first_monotonic_us')
|
||||
last_mono = row.get('last_monotonic_us')
|
||||
if first_mono and last_mono:
|
||||
print(f' 本次启动经过：{int(first_mono)/1000000:.3f}s 至 {int(last_mono)/1000000:.3f}s；启动 {row.get("boot_id", "")}')
|
||||
print(f' 本批重复次数：{row["count"]}')
|
||||
current = False
|
||||
for value in active.values():
|
||||
detail = value.get('detail', '')
|
||||
if value.get('category') == row.get('category') and (not row.get('device') or row.get('device') in detail):
|
||||
current = True
|
||||
break
|
||||
print(' 当前判断：当前采样仍有同类告警' if current else ' 当前判断：当前采样未复现，属于历史记录或已恢复')
|
||||
if row['last_at'] < row['first_at']:
|
||||
print(' 时间提示：源日志时钟发生回拨，以上首末时间不能按日历顺序判断；请结合启动 ID、单调时钟及原始 journal。')
|
||||
if row['first_at'] > row['observed_at']:
|
||||
print(' 时间提示：源日志日期晚于采集日期，源系统时钟曾异常；采集时间为 ' + row['observed_at'])
|
||||
print(' ' + row.get('detail', ''))
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='硬件管理日志：每分钟采样，查看状态和历史故障。')
|
||||
sub = parser.add_subparsers(dest='action')
|
||||
sub.add_parser('collect')
|
||||
sub.add_parser('status')
|
||||
events = sub.add_parser('events')
|
||||
events.add_argument('--since', help='例如 2026-11-02 或 2026-11-02T12:00:00+08:00')
|
||||
events.add_argument('--category', choices=sorted(NAMES))
|
||||
events.add_argument('--limit', type=int, default=50)
|
||||
events.add_argument('--json', action='store_true')
|
||||
sub.add_parser('self-test')
|
||||
args = parser.parse_args()
|
||||
if args.action == 'self-test':
|
||||
cases = [('NVRM: GPU does not have the necessary power cables connected.', 'power'),
|
||||
('NVRM: RmInitAdapter failed! (0x24:0x1c:1603)', 'gpu_init'),
|
||||
('PCIe Bus Error: severity=Correctable, type=Data Link Layer', 'pcie_correctable'),
|
||||
('PCIe Bus Error: severity=Uncorrectable (Fatal)', 'pcie_fatal'),
|
||||
('NVRM: Xid (PCI:0000:03:00): 79, GPU has fallen off the bus.', 'gpu_xid'),
|
||||
('nvme nvme0: I/O 10 timeout, reset controller', 'storage'),
|
||||
('EDAC MC0: 1 CE memory read error', 'memory_cpu')]
|
||||
for message, expected in cases:
|
||||
assert classify(message)[0] == expected, message
|
||||
assert classify('NVRM: loading NVIDIA UNIX x86_64 Kernel Module') is None
|
||||
assert classify('nvidia: module verification failed: tainting kernel') is None
|
||||
print('PASS: 7 hardware error cases and 2 benign driver messages')
|
||||
return
|
||||
if args.action == 'collect':
|
||||
BASE.mkdir(parents=True, exist_ok=True)
|
||||
LOG.mkdir(parents=True, exist_ok=True)
|
||||
with (BASE / 'lock').open('w') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
collect(load(BASE / 'state.json', {}), load(CONFIG, {}))
|
||||
elif args.action == 'events':
|
||||
show_events(args)
|
||||
else:
|
||||
show_status()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
730
server.py
Normal file
730
server.py
Normal file
@ -0,0 +1,730 @@
|
||||
#!/usr/bin/python3
|
||||
"""LAN-only, read-only hardware dashboard. Python 3.12 standard library."""
|
||||
import argparse
|
||||
import csv
|
||||
from contextlib import contextmanager
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
from http.cookies import SimpleCookie, CookieError
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
DATA = Path(os.environ.get('DASHBOARD_DATA', '/var/lib/hardware-dashboard'))
|
||||
LOGS = Path(os.environ.get('HARDWARE_LOGS', '/var/log/hardware-watch'))
|
||||
HWSTATE = Path(os.environ.get('HARDWARE_STATE', '/var/lib/hardware-watch/status.json'))
|
||||
CONFIG = Path(os.environ.get('HARDWARE_CONFIG', '/etc/hardware-watch.json'))
|
||||
INTERVAL = 10
|
||||
KEEP_DAYS = 30
|
||||
TZ = dt.timezone(dt.timedelta(hours=8))
|
||||
STOP = threading.Event()
|
||||
LOCK = threading.Lock()
|
||||
LATEST = {}
|
||||
TASK_SOCKET='/run/hardware-taskd/control.sock'
|
||||
RANGES = {'1h': 3600, '6h': 21600, '24h': 86400, '7d': 604800, '30d': 2592000}
|
||||
CATEGORY = {
|
||||
'power': ('显卡辅助供电异常', '显卡检测不到必要的外接供电。', '正常关机并断电后，检查该卡供电线两端、转接头和退针情况；仍异常时换用匹配的已知正常供电线测试。'),
|
||||
'gpu_init': ('显卡初始化失败', '系统发现了显卡，但驱动未能让它进入可用状态。', '先查看同一时段是否有供电或 PCIe 错误，再检查供电与插槽；不要直接反复重装驱动。'),
|
||||
'gpu_driver': ('显卡状态读取失败', '管理接口没有返回全部预期显卡。', '对照供电、PCIe 和驱动日志排查；查看下方原始错误。'),
|
||||
'gpu_missing': ('显卡未被发现', '实际发现的显卡少于预期，或内核报告未找到 NVIDIA GPU。', '先确认是否主动拆卡；否则正常关机断电后检查供电与插槽。'),
|
||||
'gpu_identity': ('预期显卡缺失', '当前显卡身份与监控基准不一致。', '若主动更换了显卡，更新监控基准；否则检查对应显卡是否在线。'),
|
||||
'gpu_temp': ('显卡温度偏高', '显卡温度超过配置的告警线。', '检查机箱风道和风扇，必要时降低计算负载；P100 依赖机箱送风。'),
|
||||
'pcie_correctable': ('显卡与主板通信需要重试', '显卡与主板传数据时出现了系统可以自动纠正的通信错误。这条告警本身不表示显卡损坏。', '目前显卡在线时可以继续使用并观察。如果持续报错、任务变慢或显卡消失，安排空闲时正常关机并断电，检查显卡供电线和插槽，再做换线或换槽测试。'),
|
||||
'pcie_fatal': ('PCIe 严重通信错误', '硬件连接出现不可纠正错误，可能导致设备不可用。', '先保存日志；正常关机断电后检查显卡供电、插槽及转接设备。'),
|
||||
'gpu_xid': ('显卡运行错误', 'NVIDIA 驱动记录了 GPU 运行异常。', '结合原始 Xid 编号与同一时间的温度、供电、PCIe 日志排查。'),
|
||||
'memory_cpu': ('CPU／内存硬件告警', '内核报告了机器检查或内存纠错事件。', '保留原始槽位与错误信息，由维护人员检查对应内存、CPU 和主板。'),
|
||||
'storage': ('磁盘读写／通信异常', '内核记录了存储读写错误、超时或重置。', '先确认重要数据已有备份，再检查 SMART 状态、硬盘连接和供电。'),
|
||||
'smart': ('磁盘健康告警', '磁盘 SMART 健康检查异常或无法读取。', '查看具体健康项；若出现介质错误或健康失败，优先备份并安排检修。'),
|
||||
'disk_missing': ('预期磁盘缺失', '监控没有发现预期磁盘。', '确认是否主动移除；否则断电检查连接并安排维护。'),
|
||||
'thermal': ('硬件温度告警', '内核报告过热或温度相关降速。', '检查散热器、风扇和机箱风道，必要时降低负载。'),
|
||||
'sensor_temp': ('传感器温度告警', '硬件传感器温度达到告警阈值。', '检查对应 CPU、磁盘或机箱散热。'),
|
||||
'clock': ('系统时间有变化', '历史系统时间曾跳变，原始时间可能不可靠。', '对照采集时间、启动 ID 和启动后经过时间；带“估算”的时刻不能当作精确时间。'),
|
||||
'journal': ('日志采集状态', '硬件日志采集器记录了自身的运行状态。', '若持续采集失败或状态超过三分钟未更新，请维护人员检查 hardware-watch 服务。'),
|
||||
'boot': ('系统启动记录', '采集器观察到了一个新的系统启动。', '如果不是计划内重启，查看此前是否有断电、温度或硬件异常。'),
|
||||
'monitor': ('监控运行记录', '监控服务自身的状态变化。', '查看详细信息。'),
|
||||
'web_monitor': ('网页采集异常', '网页无法取得最新硬件数据。', '检查监控服务是否正常，不要把旧数据当成实时状态。'),
|
||||
'memory_pressure': ('可用内存偏少', '系统可用内存不足总量的 5%。', '检查正在运行的任务，减少不必要的并发或安排内存扩容。'),
|
||||
}
|
||||
|
||||
def read(path, default=''):
|
||||
try:
|
||||
return Path(path).read_text().strip()
|
||||
except OSError:
|
||||
return default
|
||||
|
||||
def read_json(path, default=None):
|
||||
try:
|
||||
return json.loads(read(path))
|
||||
except (ValueError, TypeError):
|
||||
return {} if default is None else default
|
||||
|
||||
def iso(ts=None):
|
||||
return dt.datetime.fromtimestamp(time.time() if ts is None else ts, TZ).isoformat(timespec='seconds')
|
||||
|
||||
def epoch(value):
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value).timestamp()
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def run(args, timeout=6):
|
||||
try:
|
||||
p = subprocess.run(args, capture_output=True, text=True, timeout=timeout,
|
||||
env={**os.environ, 'LC_ALL': 'C'})
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return 124, '', str(exc)
|
||||
|
||||
@contextmanager
|
||||
def connect():
|
||||
db = sqlite3.connect(DATA / 'monitor.sqlite3', timeout=8)
|
||||
db.row_factory = sqlite3.Row
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def initialize():
|
||||
DATA.mkdir(parents=True, exist_ok=True)
|
||||
with connect() as db:
|
||||
db.execute('PRAGMA journal_mode=WAL')
|
||||
db.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS metrics (
|
||||
ts INTEGER PRIMARY KEY, cpu REAL, memory REAL, cpu_temp REAL,
|
||||
disk_read REAL, disk_write REAL, net_rx REAL, net_tx REAL,
|
||||
gpus TEXT NOT NULL, source TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id TEXT PRIMARY KEY, occurred REAL NOT NULL, observed REAL NOT NULL,
|
||||
category TEXT, severity TEXT, raw TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS event_time ON events(occurred DESC);
|
||||
CREATE INDEX IF NOT EXISTS event_observed ON events(observed DESC);
|
||||
CREATE INDEX IF NOT EXISTS event_category ON events(category, observed DESC);
|
||||
CREATE INDEX IF NOT EXISTS metric_source ON metrics(source, ts);
|
||||
CREATE TABLE IF NOT EXISTS imports (file TEXT PRIMARY KEY, inode TEXT, offset INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS boot_clock (boot TEXT PRIMARY KEY, anchor REAL, mono REAL);
|
||||
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
||||
''')
|
||||
|
||||
def event_id(row):
|
||||
if row.get('source') == 'kernel':
|
||||
value = [row.get(k) for k in ('boot_id', 'category', 'device', 'first_monotonic_us', 'last_monotonic_us', 'detail')]
|
||||
else:
|
||||
value = [row.get(k) for k in ('observed_at', 'category', 'key', 'event', 'detail')]
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
def import_logs(db):
|
||||
for path in sorted(LOGS.glob('*.jsonl')):
|
||||
if not path.name.startswith(('events-', 'samples-')):
|
||||
continue
|
||||
stat = path.stat()
|
||||
checkpoint = db.execute('SELECT inode,offset FROM imports WHERE file=?', (path.name,)).fetchone()
|
||||
offset = checkpoint['offset'] if checkpoint and checkpoint['inode'] == str(stat.st_ino) and checkpoint['offset'] <= stat.st_size else 0
|
||||
if offset == stat.st_size:
|
||||
continue
|
||||
with path.open('rb') as f:
|
||||
f.seek(offset)
|
||||
for _ in range(5000):
|
||||
pos = f.tell()
|
||||
line = f.readline()
|
||||
if not line or not line.endswith(b'\n'):
|
||||
f.seek(pos)
|
||||
break
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
continue
|
||||
if path.name.startswith('samples-'):
|
||||
ts = epoch(row.get('checked_at'))
|
||||
if ts is None or ts < time.time() - KEEP_DAYS * 86400:
|
||||
continue
|
||||
gpus = [{'uuid': x.get('uuid'), 'name': x.get('name'), 'pci': x.get('pci'),
|
||||
'temperature_c': x.get('temperature_c'), 'power_w': x.get('power_w'),
|
||||
'utilization': None, 'memory_used_mib': None,
|
||||
'memory_total_mib': x.get('memory_mib')} for x in row.get('gpus', [])]
|
||||
db.execute('INSERT OR IGNORE INTO metrics VALUES(?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,?,?)',
|
||||
(int(ts), json.dumps(gpus), 'legacy'))
|
||||
else:
|
||||
observed = epoch(row.get('observed_at'))
|
||||
first = epoch(row.get('first_at')) or observed
|
||||
if observed is None:
|
||||
continue
|
||||
last = epoch(row.get('last_at'))
|
||||
mono = row.get('last_monotonic_us')
|
||||
if last and mono and last <= observed + 60 and row.get('boot_id'):
|
||||
m = int(mono) / 1e6
|
||||
db.execute('''INSERT INTO boot_clock VALUES(?,?,?) ON CONFLICT(boot) DO UPDATE
|
||||
SET anchor=excluded.anchor,mono=excluded.mono WHERE excluded.mono > boot_clock.mono''',
|
||||
(row['boot_id'], last - m, m))
|
||||
db.execute('INSERT OR IGNORE INTO events VALUES(?,?,?,?,?,?)',
|
||||
(event_id(row), first, observed, row.get('category', 'monitor'),
|
||||
row.get('severity', 'INFO'), json.dumps(row, ensure_ascii=False)))
|
||||
offset = f.tell()
|
||||
db.execute('INSERT OR REPLACE INTO imports VALUES(?,?,?)', (path.name, str(stat.st_ino), offset))
|
||||
|
||||
def temperatures():
|
||||
result = []
|
||||
for path in Path('/sys/class/hwmon').glob('hwmon*'):
|
||||
chip = read(path / 'name')
|
||||
for inp in path.glob('temp*_input'):
|
||||
prefix = inp.name[:-6]
|
||||
try:
|
||||
result.append({'chip': chip, 'label': read(path / (prefix + '_label'), prefix),
|
||||
'value': int(read(inp)) / 1000})
|
||||
except ValueError:
|
||||
pass
|
||||
return result
|
||||
|
||||
def cpu_counters():
|
||||
fields = list(map(int, read('/proc/stat').splitlines()[0].split()[1:9]))
|
||||
return sum(fields), fields[3] + fields[4]
|
||||
|
||||
def counters():
|
||||
net_rx = net_tx = disk_read = disk_write = 0
|
||||
for line in read('/proc/net/dev').splitlines()[2:]:
|
||||
name, values = line.split(':', 1)
|
||||
if name.strip() == 'lo':
|
||||
continue
|
||||
fields = values.split()
|
||||
net_rx += int(fields[0]); net_tx += int(fields[8])
|
||||
disks = []
|
||||
for path in Path('/sys/block').glob('*'):
|
||||
if path.name.startswith(('loop', 'ram', 'zram', 'dm-', 'md')):
|
||||
continue
|
||||
fields = read(path / 'stat').split()
|
||||
if len(fields) >= 7:
|
||||
disk_read += int(fields[2]) * 512; disk_write += int(fields[6]) * 512
|
||||
disks.append(path.name)
|
||||
return {'net_rx': net_rx, 'net_tx': net_tx, 'disk_read': disk_read, 'disk_write': disk_write}, disks
|
||||
|
||||
def gpu_metrics():
|
||||
fields = 'index,uuid,pci.bus_id,name,utilization.gpu,temperature.gpu,power.draw,power.limit,memory.used,memory.total'
|
||||
code, out, err = run(['nvidia-smi', '--query-gpu=' + fields, '--format=csv,noheader,nounits'])
|
||||
if code:
|
||||
return [], (err or out or '无法读取显卡')[:1000]
|
||||
result = []
|
||||
for row in csv.reader(out.splitlines(), skipinitialspace=True):
|
||||
if len(row) != 10:
|
||||
continue
|
||||
item = {'index': row[0], 'uuid': row[1], 'pci': row[2], 'name': row[3]}
|
||||
for key, value in zip(('utilization', 'temperature_c', 'power_w', 'power_limit_w', 'memory_used_mib', 'memory_total_mib'), row[4:]):
|
||||
try:
|
||||
number = float(value)
|
||||
item[key] = number if math.isfinite(number) else None
|
||||
except ValueError:
|
||||
item[key] = None
|
||||
result.append(item)
|
||||
return result, None if result else 'NVIDIA 管理接口没有返回可用显卡'
|
||||
|
||||
def issue(category, detail, level='WARN', device=''):
|
||||
title, meaning, action = CATEGORY.get(category, CATEGORY['monitor'])
|
||||
return {'category': category, 'title': title, 'meaning': meaning, 'action': action,
|
||||
'detail': detail, 'severity': level, 'device': device}
|
||||
|
||||
def pci_address(value):
|
||||
match = re.search(r'([0-9a-f]{4,8}):([0-9a-f]{2}):([0-9a-f]{2}\.[0-7])$', value or '', re.I)
|
||||
if match:
|
||||
return f'{int(match[1],16):04x}:{match[2].lower()}:{match[3].lower()}'
|
||||
return value or ''
|
||||
|
||||
def device_map(gpus):
|
||||
"""Map current GPU endpoints and their actual sysfs upstream bridges to one identity."""
|
||||
result = {}
|
||||
for index, gpu in enumerate(sorted(gpus,key=lambda x:x['uuid'])):
|
||||
pci = pci_address(gpu['pci'])
|
||||
label = f'第 {index+1} 张 P100' if 'P100' in gpu.get('name','') else f'第 {index+1} 张显卡'
|
||||
entry = {'id':gpu['uuid'], 'label':label, 'pci':pci}
|
||||
result[pci] = entry
|
||||
path = Path('/sys/bus/pci/devices') / pci
|
||||
if path.exists():
|
||||
for parent in path.resolve().parents:
|
||||
if re.fullmatch(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]',parent.name):
|
||||
result.setdefault(parent.name,entry)
|
||||
return result
|
||||
|
||||
def event_component(row,current):
|
||||
address = pci_address(row.get('device',''))
|
||||
same_boot = row.get('boot_id','').replace('-','') == current.get('boot_id','').replace('-','')
|
||||
matched = current.get('device_map',{}).get(address) if same_boot else None
|
||||
if matched:
|
||||
return matched['id'], matched['label'], address
|
||||
if row.get('category','').startswith('pcie'):
|
||||
return address or 'unknown-link', '设备与主板的连接' + (f'（端口 {address[5:]}）' if address else ''), address
|
||||
return address or row.get('key') or 'system', ('设备 '+address[5:]) if address else '这台电脑', address
|
||||
|
||||
def sample(previous):
|
||||
now = time.time()
|
||||
mono = time.monotonic()
|
||||
total, idle = cpu_counters()
|
||||
cpu = None
|
||||
if previous and total > previous['cpu_total']:
|
||||
cpu = max(0, min(100, 100 * (1 - (idle - previous['cpu_idle']) / (total - previous['cpu_total']))))
|
||||
mem = {}
|
||||
for line in read('/proc/meminfo').splitlines():
|
||||
key, value = line.split(':', 1)
|
||||
mem[key] = int(value.strip().split()[0]) * 1024
|
||||
ram_total, ram_available = mem['MemTotal'], mem.get('MemAvailable', mem['MemFree'])
|
||||
count, disks = counters()
|
||||
rates = {key: None for key in count}
|
||||
if previous and mono > previous['mono']:
|
||||
rates = {key: max(0, (value - previous['counters'][key]) / (mono - previous['mono'])) / 1048576
|
||||
for key, value in count.items()}
|
||||
temps = temperatures()
|
||||
cpu_temp = max([x['value'] for x in temps if x['chip'] == 'coretemp'], default=None)
|
||||
disk_temp = max([x['value'] for x in temps if x['chip'] == 'nvme'], default=None)
|
||||
gpu, gpu_error = gpu_metrics()
|
||||
hw = read_json(HWSTATE)
|
||||
cfg = read_json(CONFIG)
|
||||
expected = cfg.get('expected_gpu_count', 2)
|
||||
faults = []
|
||||
if len(gpu) < expected:
|
||||
faults.append(issue('gpu_missing', f'预期 {expected} 块，当前管理接口可用 {len(gpu)} 块。', 'ERROR'))
|
||||
if gpu_error:
|
||||
faults.append(issue('gpu_driver', gpu_error, 'ERROR'))
|
||||
missing_uuid = set(cfg.get('expected_gpu_uuids', [])) - {x['uuid'] for x in gpu}
|
||||
if missing_uuid and not gpu_error and len(gpu) >= expected:
|
||||
faults.append(issue('gpu_identity', '当前显卡 UUID 与预期配置不同。', 'WARN'))
|
||||
for item in gpu:
|
||||
if item.get('temperature_c') is not None and item['temperature_c'] >= cfg.get('gpu_warn_c', 85):
|
||||
faults.append(issue('gpu_temp', f'{item["temperature_c"]}°C，告警线 {cfg.get("gpu_warn_c",85)}°C', 'WARN', item['pci']))
|
||||
if cpu_temp is not None and cpu_temp >= 85:
|
||||
faults.append(issue('sensor_temp', f'CPU 最高温度 {cpu_temp}°C，网页关注线 85°C。'))
|
||||
if ram_available / ram_total < .05:
|
||||
faults.append(issue('memory_pressure', f'剩余 {ram_available/1073741824:.1f} GiB。'))
|
||||
hw_age = now - (epoch(hw.get('checked_at')) or 0)
|
||||
if hw_age > 180 or hw_age < -60:
|
||||
faults.append(issue('journal', '硬件日志采集状态超过 3 分钟未更新，或系统时间异常。'))
|
||||
else:
|
||||
for value in hw.get('active', {}).values():
|
||||
matched = next((x for x in faults if x['category'] == value.get('category')), None)
|
||||
if matched:
|
||||
if value.get('severity') == 'ERROR':
|
||||
matched['severity'] = 'ERROR'
|
||||
matched['detail'] += ' ' + value.get('detail','')
|
||||
continue
|
||||
faults.append(issue(value.get('category', 'monitor'), value.get('detail', ''), value.get('severity', 'WARN')))
|
||||
disk_usage = os.statvfs('/var/lib')
|
||||
used = (disk_usage.f_blocks - disk_usage.f_bfree) * disk_usage.f_frsize
|
||||
capacity = disk_usage.f_blocks * disk_usage.f_frsize
|
||||
cpu_name = next((line.split(':', 1)[1].strip() for line in read('/proc/cpuinfo').splitlines()
|
||||
if line.startswith('model name')), 'CPU')
|
||||
cpu_sockets = len(set(line.split(':',1)[1].strip() for line in read('/proc/cpuinfo').splitlines()
|
||||
if line.startswith('physical id')))
|
||||
latest = {'ts': now, 'checked_at': iso(now), 'host': socket.gethostname(), 'address': '192.168.1.37',
|
||||
'boot_id': read('/proc/sys/kernel/random/boot_id'), 'monotonic_s': mono,
|
||||
'uptime_s': float(read('/proc/uptime').split()[0]), 'cpu': cpu, 'cpu_name': cpu_name,
|
||||
'cpu_threads': os.cpu_count(), 'cpu_sockets': cpu_sockets, 'cpu_temp': cpu_temp, 'load': list(os.getloadavg()),
|
||||
'memory': (1 - ram_available / ram_total) * 100, 'memory_total_gib': ram_total / 1073741824,
|
||||
'memory_available_gib': ram_available / 1073741824, 'gpus': gpu,
|
||||
'expected_gpu_count': expected, 'gpu_warn_c': cfg.get('gpu_warn_c',85),
|
||||
'gpu_error': gpu_error, 'disk_temp': disk_temp, 'disks': disks, 'rates': rates,
|
||||
'device_map': device_map(gpu),
|
||||
'disk_used_gib': used / 1073741824, 'disk_total_gib': capacity / 1073741824,
|
||||
'disk_usage_pct': used / capacity * 100, 'smart': hw.get('smart', {}),
|
||||
'hardware_checked_at': hw.get('checked_at'), 'hardware_stale': hw_age > 180 or hw_age < -60,
|
||||
'hardware_active': hw.get('active', {}) if 0 <= hw_age <= 180 else {},
|
||||
'issues': faults, 'sensors': temps, 'interval': INTERVAL}
|
||||
return latest, {'cpu_total': total, 'cpu_idle': idle, 'mono': mono, 'counters': count}
|
||||
|
||||
def sampler():
|
||||
previous = None
|
||||
last_clean = 0
|
||||
while not STOP.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
current, previous = sample(previous)
|
||||
with connect() as db:
|
||||
import_logs(db)
|
||||
db.execute('INSERT OR REPLACE INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)',
|
||||
(int(current['ts']), current['cpu'], current['memory'], current['cpu_temp'],
|
||||
current['rates']['disk_read'], current['rates']['disk_write'], current['rates']['net_rx'],
|
||||
current['rates']['net_tx'], json.dumps(current['gpus']), 'live'))
|
||||
db.execute('INSERT OR REPLACE INTO boot_clock VALUES(?,?,?)',
|
||||
(current['boot_id'].replace('-', ''), current['ts'] - current['monotonic_s'], current['monotonic_s']))
|
||||
if time.time() - last_clean >= 3600:
|
||||
db.execute('DELETE FROM metrics WHERE ts < ?', (time.time() - KEEP_DAYS * 86400,))
|
||||
db.execute('DELETE FROM events WHERE observed < ?', (time.time() - 90 * 86400,))
|
||||
last_clean = time.time()
|
||||
current['history_started_at'] = db.execute("SELECT MIN(ts) FROM metrics WHERE source='live'").fetchone()[0]
|
||||
recent = db.execute('SELECT raw FROM events WHERE observed>? AND category IN (?,?,?)',
|
||||
(time.time()-180, 'pcie_fatal', 'pcie_correctable', 'power')).fetchall()
|
||||
for record in recent:
|
||||
row = json.loads(record['raw'])
|
||||
source_boot = row.get('boot_id', '').replace('-', '')
|
||||
source_mono = row.get('last_monotonic_us')
|
||||
if source_boot != current['boot_id'].replace('-', '') or source_mono is None:
|
||||
continue
|
||||
age = current['monotonic_s'] - int(source_mono) / 1e6
|
||||
row_component = event_component(row,current)[0]
|
||||
already_present = any(x['category'] == row['category'] and
|
||||
event_component({'boot_id':current['boot_id'],'device':x.get('device',''),'category':x['category']},current)[0] == row_component
|
||||
for x in current['issues'])
|
||||
if 0 <= age <= 120 and not already_present:
|
||||
severity = 'WARN' if row['category'] == 'pcie_correctable' else 'ERROR'
|
||||
report = issue(row['category'], '最近两分钟仍有新的通信报告。' if row['category']=='pcie_correctable' else '最近两分钟仍有新报错。', severity, row.get('device',''))
|
||||
report['component_label'] = event_component(row,current)[1]
|
||||
current['issues'].append(report)
|
||||
with LOCK:
|
||||
LATEST.clear(); LATEST.update(current)
|
||||
except Exception:
|
||||
logging.exception('Sample failed')
|
||||
with LOCK:
|
||||
LATEST['collector_error'] = '网页采集失败，请查看服务日志。'
|
||||
STOP.wait(max(.2, INTERVAL - (time.monotonic() - started)))
|
||||
|
||||
def snapshot():
|
||||
with LOCK:
|
||||
data = json.loads(json.dumps(LATEST))
|
||||
data['server_ts'] = time.time()
|
||||
data['stale'] = not data.get('ts') or time.time() - data.get('ts', 0) > 35
|
||||
return data
|
||||
|
||||
def history(range_name):
|
||||
seconds = RANGES.get(range_name, 86400)
|
||||
end = int(time.time()); start = end - seconds
|
||||
step = max(INTERVAL, math.ceil(seconds / 360))
|
||||
with connect() as db:
|
||||
rows = db.execute('''SELECT CAST((ts-?)/? AS INTEGER) AS b,AVG(cpu) cpu,AVG(memory) memory,
|
||||
AVG(cpu_temp) cpu_temp,AVG(disk_read) disk_read,AVG(disk_write) disk_write,
|
||||
AVG(net_rx) net_rx,AVG(net_tx) net_tx,SUM(source='live') live
|
||||
FROM metrics WHERE ts>=? AND ts<=? GROUP BY b ORDER BY b''', (start,step,start,end)).fetchall()
|
||||
gpu_rows = db.execute('''SELECT CAST((m.ts-?)/? AS INTEGER) AS b,
|
||||
json_extract(j.value,'$.uuid') uuid, MAX(json_extract(j.value,'$.name')) name,
|
||||
AVG(json_extract(j.value,'$.utilization')) utilization,
|
||||
AVG(json_extract(j.value,'$.temperature_c')) temperature_c,
|
||||
AVG(json_extract(j.value,'$.power_w')) power_w,
|
||||
AVG(json_extract(j.value,'$.memory_used_mib')) memory_used_mib
|
||||
FROM metrics m,json_each(m.gpus) j WHERE m.ts>=? AND m.ts<=?
|
||||
GROUP BY b,uuid ORDER BY b''', (start,step,start,end)).fetchall()
|
||||
points = {row['b']: {'ts': start + row['b'] * step, **{k: row[k] for k in row.keys() if k != 'b'}, 'gpus': {}} for row in rows}
|
||||
devices = {}
|
||||
for row in gpu_rows:
|
||||
if row['b'] in points and row['uuid']:
|
||||
points[row['b']]['gpus'][row['uuid']] = {key: row[key] for key in ('utilization','temperature_c','power_w','memory_used_mib')}
|
||||
devices[row['uuid']] = row['name']
|
||||
return {'start': start, 'end': end, 'step': step, 'range': range_name,
|
||||
'points': list(points.values()), 'devices': devices}
|
||||
|
||||
def source_times(row, anchors):
|
||||
observed = epoch(row.get('observed_at')) or time.time()
|
||||
first = epoch(row.get('first_at')) or observed
|
||||
last = epoch(row.get('last_at')) or first
|
||||
adjusted = first > observed + 60 or last < first
|
||||
note = ''
|
||||
if adjusted:
|
||||
boot = row.get('boot_id', '').replace('-', '')
|
||||
if boot in anchors and row.get('first_monotonic_us') and row.get('last_monotonic_us'):
|
||||
first = anchors[boot] + int(row['first_monotonic_us']) / 1e6
|
||||
last = anchors[boot] + int(row['last_monotonic_us']) / 1e6
|
||||
note = '原始系统时间曾异常；发生时间按同次启动的校时记录估算，原始时间保留如下。'
|
||||
else:
|
||||
first = last = observed
|
||||
note = '原始系统时间不可靠，当前按监控读到记录的时间展示，无法确认精确发生时刻。'
|
||||
return first, last, note
|
||||
|
||||
def merge_incidents(items):
|
||||
"""Group reports, never claim a log line equals an independent hardware failure."""
|
||||
buckets = {}
|
||||
output = []
|
||||
for item in sorted(items,key=lambda x:x['first_ts']):
|
||||
if item['category'] in ('boot','clock','journal','monitor') or item['raw'].get('event') == 'RECOVERED':
|
||||
item.update(record_count=1, signal_reports=0, raw_examples=[item['raw']])
|
||||
output.append(item)
|
||||
continue
|
||||
key = (item['category'],item['raw'].get('boot_id','').replace('-',''),item['component_id'])
|
||||
group = buckets.get(key)
|
||||
if group is None or item['first_ts'] > group['last_ts'] + 300:
|
||||
group = dict(item)
|
||||
group.update(record_count=0, count=0, signal_reports=0, raw_examples=[],
|
||||
observed_first_at=item['observed_at'], devices=[])
|
||||
group['id'] = 'incident-' + hashlib.sha256(json.dumps([*key,item['first_ts']]).encode()).hexdigest()[:24]
|
||||
buckets[key] = group
|
||||
output.append(group)
|
||||
group['first_ts'] = min(group['first_ts'],item['first_ts'])
|
||||
group['first_at'] = iso(group['first_ts'])
|
||||
if item['last_ts'] >= group['last_ts']:
|
||||
for field in ('last_ts','last_at','raw'):
|
||||
group[field] = item[field]
|
||||
group['record_count'] += 1
|
||||
group['count'] += item['count']
|
||||
is_report = item['category'] != 'pcie_correctable' or 'PCIe Bus Error:' in item['raw'].get('detail','')
|
||||
group['signal_reports'] += item['count'] if is_report else 0
|
||||
group['observed_first_at'] = min(group['observed_first_at'],item['observed_at'])
|
||||
group['observed_at'] = max(group['observed_at'],item['observed_at'])
|
||||
if item['device'] and item['device'] not in group['devices']:
|
||||
group['devices'].append(item['device'])
|
||||
if len(group['raw_examples']) < 8 and not any(x.get('detail') == item['raw'].get('detail') for x in group['raw_examples']):
|
||||
group['raw_examples'].append(item['raw'])
|
||||
priority = {'ERROR':3,'WARN':2,'INFO':1}
|
||||
if priority.get(item['severity'],0)>priority.get(group['severity'],0):
|
||||
group['severity']=item['severity']
|
||||
state_priority = {'unknown':6,'active':5,'related':4,'history':2,'recovered':1,'info':0}
|
||||
if state_priority.get(item['state'],0)>state_priority.get(group['state'],0):
|
||||
group['state']=item['state'];group['state_label']=item['state_label']
|
||||
if item['time_note']:
|
||||
group['time_note']=item['time_note']
|
||||
return output
|
||||
|
||||
def explain_item(item,current):
|
||||
online = len(current.get('gpus',[]))
|
||||
expected = current.get('expected_gpu_count',2)
|
||||
usable = online >= expected and not current.get('stale')
|
||||
if item['category']=='pcie_correctable':
|
||||
known_gpu=item.get('component_id','').lower().startswith('gpu')
|
||||
device_word='显卡' if known_gpu else '设备'
|
||||
item['title']=device_word+'连接反复重试' if item.get('signal_reports',0)>1 else device_word+'连接出现一次重试'
|
||||
if not item.get('signal_reports'):
|
||||
item['title']='设备连接出现重试记录'
|
||||
item['meaning']='主板和'+device_word+'传数据时发现通信错误，系统会自动纠正或重发。这条记录不能证明'+device_word+'已经损坏。'
|
||||
item['impact']=f'当前 {online}/{expected} 块显卡可被系统正常读取，暂未看到掉卡。' if usable else '当前未能确认全部显卡正常，请先看顶部的实时状态。'
|
||||
item['urgency']='可以继续使用，建议安排空闲时检查连接' if usable else '先确认显卡是否在线，再安排维护'
|
||||
item['next_steps']=['现在观察计算任务有没有报错、变慢或中断。',
|
||||
'若通信重试持续出现，在业务空闲时正常关机并彻底断电。',
|
||||
'检查显卡供电线两端和插槽；仍有问题时由维护人员换线、换槽对照测试。']
|
||||
if item['state']=='active':
|
||||
item['state_label']='仍在重试 · 显卡可用' if usable and known_gpu else '仍有连接重试 · 需要关注'
|
||||
elif item['category']=='power':
|
||||
item['meaning']='系统当时检测不到这张显卡需要的外接供电，所以显卡可能无法启动。线接触不良、线材、电源或卡上检测电路都可能引起这一现象。'
|
||||
item['impact']=f'当前 {online}/{expected} 块显卡可用；这条是以前的供电异常记录。' if usable else '当前显卡状态不完整，请检查顶部告警。'
|
||||
item['urgency']='历史记录，当前先观察是否再次出现' if usable else '需要检查辅助供电'
|
||||
item['next_steps']=[item['action']]
|
||||
else:
|
||||
item['impact']='最近采样未再发现该问题。' if item['state']=='history' else item['state_label']
|
||||
item['urgency']='查看说明，按建议处理'
|
||||
item['next_steps']=[item['action']]
|
||||
if item['state']=='history':
|
||||
item['state_label']='以前的记录 · 当前未再发现'
|
||||
if item['state']=='unknown':
|
||||
item['impact']='实时数据已过期，暂时无法判断现在是否正常。'
|
||||
item['urgency']='先恢复监控连接，再判断是否需要维修'
|
||||
item['current_checked_at']=current.get('checked_at')
|
||||
return item
|
||||
|
||||
def events(params):
|
||||
now = time.time()
|
||||
days = min(90, max(1, int(params.get('days', ['7'])[0])))
|
||||
start = epoch(params.get('from', [''])[0]) or now - days * 86400
|
||||
end = epoch(params.get('to', [''])[0]) or now + 60
|
||||
if len(params.get('to', [''])[0]) == 10:
|
||||
end += 86399
|
||||
category = params.get('category', [''])[0]
|
||||
severity = params.get('severity', [''])[0]
|
||||
query = params.get('q', [''])[0][:120].lower()
|
||||
page = max(0, min(10000, int(params.get('page', ['0'])[0])))
|
||||
where = ['((occurred>=? AND occurred<=?) OR (observed>=? AND observed<=?))']
|
||||
values = [start,end,start,end]
|
||||
if category:
|
||||
where.append('category=?'); values.append(category)
|
||||
if severity=='problem':
|
||||
where.append("severity IN ('ERROR','WARN')")
|
||||
elif severity:
|
||||
where.append('severity=?'); values.append(severity)
|
||||
with connect() as db:
|
||||
anchors = {x['boot'].replace('-', ''): x['anchor'] for x in db.execute('SELECT * FROM boot_clock')}
|
||||
records = db.execute('SELECT * FROM events WHERE ' + ' AND '.join(where) + ' ORDER BY observed DESC LIMIT 15000', values).fetchall()
|
||||
current = snapshot()
|
||||
items = []
|
||||
for record in records:
|
||||
row = json.loads(record['raw'])
|
||||
first,last,note = source_times(row,anchors)
|
||||
if max(first,last) < start or min(first,last) > end:
|
||||
continue
|
||||
title,meaning,action = CATEGORY.get(row.get('category'), CATEGORY['monitor'])
|
||||
ongoing = False
|
||||
related = False
|
||||
component_id,component_label,address = event_component(row,current)
|
||||
for problem in current.get('issues', []):
|
||||
if problem['category'] == row.get('category'):
|
||||
other = event_component({'boot_id':current.get('boot_id',''),'device':problem.get('device',''),'category':problem['category']},current)[0]
|
||||
related = component_id==other or not problem.get('device')
|
||||
if related:
|
||||
break
|
||||
if row.get('key') and row['key'] in current.get('hardware_active', {}):
|
||||
ongoing = row.get('event') != 'RECOVERED'
|
||||
if not current.get('stale') and related and row.get('source') == 'kernel':
|
||||
mono = row.get('last_monotonic_us')
|
||||
same_boot = row.get('boot_id', '').replace('-','') == current.get('boot_id','').replace('-','')
|
||||
if same_boot and mono:
|
||||
ongoing = 0 <= current.get('monotonic_s',0) - int(mono)/1e6 <= 120
|
||||
if current.get('stale'):
|
||||
state,label = 'unknown','当前状态未知'
|
||||
elif row.get('event') == 'RECOVERED':
|
||||
state,label = 'recovered','已记录恢复'
|
||||
elif ongoing:
|
||||
state,label = 'active','当前仍有同类异常'
|
||||
elif related:
|
||||
state,label = 'related','历史记录 · 当前另有同类告警'
|
||||
else:
|
||||
state,label = 'history','历史记录 · 当前未复现'
|
||||
if row.get('severity') == 'INFO' and row.get('event') != 'RECOVERED':
|
||||
state,label = 'info','运行记录'
|
||||
items.append({'id': record['id'], 'title': title, 'meaning': meaning, 'action': action,
|
||||
'severity': row.get('severity','INFO'), 'category': row.get('category'),
|
||||
'device': row.get('device',''), 'first_ts': first,'last_ts': last,
|
||||
'first_at': iso(first),'last_at': iso(last),'observed_at': row.get('observed_at'),
|
||||
'count': row.get('count',1),'time_note': note,'state': state,'state_label': label,'raw': row,
|
||||
'component_id':component_id,'component_label':component_label})
|
||||
raw_total=len(items)
|
||||
view=params.get('view',['issues'])[0]
|
||||
if view!='raw':
|
||||
items=merge_incidents(items)
|
||||
else:
|
||||
for item in items:
|
||||
item.update(record_count=1,signal_reports=item['count'] if item['category']!='pcie_correctable' or 'PCIe Bus Error:' in item['raw'].get('detail','') else 0,
|
||||
raw_examples=[item['raw']])
|
||||
for item in items:
|
||||
explain_item(item,current)
|
||||
if query:
|
||||
items=[item for item in items if query in json.dumps(item,ensure_ascii=False).lower()]
|
||||
raw_total=sum(item.get('record_count',1) for item in items)
|
||||
items.sort(key=lambda x: x['last_ts'],reverse=True)
|
||||
offset = page * 30
|
||||
return {'items': items[offset:offset+30], 'total': len(items),'raw_total':raw_total,'view':view,'page': page,
|
||||
'more': offset+30 < len(items), 'categories': [{'id':key,'name':value[0]} for key,value in CATEGORY.items()]}
|
||||
|
||||
def task_rpc(action,payload=None,token='',csrf='',client=''):
|
||||
request={'action':action,'payload':payload or {},'token':token,'csrf':csrf,'client':client}
|
||||
try:
|
||||
with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as conn:
|
||||
conn.settimeout(25);conn.connect(TASK_SOCKET)
|
||||
conn.sendall(json.dumps(request,ensure_ascii=False).encode()+b'\n')
|
||||
with conn.makefile('rb') as stream:
|
||||
raw=stream.readline(1024*1024)
|
||||
return json.loads(raw)
|
||||
except (OSError,ValueError) as exc:
|
||||
logging.warning('Task broker unavailable: %s',type(exc).__name__)
|
||||
return {'ok':False,'status':503,'error':'定时任务管理服务暂不可用；设备监控仍可正常查看。'}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = 'HardwareDashboard/1.0'
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
if len(args)>1 and str(args[1]) not in ('200','304'):
|
||||
logging.info('%s %s',self.client_address[0], fmt % args)
|
||||
|
||||
def send(self, body, content_type, status=200, attachment=None,extra_headers=None):
|
||||
if isinstance(body,str):
|
||||
body = body.encode('utf-8')
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type',content_type)
|
||||
self.send_header('Content-Length',str(len(body)))
|
||||
self.send_header('Cache-Control','no-store')
|
||||
self.send_header('X-Content-Type-Options','nosniff')
|
||||
self.send_header('X-Frame-Options','DENY')
|
||||
self.send_header('Content-Security-Policy',"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'")
|
||||
if attachment:
|
||||
self.send_header('Content-Disposition',f'attachment; filename="{attachment}"')
|
||||
for key,value in (extra_headers or {}).items():self.send_header(key,value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
parsed = urlparse(self.path)
|
||||
params = parse_qs(parsed.query)
|
||||
try:
|
||||
if parsed.path in ('/api/tasks/auth','/api/tasks','/api/tasks/item','/api/tasks/runs','/api/tasks/runlog','/api/tasks/history'):
|
||||
action={'/api/tasks/auth':'auth.status','/api/tasks':'tasks.list','/api/tasks/item':'tasks.get',
|
||||
'/api/tasks/runs':'tasks.runs','/api/tasks/runlog':'tasks.runlog','/api/tasks/history':'tasks.history'}[parsed.path]
|
||||
payload={'id':params.get('id',[''])[0],'run_id':params.get('run_id',[''])[0]}
|
||||
return self.task_response(action,payload)
|
||||
elif parsed.path == '/api/status':
|
||||
data = snapshot()
|
||||
elif parsed.path == '/api/history':
|
||||
data = history(params.get('range',['24h'])[0])
|
||||
elif parsed.path == '/api/events':
|
||||
data = events(params)
|
||||
elif parsed.path == '/api/events.csv':
|
||||
output = io.StringIO(); writer = csv.writer(output)
|
||||
writer.writerow(['问题开始时间','最近发生时间','监控最近发现时间','原级别','问题','当前判断','设备','合并技术记录数','日志总行数','报错报告数（非独立故障次数）','维护建议','时间说明','原始日志样例'])
|
||||
for row in events(params)['items']:
|
||||
values = [row['first_at'],row['last_at'],row['observed_at'],row['severity'],row['title'],row['state_label'],row.get('component_label',row['device']),row.get('record_count',1),row['count'],row.get('signal_reports',0),row['action'],row['time_note'],row['raw'].get('detail','')]
|
||||
writer.writerow(["'"+v if isinstance(v,str) and v.startswith(('=','+','-','@')) else v for v in values])
|
||||
return self.send('\ufeff'+output.getvalue(),'text/csv; charset=utf-8',attachment='hardware-events.csv')
|
||||
elif parsed.path in ('/','/app.js','/tasks.js','/style.css','/favicon.svg'):
|
||||
filename = 'index.html' if parsed.path=='/' else parsed.path[1:]
|
||||
mime = {'index.html':'text/html; charset=utf-8','app.js':'text/javascript; charset=utf-8','tasks.js':'text/javascript; charset=utf-8','style.css':'text/css; charset=utf-8','favicon.svg':'image/svg+xml'}
|
||||
return self.send((ROOT/'static'/filename).read_bytes(),mime[filename])
|
||||
else:
|
||||
return self.send('Not found','text/plain',404)
|
||||
self.send(json.dumps(data,ensure_ascii=False,allow_nan=False),'application/json; charset=utf-8')
|
||||
except (ValueError,TypeError):
|
||||
self.send('{"error":"参数格式不正确"}','application/json; charset=utf-8',400)
|
||||
except (BrokenPipeError,ConnectionResetError):
|
||||
pass
|
||||
except Exception:
|
||||
logging.exception('Request failed')
|
||||
self.send('{"error":"读取失败，请稍后重试"}','application/json; charset=utf-8',500)
|
||||
|
||||
def task_response(self,action,payload):
|
||||
token=''
|
||||
try:
|
||||
cookie=SimpleCookie();cookie.load(self.headers.get('Cookie',''))
|
||||
if 'hw_task_session' in cookie:token=cookie['hw_task_session'].value
|
||||
except CookieError:pass
|
||||
result=task_rpc(action,payload,token,self.headers.get('X-Task-CSRF',''),self.client_address[0])
|
||||
headers={}
|
||||
if result.get('ok'):
|
||||
data=result['data']
|
||||
if action=='auth.login':
|
||||
secret=data.pop('token')
|
||||
headers['Set-Cookie']=f'hw_task_session={secret}; Path=/api/tasks; Max-Age={data["max_age"]}; HttpOnly; SameSite=Strict'
|
||||
elif action in ('auth.logout','auth.password'):
|
||||
headers['Set-Cookie']='hw_task_session=; Path=/api/tasks; Max-Age=0; HttpOnly; SameSite=Strict'
|
||||
self.send(json.dumps(data,ensure_ascii=False),'application/json; charset=utf-8',extra_headers=headers)
|
||||
else:
|
||||
self.send(json.dumps({'error':result.get('error','任务操作失败。')},ensure_ascii=False),
|
||||
'application/json; charset=utf-8',result.get('status',500))
|
||||
|
||||
def do_POST(self):
|
||||
parsed=urlparse(self.path)
|
||||
actions={'/api/tasks/login':'auth.login','/api/tasks/logout':'auth.logout','/api/tasks/password':'auth.password',
|
||||
'/api/tasks/preview':'tasks.preview','/api/tasks/save':'tasks.save','/api/tasks/enable':'tasks.enable',
|
||||
'/api/tasks/delete':'tasks.delete','/api/tasks/run':'tasks.run','/api/tasks/stop':'tasks.stop'}
|
||||
if parsed.path not in actions:return self.send('Not found','text/plain',404)
|
||||
origin=self.headers.get('Origin','')
|
||||
host=self.headers.get('Host','')
|
||||
if urlparse(origin).netloc!=host or urlparse(origin).scheme not in ('http','https') or urlparse(origin).hostname not in ('192.168.1.37','localhost','127.0.0.1'):
|
||||
return self.send('{"error":"请从本机监控页面发起操作。"}','application/json; charset=utf-8',403)
|
||||
try:
|
||||
length=int(self.headers.get('Content-Length','0'))
|
||||
if not 0<length<=131072:raise ValueError()
|
||||
if not self.headers.get('Content-Type','').startswith('application/json'):raise ValueError()
|
||||
payload=json.loads(self.rfile.read(length))
|
||||
if not isinstance(payload,dict):raise ValueError()
|
||||
except (ValueError,UnicodeDecodeError):
|
||||
return self.send('{"error":"请求格式不正确或内容超过128KiB。"}','application/json; charset=utf-8',400)
|
||||
self.task_response(actions[parsed.path],payload)
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--host',default='0.0.0.0'); parser.add_argument('--port',type=int,default=8090)
|
||||
args=parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')
|
||||
initialize()
|
||||
worker=threading.Thread(target=sampler,daemon=True); worker.start()
|
||||
server=ThreadingHTTPServer((args.host,args.port),Handler)
|
||||
server.daemon_threads=True
|
||||
logging.info('Hardware dashboard listening on %s:%s',args.host,args.port)
|
||||
try:
|
||||
server.serve_forever(poll_interval=.5)
|
||||
finally:
|
||||
STOP.set(); server.server_close(); worker.join(timeout=10)
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
171
static/app.js
Normal file
171
static/app.js
Normal file
@ -0,0 +1,171 @@
|
||||
'use strict';
|
||||
const $ = id => document.getElementById(id);
|
||||
const colors = {green:'#b3ee80',blue:'#6bbcff',purple:'#b4a1f4',amber:'#f5be6a',red:'#ff8c86'};
|
||||
const initialSection=location.hash.slice(1);
|
||||
let latest = null, selectedRange = '24h', historyData = null, eventPage = 0, eventParams = new URLSearchParams({days:'7',severity:'problem',view:'issues'});
|
||||
let eventData = null, statusBusy = false, historySequence = 0, eventsBusy = false, eventSequence = 0, eventController = null;
|
||||
function el(tag, cls, text) { const e=document.createElement(tag);if(cls)e.className=cls;if(text!==undefined)e.textContent=text;return e; }
|
||||
function icon(name) {const svg=document.createElementNS('http://www.w3.org/2000/svg','svg');const use=document.createElementNS(svg.namespaceURI,'use');use.setAttribute('href','#'+name);svg.append(use);return svg;}
|
||||
function valid(v) {return v!==null&&v!==undefined&&Number.isFinite(Number(v));}
|
||||
function num(v, digits=0) {return valid(v)?Number(v).toFixed(digits):'—';}
|
||||
function when(v, full=true) {if(!v)return '—';const d=typeof v==='number'?new Date(v*1000):new Date(v);if(Number.isNaN(d.getTime()))return '时间未知';return new Intl.DateTimeFormat('zh-CN',{timeZone:'Asia/Shanghai',...(full?{year:'numeric',month:'2-digit',day:'2-digit'}:{}),hour:'2-digit',minute:'2-digit',second:'2-digit',hour12:false}).format(d);}
|
||||
function duration(s){if(!valid(s))return '—';s=Math.max(0,Math.round(s));const d=Math.floor(s/86400),h=Math.floor(s%86400/3600),m=Math.floor(s%3600/60);return `${d?d+' 天 ':''}${h} 小时 ${m} 分钟`;}
|
||||
function toast(text){$('toast').textContent=text;$('toast').hidden=false;clearTimeout(toast.timer);toast.timer=setTimeout(()=>$('toast').hidden=true,4500);}
|
||||
async function api(path){const response=await fetch(path,{signal:AbortSignal.timeout(9000),cache:'no-store'});if(!response.ok)throw new Error('HTTP '+response.status);return response.json();}
|
||||
function meter(id,value){$(id).style.width=Math.min(100,Math.max(0,Number(value)||0))+'%';}
|
||||
function metric(id,value,unit,digits=0){$(id).replaceChildren(document.createTextNode(num(value,digits)),el('small','',unit));}
|
||||
function connectionLost(message){$('connection').replaceChildren(el('i','dot'),document.createTextNode('连接中断'));$('connection').firstChild.style.background=colors.red;$('health-banner').className='health-banner error';$('health-title').textContent='暂时无法确认设备状态';$('health-description').textContent=message||'网页与设备的连接已中断。下面保留的是上次数据，请勿当作实时状态。';$('health-icon').replaceChildren(icon('alert').firstChild);}
|
||||
function renderStatus(data){
|
||||
if(!data.ts){connectionLost('采集器正在启动，暂未取得实时数据，请稍候。');return;}
|
||||
const readingAnchor=captureSectionAnchor();
|
||||
latest=data;
|
||||
$('host-name').textContent=data.host||'设备';$('last-update').textContent=when(data.ts,false);
|
||||
if(data.stale){connectionLost('设备采样超过 35 秒未更新，当前状态未知。');return;}
|
||||
$('connection').replaceChildren(el('i','dot'),document.createTextNode('实时连接 · 10 秒'));
|
||||
const issues=data.issues||[],error=issues.some(x=>x.severity==='ERROR'),warn=issues.length>0;
|
||||
$('health-banner').className='health-banner'+(error?' error':warn?' warning':'');
|
||||
const onlyRetries=warn&&issues.every(x=>x.category==='pcie_correctable')&&data.gpus.length>=data.expected_gpu_count;
|
||||
$('health-title').textContent=error?'设备存在故障，需要处理':onlyRetries?'显卡还能用，但连接需要检查':warn?'设备需要关注':'设备运行正常';
|
||||
$('health-description').textContent=onlyRetries?'显卡当前在线，主板与显卡之间仍有通信重试。可以继续观察，安排空闲时检查连接。':warn?`${issues.length} 项需要关注，具体原因和处理建议见下方。`:`${data.gpus.length}/${data.expected_gpu_count} 块计算显卡在线 · 当前采样未发现异常`;
|
||||
$('health-icon').replaceChildren(icon(warn?'alert':'check').firstChild);
|
||||
const list=$('current-issues');list.hidden=!issues.length;list.replaceChildren();
|
||||
issues.forEach(item=>{const card=el('article','issue'+(item.severity==='ERROR'?' error':''));const device=item.component_label?item.component_label+'：':'';card.append(el('strong','',device+item.title),el('p','',item.meaning+' '+item.detail),el('small','', '怎么处理：'+item.action));list.append(card);});
|
||||
$('nav-alert').hidden=!issues.length;$('nav-alert').textContent=String(issues.length);
|
||||
metric('cpu-value',data.cpu,'%',1);meter('cpu-bar',data.cpu);$('cpu-threads').textContent=(data.cpu_sockets?data.cpu_sockets+' 路 CPU · ':'')+(data.cpu_threads||'—')+' 线程';$('cpu-temp').textContent=num(data.cpu_temp)+' °C';$('cpu-temp').title=data.cpu_name;
|
||||
metric('memory-value',data.memory,'%');meter('memory-bar',data.memory);$('memory-used').textContent=`${num(data.memory_total_gib-data.memory_available_gib,1)} / ${num(data.memory_total_gib,1)} GiB`;$('memory-free').textContent=num(data.memory_available_gib,1)+' GiB 可用';
|
||||
metric('gpu-count',data.gpus.length,`/ ${data.expected_gpu_count} 在线`);$('gpu-online-text').replaceChildren(el('i','dot'),document.createTextNode(data.gpus.length>=data.expected_gpu_count?'全部预期显卡可用':'显卡数量少于预期'));$('gpu-online-text').firstChild.style.background=data.gpus.length>=data.expected_gpu_count?colors.green:colors.red;$('gpu-type').textContent=data.gpus[0]?.name?.includes('P100')?'Tesla P100':'NVIDIA';
|
||||
const disks=Object.values(data.smart||{}),diskBad=!data.hardware_stale&&disks.some(x=>x.passed===false||x.critical_warning),diskKnown=!data.hardware_stale&&disks.length&&disks.every(x=>x.available&&x.passed===true);
|
||||
$('disk-health').textContent=diskBad?'需要检查':diskKnown?'健康正常':'状态未知';$('disk-health').style.color=diskBad?colors.red:diskKnown?colors.green:colors.amber;
|
||||
$('disk-health-detail').textContent=diskKnown?'SMART 检查通过':diskBad?'请展开磁盘相关日志':'尚未取得完整 SMART 数据';$('disk-space').textContent=`已用 ${num(data.disk_used_gib,0)} / ${num(data.disk_total_gib,0)} GiB`;$('disk-temp').textContent=num(data.disk_temp)+' °C';
|
||||
renderGPUs(data);
|
||||
$('uptime').textContent=duration(data.uptime_s);['disk-read','disk-write','net-rx','net-tx'].forEach(id=>$(id).textContent=num(data.rates?.[id.replace('-','_')],2)+' MiB/s');
|
||||
const start=data.history_started_at?when(data.history_started_at):'本次部署';
|
||||
$('history-note').textContent=`CPU、内存和显卡使用率从 ${start} 开始记录。已有的显卡温度、功耗历史已补录（旧记录约 15 分钟一条）；空白表示该时段没有采样。曲线为时段均值。`;
|
||||
restoreSectionAnchor(readingAnchor);queueNavigation();
|
||||
}
|
||||
function renderGPUs(data){
|
||||
const holder=$('gpu-grid');holder.replaceChildren();
|
||||
const gpus=[...data.gpus].sort((a,b)=>a.uuid.localeCompare(b.uuid));
|
||||
gpus.forEach((gpu,index)=>{
|
||||
const card=el('article','gpu-card');card.style.setProperty('--accent',index%2?colors.amber:colors.blue);
|
||||
const head=el('div','gpu-head');const state=el('span','gpu-state');state.append(el('i','dot'),document.createTextNode('在线'));head.append(el('span','gpu-label','GPU '+String(index+1).padStart(2,'0')),el('h3','',gpu.name?.replace('Tesla ','')||'NVIDIA GPU'),state);
|
||||
const content=el('div','gpu-content'),util=el('div','gpu-util');util.append(document.createTextNode(num(gpu.utilization)),el('small','','%'),el('span','',valid(gpu.utilization)&&gpu.utilization>5?'正在计算':'空闲 / 低负载'));
|
||||
const numbers=el('div','gpu-numbers');[['温度',gpu.temperature_c,'°C'],['功耗',gpu.power_w,'W']].forEach(([label,value,unit])=>{const cell=el('div','gpu-number'),strong=el('strong','',num(value));strong.append(el('small','',unit));if(unit==='°C'&&value>=data.gpu_warn_c)strong.style.color=colors.red;cell.append(el('span','',label),strong);numbers.append(cell);});content.append(util,numbers);
|
||||
const vram=el('div','gpu-vram');vram.append(el('span','','显存占用'),el('strong','',`${num(valid(gpu.memory_used_mib)?gpu.memory_used_mib/1024:null,1)} / ${num(valid(gpu.memory_total_mib)?gpu.memory_total_mib/1024:null,0)} GiB`));
|
||||
const bar=el('div','meter'),fill=el('i');fill.style.width=(gpu.memory_total_mib?Math.min(100,gpu.memory_used_mib/gpu.memory_total_mib*100):0)+'%';bar.append(fill);
|
||||
const meta=el('div','gpu-meta');meta.append(el('span','',gpu.pci.replace(/^00000000:/,'0000:')),el('span','',gpu.uuid.slice(0,16)+'…'),el('span','',`功耗上限 ${num(gpu.power_limit_w)}W`));meta.title=gpu.uuid;
|
||||
card.append(head,content,vram,bar,meta);holder.append(card);
|
||||
});
|
||||
for(let i=gpus.length;i<data.expected_gpu_count;i++){const card=el('article','gpu-card');card.style.setProperty('--accent',colors.red);card.append(el('h3','','显卡未就绪'),el('p','','当前未读取到全部预期显卡。请查看顶部异常说明和下方故障日志。'));holder.append(card);}
|
||||
}
|
||||
async function loadStatus(){if(statusBusy)return;statusBusy=true;try{renderStatus(await api('/api/status'));}catch(error){connectionLost();}finally{statusBusy=false;}}
|
||||
function svgNode(tag,attributes={},text){const e=document.createElementNS('http://www.w3.org/2000/svg',tag);Object.entries(attributes).forEach(([k,v])=>e.setAttribute(k,v));if(text!==undefined)e.textContent=text;return e;}
|
||||
function chart(id,legendId,series,data,unit,maxValue){
|
||||
const container=$(id),legend=$(legendId);container.replaceChildren();legend.replaceChildren();
|
||||
series.forEach(s=>{const item=el('span','legend-item'),swatch=el('i','legend-swatch');swatch.style.background=s.color;item.append(swatch,document.createTextNode(s.name));legend.append(item);});
|
||||
const width=560,height=190,left=36,right=12,top=10,bottom=27,w=width-left-right,h=height-top-bottom;
|
||||
const svg=svgNode('svg',{viewBox:`0 0 ${width} ${height}`,preserveAspectRatio:'none','aria-hidden':'true'});
|
||||
const values=series.flatMap(s=>data.points.map(p=>s.value(p)).filter(valid));
|
||||
const ceiling=Math.max(maxValue||50,Math.ceil(Math.max(0,...values)/25)*25);
|
||||
const x=t=>left+(t-data.start)/(data.end-data.start)*w,y=v=>top+h-(v/ceiling)*h;
|
||||
for(let i=0;i<=4;i++){const v=ceiling*i/4,yy=y(v);svg.append(svgNode('line',{x1:left,y1:yy,x2:width-right,y2:yy,class:'grid-line'}),svgNode('text',{x:left-8,y:yy+3,'text-anchor':'end'},num(v)));}
|
||||
for(let i=0;i<=4;i++){const t=data.start+(data.end-data.start)*i/4;const label=new Intl.DateTimeFormat('zh-CN',{timeZone:'Asia/Shanghai',...(data.end-data.start>86400?{month:'2-digit',day:'2-digit'}:{hour:'2-digit',minute:'2-digit',hour12:false})}).format(new Date(t*1000));svg.append(svgNode('text',{x:x(t),y:height-6,'text-anchor':i===0?'start':i===4?'end':'middle'},label));}
|
||||
series.forEach(s=>{
|
||||
const points=data.points.filter(p=>valid(s.value(p)));let path='',last=null;
|
||||
points.forEach(p=>{const gap=last? p.ts-last.ts:0;const maxGap=p.live||last?.live?Math.max(data.step*3,45):Math.max(data.step*3,1250);path+=(last&&gap<=maxGap?'L':'M')+x(p.ts).toFixed(2)+','+y(s.value(p)).toFixed(2);last=p;});
|
||||
if(path)svg.append(svgNode('path',{d:path,stroke:s.color,class:'series-line'}));
|
||||
if(points.length<4)points.forEach(p=>svg.append(svgNode('circle',{cx:x(p.ts),cy:y(s.value(p)),r:2.4,fill:s.color})));
|
||||
});
|
||||
if(unit==='°C'){svg.append(svgNode('line',{x1:left,x2:width-right,y1:y(85),y2:y(85),stroke:colors.amber,'stroke-width':'.7','stroke-dasharray':'5 6',opacity:'.4'}));}
|
||||
const cursor=svgNode('line',{x1:0,x2:0,y1:top,y2:height-bottom,class:'cursor',visibility:'hidden'}),overlay=svgNode('rect',{x:left,y:top,width:w,height:h,class:'chart-overlay'});svg.append(cursor,overlay);container.append(svg);
|
||||
const tooltip=el('div','chart-tooltip');tooltip.hidden=true;container.append(tooltip);
|
||||
if(!values.length){const empty=el('div','chart-empty');empty.append(el('span','','这个时段还没有性能数据'),el('small','','采集运行后，曲线会自动出现'));container.append(empty);}
|
||||
overlay.addEventListener('pointermove',event=>{
|
||||
const bounds=svg.getBoundingClientRect(),px=Math.max(left,Math.min(width-right,(event.clientX-bounds.left)/bounds.width*width));
|
||||
const t=data.start+(px-left)/w*(data.end-data.start);let closest=null;
|
||||
data.points.forEach(p=>{if(!closest||Math.abs(p.ts-t)<Math.abs(closest.ts-t))closest=p;});
|
||||
cursor.setAttribute('x1',px);cursor.setAttribute('x2',px);cursor.setAttribute('visibility','visible');tooltip.hidden=false;tooltip.replaceChildren();
|
||||
const available=closest&&Math.abs(closest.ts-t)<=Math.max(data.step*1.5,closest.live?20:900);
|
||||
tooltip.append(el('strong','',when(available?closest.ts:t)));
|
||||
if(available){series.forEach(s=>{const row=el('p'),value=s.value(closest);row.append(el('span','',s.name),el('span','',valid(value)?num(value,1)+' '+unit:'未采集'));tooltip.append(row);});}else tooltip.append(el('p','','该时段没有采样'));
|
||||
tooltip.style.left=Math.max(3,Math.min(container.clientWidth-190,(event.clientX-bounds.left)+12))+'px';
|
||||
});overlay.addEventListener('pointerleave',()=>{cursor.setAttribute('visibility','hidden');tooltip.hidden=true;});
|
||||
}
|
||||
function renderHistory(data){
|
||||
historyData=data;
|
||||
const devices=Object.keys(data.devices).sort();const palette=[colors.blue,colors.amber,colors.purple,colors.green];
|
||||
const gpuSeries=key=>devices.map((uuid,i)=>({name:'GPU '+String(i+1).padStart(2,'0'),color:palette[i%palette.length],value:p=>p.gpus?.[uuid]?.[key]}));
|
||||
chart('chart-system','legend-system',[{name:'CPU',color:colors.green,value:p=>p.cpu},{name:'内存',color:colors.purple,value:p=>p.memory}],data,'%',100);
|
||||
chart('chart-gpu','legend-gpu',gpuSeries('utilization'),data,'%',100);
|
||||
chart('chart-temp','legend-temp',[{name:'CPU 最高温度',color:colors.green,value:p=>p.cpu_temp},...gpuSeries('temperature_c')],data,'°C',100);
|
||||
chart('chart-power','legend-power',gpuSeries('power_w'),data,'W',Math.max(250,...data.points.flatMap(p=>Object.values(p.gpus||{}).map(g=>g.power_w||0)))*1.08);
|
||||
$('history-caption').textContent=`${when(data.start)} — ${when(data.end)} · 曲线为时段均值，移入可查看数值`;
|
||||
}
|
||||
async function loadHistory(){const seq=++historySequence;try{const data=await api('/api/history?range='+selectedRange);if(seq===historySequence)renderHistory(data);}catch(error){if(seq===historySequence)$('history-caption').textContent='曲线暂时无法读取，请稍后重试。';}}
|
||||
function addTime(container,label,value){const cell=el('div');cell.append(el('span','',label),el('strong','',value));container.append(cell);}
|
||||
function renderEvents(data,preservePosition=true){
|
||||
eventData=data;const holder=$('event-list');
|
||||
const opened=new Set([...holder.querySelectorAll('details[open]')].map(x=>x.dataset.id));
|
||||
const visible=preservePosition?[...holder.querySelectorAll('.event-row')].find(x=>x.getBoundingClientRect().bottom>70&&x.getBoundingClientRect().top<innerHeight):null;
|
||||
const anchor=visible?{id:visible.dataset.id,top:visible.getBoundingClientRect().top}:null;
|
||||
holder.replaceChildren();
|
||||
if($('category').options.length===1){data.categories.forEach(item=>{const option=el('option','',item.name);option.value=item.id;$('category').append(option);});}
|
||||
$('event-count').textContent=data.view==='raw'?`共 ${data.total} 条技术记录`:`合并后 ${data.total} 段问题记录 · 来自 ${data.raw_total} 条技术记录，不是故障次数`;
|
||||
if(!data.items.length){const empty=el('div','empty-events');empty.append(el('strong','','这个范围内没有匹配记录'),el('span','','可以扩大日期范围，或选择“全部问题”。'));holder.append(empty);}
|
||||
data.items.forEach(item=>{
|
||||
const details=el('details','event-row');details.dataset.id=item.id;details.open=opened.has(item.id);
|
||||
const summary=el('summary'),severity=item.severity==='ERROR'?'error':item.severity==='INFO'?'info':'';
|
||||
const main=el('div','event-main');main.append(el('strong','',item.title),el('p','',item.component_label||'这台电脑'));
|
||||
const time=el('div','event-time',when(item.last_ts));time.append(el('small','',item.time_note?'最近一次 · 时间有估算':'最近一次发生'));
|
||||
const historical=item.state==='history'||item.state==='related';const badge=item.severity==='ERROR'?(historical?'曾有故障':'需处理'):item.severity==='WARN'?(historical?'曾需关注':'需关注'):'记录';
|
||||
summary.append(el('span','event-level '+severity,badge),main,el('span','event-state '+item.state+' '+(item.severity==='WARN'?'warning':''),item.state_label),time,el('span','event-chevron','›'));
|
||||
const detail=el('div','event-detail');
|
||||
[['这是什么问题',item.meaning],['现在还能不能用',item.impact]].forEach(([label,text])=>{const answer=el('div','plain-answer');answer.append(el('strong','',label),el('p','',text));detail.append(answer);});
|
||||
const action=el('div','maintenance');action.append(el('strong','','你现在需要做什么'),el('p','maintenance-priority',item.urgency));const steps=el('ol');(item.next_steps||[item.action]).forEach(text=>steps.append(el('li','',text)));action.append(steps);detail.append(action);
|
||||
const times=el('div','time-grid friendly-times');addTime(times,item.time_note?'这段问题开始（估算）':'这段问题从何时开始',when(item.first_ts));addTime(times,'最近一次发生',when(item.last_ts));addTime(times,'监控最近读到记录',when(item.observed_at));addTime(times,'当前状态检查于',when(item.current_checked_at));detail.append(times);
|
||||
detail.append(el('p','record-summary',`已合并 ${item.record_count||1} 条技术记录，共 ${item.count} 行日志。${item.signal_reports?'其中有 '+item.signal_reports+' 条报错报告。':''}这些数字不是设备损坏次数。`));
|
||||
if(item.time_note)detail.append(el('div','time-warning',item.time_note));
|
||||
const technical=el('details','technical-details');technical.dataset.id=item.id+'-technical';technical.open=opened.has(technical.dataset.id);technical.append(el('summary','','查看原始技术日志（维护人员用）'));
|
||||
const evidence=el('div','technical-body'),rawTimes=el('div','time-grid');
|
||||
if(item.raw.first_monotonic_us)addTime(rawTimes,'原始批次：该次启动后经过',num(Number(item.raw.first_monotonic_us)/1e6,3)+' 秒 — '+num(Number(item.raw.last_monotonic_us)/1e6,3)+' 秒');
|
||||
if(item.raw.boot_id)addTime(rawTimes,'启动 ID',item.raw.boot_id);
|
||||
if(item.raw.first_at)addTime(rawTimes,'原始批次首次时间（未改写）',when(item.raw.first_at));
|
||||
if(item.raw.last_at)addTime(rawTimes,'原始批次末次时间（未改写）',when(item.raw.last_at));
|
||||
if(item.devices?.length)addTime(rawTimes,'涉及的硬件地址',item.devices.join(' / '));
|
||||
evidence.append(rawTimes);(item.raw_examples||[item.raw]).forEach(row=>evidence.append(el('pre','raw-log',row.detail||JSON.stringify(row,null,2))));technical.append(evidence);detail.append(technical);
|
||||
details.append(summary,detail);holder.append(details);
|
||||
});
|
||||
$('page-label').textContent=`第 ${data.page+1} 页 · 每页最多 30 条`;$('previous-page').disabled=data.page===0;$('next-page').disabled=!data.more;
|
||||
if(anchor&&!navigationLock){const replacement=[...holder.querySelectorAll('.event-row')].find(x=>x.dataset.id===anchor.id);if(replacement){const delta=replacement.getBoundingClientRect().top-anchor.top;if(Math.abs(delta)>1)scrollPage(scrollYValue()+delta,false);}}
|
||||
queueNavigation();
|
||||
}
|
||||
async function loadEvents(force=false){if(eventsBusy&&!force)return;if(force)eventController?.abort();const seq=++eventSequence;eventsBusy=true;const controller=new AbortController();eventController=controller;const timer=setTimeout(()=>controller.abort(),10000);try{const params=new URLSearchParams(eventParams);params.set('page',eventPage);const response=await fetch('/api/events?'+params.toString(),{signal:controller.signal,cache:'no-store'});if(!response.ok)throw new Error('HTTP '+response.status);const data=await response.json();if(seq===eventSequence)renderEvents(data,!force);}catch(error){if(seq===eventSequence&&error.name!=='AbortError')$('event-count').textContent='暂时无法读取记录，请点击“查询日志”重试。';}finally{clearTimeout(timer);if(seq===eventSequence)eventsBusy=false;}}
|
||||
function applyEventFilters(){if($('date-from').value&&$('date-to').value&&$('date-from').value>$('date-to').value){toast('开始日期不能晚于结束日期。');return;}eventParams=new URLSearchParams({days:'7'});[['view','event-view'],['category','category'],['severity','severity'],['from','date-from'],['to','date-to'],['q','event-search']].forEach(([key,id])=>{if($(id).value)eventParams.set(key,$(id).value);});eventPage=0;$('event-count').textContent='正在按所选条件查询…';loadEvents(true);}
|
||||
$('event-filter').addEventListener('submit',event=>{event.preventDefault();applyEventFilters();});
|
||||
['event-view','category','severity','date-from','date-to'].forEach(id=>$(id).addEventListener('change',applyEventFilters));let searchTimer;$('event-search').addEventListener('input',()=>{clearTimeout(searchTimer);searchTimer=setTimeout(applyEventFilters,400);});
|
||||
$('previous-page').addEventListener('click',()=>{if(!eventsBusy&&eventPage>0){eventPage--;loadEvents(true);}});$('next-page').addEventListener('click',()=>{if(!eventsBusy&&eventData?.more){eventPage++;loadEvents(true);}});
|
||||
$('export-events').addEventListener('click',()=>{const params=new URLSearchParams(eventParams);params.set('page',eventPage);window.location.assign('/api/events.csv?'+params.toString());});
|
||||
$('ranges').addEventListener('click',event=>{const button=event.target.closest('button[data-range]');if(!button)return;selectedRange=button.dataset.range;document.querySelectorAll('#ranges button').forEach(x=>x.classList.toggle('selected',x===button));$('history-caption').textContent='正在加载所选范围…';loadHistory();});
|
||||
$('fullscreen').addEventListener('click',async()=>{try{if(document.fullscreenElement)await document.exitFullscreen();else await document.body.requestFullscreen();}catch(error){toast('此浏览器不支持网页全屏，可按 F11。');}});
|
||||
const sections=[{id:'overview',name:'设备总览'},{id:'trends',name:'性能趋势'},{id:'events',name:'问题与维护'},{id:'tasks',name:'定时任务'},{id:'guide',name:'使用说明'}];
|
||||
let navigationLock=null,navigationTimer=null,navigationFrame=null,currentSection='overview',userHasScrolled=false;
|
||||
function scrollRoot(){return document.fullscreenElement===document.body?document.body:document.scrollingElement;}
|
||||
function scrollYValue(){return scrollRoot()?.scrollTop??window.scrollY;}
|
||||
function scrollPage(top,smooth=true){const options={top,behavior:smooth&&!matchMedia('(prefers-reduced-motion: reduce)').matches?'smooth':'instant'};if(document.fullscreenElement===document.body)document.body.scrollTo(options);else window.scrollTo(options);}
|
||||
function captureSectionAnchor(){if(navigationLock||scrollYValue()<200||currentSection==='overview')return null;const target=$(currentSection);return target?{id:currentSection,top:target.getBoundingClientRect().top}:null;}
|
||||
function restoreSectionAnchor(anchor){if(!anchor||navigationLock)return;const target=$(anchor.id);if(target){const delta=target.getBoundingClientRect().top-anchor.top;if(Math.abs(delta)>1)scrollPage(scrollYValue()+delta,false);}}
|
||||
function markSection(id,updateHash=true){const index=Math.max(0,sections.findIndex(x=>x.id===id));currentSection=id;document.querySelectorAll('[data-section-link]').forEach(link=>{const active=link.dataset.sectionLink===id;link.classList.toggle('active',active);if(active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current');});$('current-section').textContent=sections[index].name;$('next-section').textContent=index<sections.length-1?'下一项：'+sections[index+1].name+' ↓':'↑ 回到设备总览';if(updateHash&&location.hash!=='#'+id)history.replaceState(null,'','#'+id);}
|
||||
function synchronizeNavigation(){navigationFrame=null;const root=scrollRoot();if(!root)return;const y=scrollYValue(),total=Math.max(1,root.scrollHeight-root.clientHeight);$('reading-progress').style.width=Math.min(100,y/total*100)+'%';$('back-to-top').hidden=y<500;if(document.querySelector('.task-dialog[open]')){if(currentSection!=='tasks')markSection('tasks');return;}if(navigationLock)return;let active='overview';sections.forEach(section=>{if($(section.id).getBoundingClientRect().top<=105)active=section.id;});if(y>0&&y>=total-4)active='guide';if(active!==currentSection)markSection(active);}
|
||||
function queueNavigation(){if(!navigationFrame)navigationFrame=requestAnimationFrame(synchronizeNavigation);}
|
||||
function navigateSection(id,smooth=true){if(!sections.some(s=>s.id===id))return;userHasScrolled=true;clearTimeout(navigationTimer);navigationLock=id;markSection(id);scrollPage(id==='overview'?0:Math.max(0,$(id).getBoundingClientRect().top+scrollYValue()-76),smooth);navigationTimer=setTimeout(()=>{navigationLock=null;queueNavigation();},smooth?1300:50);}
|
||||
document.querySelectorAll('[data-section-link]').forEach(link=>link.addEventListener('click',event=>{event.preventDefault();navigateSection(link.dataset.sectionLink);}));
|
||||
$('back-to-top').addEventListener('click',()=>navigateSection('overview'));$('next-section').addEventListener('click',()=>{const index=sections.findIndex(s=>s.id===currentSection);navigateSection(sections[index+1]?.id||'overview');});
|
||||
window.addEventListener('scroll',()=>{queueNavigation();if(navigationLock){clearTimeout(navigationTimer);navigationTimer=setTimeout(()=>{navigationLock=null;queueNavigation();},180);}},{passive:true,capture:true});
|
||||
function manualScrollIntent(){userHasScrolled=true;if(navigationLock){scrollPage(scrollYValue(),false);clearTimeout(navigationTimer);navigationLock=null;}queueNavigation();}
|
||||
['wheel','touchstart'].forEach(name=>window.addEventListener(name,manualScrollIntent,{passive:true}));
|
||||
window.addEventListener('keydown',event=>{if(['PageDown','PageUp','ArrowDown','ArrowUp','Home','End',' '].includes(event.key))manualScrollIntent();});
|
||||
window.addEventListener('hashchange',()=>navigateSection(location.hash.slice(1),false));window.addEventListener('resize',queueNavigation);document.addEventListener('fullscreenchange',queueNavigation);
|
||||
window.addEventListener('online',()=>{loadStatus();loadHistory();loadEvents();});
|
||||
Promise.allSettled([loadStatus(),loadHistory(),loadEvents()]).then(()=>{if(!userHasScrolled&§ions.some(s=>s.id===initialSection))navigateSection(initialSection,false);else queueNavigation();});
|
||||
setInterval(loadStatus,10000);setInterval(()=>{if(!document.hidden){loadHistory();loadEvents();}},30000);
|
||||
document.addEventListener('visibilitychange',()=>{if(!document.hidden){loadStatus();loadHistory();loadEvents();}});
|
||||
1
static/favicon.svg
Normal file
1
static/favicon.svg
Normal file
@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><rect width="64" height="64" rx="15" fill="#142229"/><path d="M12 35h9l7-17 9 29 7-17h8" fill="none" stroke="#b4f27a" stroke-width="5" stroke-linejoin="round"/></svg>
|
||||
|
After Width: | Height: | Size: 227 B |
81
static/index.html
Normal file
81
static/index.html
Normal file
File diff suppressed because one or more lines are too long
465
static/style.css
Normal file
465
static/style.css
Normal file
@ -0,0 +1,465 @@
|
||||
:root{--bg:#0c1218;--panel:#141d25;--panel2:#19242d;--line:#28333e;--text:#e9eff2;--muted:#8b9ba8;--green:#b3ee80;--blue:#6bbcff;--purple:#b4a1f4;--amber:#f5be6a;--red:#ff8c86;--sans:Inter,"Segoe UI","Microsoft YaHei",system-ui,sans-serif;--mono:"Cascadia Code",Consolas,monospace}
|
||||
*{box-sizing:border-box}
|
||||
html{scroll-behavior:smooth;scroll-padding-top:28px}
|
||||
body{margin:0;background:var(--bg);color:var(--text);font:14px/1.6 var(--sans);letter-spacing:.1px}
|
||||
button,input,select{font:inherit}
|
||||
button,a,input,select,summary{-webkit-tap-highlight-color:transparent}
|
||||
button{cursor:pointer}
|
||||
button:disabled{cursor:default;opacity:.35}
|
||||
button:focus-visible,a:focus-visible,input:focus-visible,select:focus-visible,summary:focus-visible{outline:2px solid var(--green);outline-offset:4px}
|
||||
a{color:inherit;text-decoration:none}
|
||||
svg{width:20px;height:20px;fill:none;stroke:currentColor;stroke-width:1.6;stroke-linecap:round;stroke-linejoin:round;flex-shrink:0}
|
||||
.sprite{position:absolute;width:0;height:0;overflow:hidden}
|
||||
[hidden]{display:none!important}
|
||||
.sidebar{position:fixed;inset:0 auto 0 0;width:220px;background:#10181f;border-right:1px solid var(--line);display:flex;flex-direction:column;padding:32px 20px;z-index:10}
|
||||
.brand{display:flex;gap:12px;align-items:center;font-size:19px;font-weight:650;letter-spacing:.5px}
|
||||
.brand small{display:block;font-size:8px;color:var(--muted);letter-spacing:1.6px;margin-top:1px}
|
||||
.brand-mark{width:39px;height:39px;border-radius:11px;background:var(--green);color:#18271b;display:grid;place-items:center}
|
||||
.brand-mark svg{width:26px;height:26px;stroke-width:2}
|
||||
.nav-caption{font-size:10px;color:#647888;letter-spacing:1.5px;margin:46px 12px 12px}
|
||||
.sidebar nav{display:grid;gap:6px}
|
||||
.sidebar nav a{padding:12px;border-radius:7px;display:flex;gap:11px;align-items:center;color:#95a5b1;font-size:13px}
|
||||
.sidebar nav a:hover{background:#1b2832;color:white}
|
||||
.sidebar nav a.active{background:#24332b;color:var(--green)}
|
||||
.nav-dot{width:5px;height:5px;background:var(--green);border-radius:50%;margin-left:auto}
|
||||
.nav-count{font:10px var(--mono);background:#5a302e;color:var(--red);padding:2px 5px;border-radius:4px;margin-left:auto}
|
||||
.sidebar-node{margin-top:auto;padding:24px 13px 28px;display:grid;gap:7px}
|
||||
.tiny-label{font-size:10px;color:#697d8a;letter-spacing:1px}
|
||||
.sidebar-node strong{font:12px var(--mono);display:flex;align-items:center;gap:8px;margin-top:4px}
|
||||
.sidebar-node>span:not(.tiny-label){font:12px var(--mono);color:var(--muted);padding-left:14px}
|
||||
.node-rule{height:1px;background:var(--line);margin:18px 0 4px}
|
||||
.sidebar-node p{font-size:11px;color:#788c9a;line-height:1.9;margin:0}
|
||||
.sidebar-footer{border-top:1px solid var(--line);padding:20px 12px 0;font-size:10px;color:#7e929f;display:flex;gap:8px;align-items:center}
|
||||
.dot{display:inline-block;width:6px;height:6px;border-radius:50%;background:var(--green);flex-shrink:0}
|
||||
.sidebar-footer .dot{background:#58776b}
|
||||
main{margin-left:220px;padding:34px 36px 20px;max-width:1920px}
|
||||
.page-header{display:flex;justify-content:space-between;gap:20px;align-items:center;margin-bottom:27px}
|
||||
.eyebrow{font-size:10px;font-weight:600;letter-spacing:1.2px;color:#8094a3}
|
||||
h1{font-size:28px;letter-spacing:.3px;font-weight:600;margin:7px 0 5px;line-height:1.4}
|
||||
.title-dot{color:var(--green);padding-left:1px}
|
||||
.page-header p{margin:0;color:var(--muted);font-size:11px}
|
||||
.separator{color:#465865;padding:0 9px}
|
||||
#host-name{font-family:var(--mono)}
|
||||
.header-actions{display:flex;align-items:center;gap:17px}
|
||||
.live-badge{font-size:11px;display:flex;gap:7px;align-items:center;color:#b8c7cf;white-space:nowrap}
|
||||
.icon-button{background:var(--panel);border:1px solid var(--line);width:36px;height:36px;border-radius:7px;color:#b8c7cf;display:grid;place-items:center}
|
||||
.icon-button:hover,.text-button:hover{border-color:#61717d;color:white}
|
||||
.health-banner{display:flex;gap:15px;align-items:center;background:linear-gradient(105deg,#1c3027,#17221f 70%);border:1px solid #36503c;border-radius:9px;padding:18px 21px;margin-bottom:20px}
|
||||
.health-icon{width:37px;height:37px;border-radius:50%;background:#294c30;color:var(--green);display:grid;place-items:center}
|
||||
.health-icon svg{width:23px;height:23px;stroke-width:2}
|
||||
.health-text strong{font-size:16px;font-weight:600;color:#d7edcc}
|
||||
.health-text p{font-size:11px;color:#92b49d;margin:3px 0 0}
|
||||
.health-time{margin-left:auto;padding-left:24px;border-left:1px solid #33463d;text-align:right;display:grid;gap:3px;white-space:nowrap}
|
||||
.health-time span{font-size:10px;color:#88a393}
|
||||
.health-time strong{font:11px var(--mono);font-weight:400;color:#c8d7cd}
|
||||
.health-banner.warning{background:linear-gradient(100deg,#332b20,#211f1c);border-color:#5e4a2c}
|
||||
.warning .health-icon{background:#554025;color:var(--amber)}
|
||||
.warning .health-text strong{color:#f7d99f}
|
||||
.warning .health-text p{color:#c0a77f}
|
||||
.health-banner.error{background:linear-gradient(100deg,#352322,#221c1f);border-color:#60403c}
|
||||
.error .health-icon{background:#5b302d;color:var(--red)}
|
||||
.error .health-text strong{color:#ffc2bd}
|
||||
.error .health-text p{color:#ca9c98}
|
||||
.health-banner.loading{background:var(--panel);border-color:var(--line)}
|
||||
.loading .health-icon{background:var(--panel2);color:var(--muted)}
|
||||
.loading .health-text strong{color:var(--text)}
|
||||
.loading .health-text p{color:var(--muted)}
|
||||
.current-issues{display:grid;gap:10px;margin:0 0 20px}
|
||||
.issue{border:1px solid #67422f;background:#271f1c;border-radius:8px;padding:16px 20px}
|
||||
.issue strong{color:var(--amber);font-size:14px}
|
||||
.issue p{margin:5px 0;font-size:12px;color:#c6bbb1}
|
||||
.issue small{color:#a89583}
|
||||
.issue.error{border-color:#673b3b;background:#281d20}
|
||||
.issue.error strong{color:var(--red)}
|
||||
.overview-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}
|
||||
.metric-card{border:1px solid var(--line);background:var(--panel);border-radius:9px;padding:18px 20px 15px;min-width:0}
|
||||
.card-top{display:flex;align-items:center;justify-content:space-between;color:#a6b6c1;font-size:12px}
|
||||
.card-top svg{color:#647e8e;width:19px;height:19px}
|
||||
.metric-value{font:38px/1.25 var(--mono);letter-spacing:-1.8px;margin:17px 0 12px;color:#eef4f4;white-space:nowrap}
|
||||
.metric-value small{font:12px var(--sans);color:var(--muted);letter-spacing:0;margin-left:7px}
|
||||
.metric-value.word{font:28px/1.7 var(--sans);letter-spacing:1px;margin-top:12px;margin-bottom:6px;color:var(--green)}
|
||||
.meter{height:4px;border-radius:4px;background:#26343e;overflow:hidden}
|
||||
.meter i{display:block;height:100%;width:0;background:var(--green);border-radius:4px;transition:width .6s}
|
||||
.meter.violet i{background:var(--purple)}
|
||||
.metric-bottom{display:flex;justify-content:space-between;align-items:center;margin-top:15px;gap:5px;white-space:nowrap;font-size:10px;color:#8195a3}
|
||||
.metric-bottom strong{color:#b9c6cd;font-size:10px;font-weight:400}
|
||||
.gpu-online-label,.disk-health-detail{font-size:11px;color:#a4b5bf;display:flex;gap:6px;align-items:center;height:4px;line-height:16px;margin-top:17px;margin-bottom:5px}
|
||||
.disk-health-detail{height:10px;margin-top:5px}
|
||||
.section-heading{display:flex;align-items:center;justify-content:space-between;gap:18px;margin:34px 0 16px;scroll-margin-top:24px}
|
||||
.section-heading h2{font-size:19px;font-weight:600;letter-spacing:.4px;margin:5px 0}
|
||||
.section-heading p{font-size:11px;color:var(--muted);margin:5px 0 0}
|
||||
.section-heading.compact{margin:25px 0 13px}
|
||||
.section-heading.compact>div{display:flex;align-items:center;gap:15px}
|
||||
.compact h2{font-size:15px;margin:0}
|
||||
.compact span{font-size:10px;color:var(--muted)}
|
||||
.subtle-pill{border:1px solid var(--line);padding:3px 9px;border-radius:4px;white-space:nowrap;font:10px var(--mono)!important;color:#91a6b3!important}
|
||||
.gpu-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px}
|
||||
.gpu-card{border:1px solid var(--line);border-radius:9px;background:var(--panel);padding:19px 21px;position:relative;overflow:hidden;--accent:var(--blue)}
|
||||
.gpu-card::before{content:"";position:absolute;top:19px;bottom:19px;left:0;width:3px;background:var(--accent);border-radius:0 4px 4px 0}
|
||||
.gpu-card .gpu-head{display:flex;gap:10px;align-items:center}
|
||||
.gpu-label{font:11px var(--mono);color:var(--accent);background:#213443;padding:3px 7px;border-radius:4px}
|
||||
.gpu-card h3{font-size:13px;margin:0;font-weight:500}
|
||||
.gpu-state{font-size:10px;color:var(--green);margin-left:auto;display:flex;gap:6px;align-items:center}
|
||||
.gpu-card .gpu-content{display:flex;gap:28px;align-items:center;margin:19px 0 11px}
|
||||
.gpu-util{font:43px/1 var(--mono);letter-spacing:-1.8px;min-width:115px}
|
||||
.gpu-util small{font:12px var(--sans);color:var(--muted);margin-left:3px;letter-spacing:0}
|
||||
.gpu-util span{display:block;font:10px var(--sans);color:var(--muted);letter-spacing:0;margin-top:8px}
|
||||
.gpu-numbers{flex:1;display:grid;grid-template-columns:1fr 1fr;gap:14px}
|
||||
.gpu-number>span{display:block;font-size:10px;color:var(--muted);margin-bottom:4px}
|
||||
.gpu-number strong{font:20px var(--mono)}
|
||||
.gpu-number small{font:10px var(--sans);color:#899daa;margin-left:5px}
|
||||
.gpu-vram{display:flex;justify-content:space-between;gap:10px;font-size:10px;color:var(--muted);margin:16px 0 7px}
|
||||
.gpu-vram strong{font:10px var(--mono);color:#b3c2cb}
|
||||
.gpu-card .meter{height:3px}
|
||||
.gpu-card .meter i{background:var(--accent)}
|
||||
.gpu-meta{display:flex;justify-content:space-between;gap:9px;border-top:1px solid #24313b;padding-top:11px;margin-top:13px;font:9px var(--mono);color:#738c9d;white-space:nowrap}
|
||||
.skeleton{display:grid;place-items:center;color:var(--muted);min-height:230px;animation:fade 2s infinite}
|
||||
@keyframes fade{50%{opacity:.45}
|
||||
}
|
||||
.system-strip{display:flex;justify-content:space-between;flex-wrap:wrap;gap:12px 20px;margin-top:14px;border:1px solid #202c35;border-radius:7px;background:#111a21;padding:13px 17px;color:#7d939f;font-size:10px}
|
||||
.system-strip span{display:flex;gap:8px;align-items:center}
|
||||
.system-strip svg{width:13px;height:13px}
|
||||
.system-strip strong{font:10px var(--mono);color:#afc1cd}
|
||||
.segmented{display:flex;gap:2px;padding:4px;border:1px solid var(--line);background:#121c23;border-radius:7px;white-space:nowrap}
|
||||
.segmented button{border:none;color:var(--muted);background:none;padding:6px 11px;border-radius:4px;font-size:11px}
|
||||
.segmented button.selected{background:#304131;color:#cff5b3}
|
||||
.segmented button:hover{color:white}
|
||||
.history-note{font-size:10px;color:#8398a6;border-left:2px solid #3d505e;padding-left:10px;margin:-4px 0 15px;line-height:1.8}
|
||||
.chart-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px}
|
||||
.chart-card{border:1px solid var(--line);border-radius:9px;background:var(--panel);padding:18px 20px 12px;min-width:0}
|
||||
.chart-heading{display:flex;justify-content:space-between;gap:10px;align-items:center}
|
||||
.chart-heading h3{margin:0;font-size:13px;font-weight:500}
|
||||
.chart-heading>span{font:10px var(--mono);color:#728c9e}
|
||||
.chart-legend{display:flex;gap:17px;flex-wrap:wrap;margin:13px 0 4px;font-size:9px;color:#a6b8c5;min-height:17px}
|
||||
.legend-item{display:flex;align-items:center;gap:6px}
|
||||
.legend-swatch{width:13px;height:3px;border-radius:3px}
|
||||
.chart{height:190px;position:relative}
|
||||
.chart svg{width:100%;height:100%;stroke:none;overflow:visible;display:block}
|
||||
.chart text{font:9px var(--mono);fill:#6e8798}
|
||||
.chart .grid-line{stroke:#27343e;stroke-dasharray:3 5;stroke-width:.6}
|
||||
.chart .series-line{fill:none;stroke-width:1.7;stroke-linejoin:round;stroke-linecap:round}
|
||||
.chart .cursor{stroke:#93a6b1;stroke-width:.7;stroke-dasharray:4 3}
|
||||
.chart .chart-overlay{fill:transparent;cursor:crosshair}
|
||||
.chart-tooltip{position:absolute;top:18px;background:#202d37;border:1px solid #455866;border-radius:7px;padding:9px 12px;z-index:2;box-shadow:0 8px 24px #0007;font-size:10px;pointer-events:none;min-width:140px}
|
||||
.chart-tooltip strong{display:block;color:#dce8ef;font:10px var(--mono);margin-bottom:7px}
|
||||
.chart-tooltip p{margin:3px 0;display:flex;justify-content:space-between;gap:15px;color:#b5c5d0}
|
||||
.chart-empty{position:absolute;inset:0;display:grid;place-content:center;text-align:center;color:#75909e;font-size:11px;pointer-events:none;gap:3px}
|
||||
.chart-empty small{font-size:9px;color:#5c7482}
|
||||
.text-button{display:inline-flex;align-items:center;justify-content:center;gap:7px;background:#18242d;border:1px solid #34444f;border-radius:5px;padding:7px 12px;color:#afc2cc;font-size:11px;white-space:nowrap}
|
||||
.text-button svg{width:14px;height:14px}
|
||||
.event-panel{border:1px solid var(--line);background:var(--panel);border-radius:9px;overflow:hidden}
|
||||
.event-toolbar{padding:17px 20px;display:flex;align-items:flex-end;gap:10px;border-bottom:1px solid var(--line);flex-wrap:wrap}
|
||||
.event-toolbar label{display:grid;gap:5px;font-size:10px;color:#8ea3b1}
|
||||
.event-toolbar select,.event-toolbar input{border:1px solid #34434e;border-radius:5px;background:#101a22;color:#cedae2;height:33px;padding:5px 9px;font-size:11px;max-width:200px;min-width:110px}
|
||||
.event-toolbar input[type=date]{max-width:135px;font-family:var(--mono);font-size:10px}
|
||||
.event-toolbar .search-label{flex:1;min-width:150px}
|
||||
.event-toolbar input[type=search]{width:100%;max-width:none}
|
||||
.primary-button{border:1px solid #b3e98a;background:var(--green);color:#233423;padding:7px 14px;border-radius:5px;font-size:11px;font-weight:600;height:33px;white-space:nowrap}
|
||||
.primary-button:hover{background:#c7fa9d}
|
||||
.event-caption{display:flex;justify-content:space-between;gap:15px;font-size:10px;color:#8298a8;padding:13px 20px;background:#121c24}
|
||||
.event-caption>span:first-child{color:#b1c3ce}
|
||||
.event-row{border-top:1px solid #26343f}
|
||||
.event-row>summary{display:flex;gap:13px;align-items:center;padding:15px 20px;list-style:none;cursor:pointer}
|
||||
.event-row>summary::-webkit-details-marker{display:none}
|
||||
.event-row>summary:hover{background:#1a2630}
|
||||
.event-level{font-size:10px;border:1px solid #4c3d2c;background:#332b20;color:var(--amber);border-radius:4px;padding:2px 6px;white-space:nowrap}
|
||||
.event-level.error{border-color:#663c3b;background:#392526;color:var(--red)}
|
||||
.event-level.info{border-color:#354b5b;background:#23323f;color:#9ab7cf}
|
||||
.event-main{flex:1;min-width:0}
|
||||
.event-main strong{display:block;font-size:12px;font-weight:500}
|
||||
.event-main p{margin:3px 0 0;color:#7891a2;font-size:9px;font-family:var(--mono);overflow-wrap:anywhere}
|
||||
.event-state{border:1px solid #344651;background:#1a2932;color:#93acbb;padding:3px 7px;border-radius:4px;font-size:9px;white-space:nowrap}
|
||||
.event-state.active{color:var(--red);background:#382628;border-color:#654044}
|
||||
.event-state.recovered{color:var(--green);background:#24372c;border-color:#3c5b40}
|
||||
.event-state.related{color:var(--amber);background:#372c20;border-color:#665036}
|
||||
.event-time{text-align:right;font:10px var(--mono);color:#afc1cd;white-space:nowrap;min-width:135px}
|
||||
.event-time small{display:block;font:9px var(--sans);color:#748d9e;margin-top:2px}
|
||||
.event-chevron{font-size:14px;color:#738b99;transform:rotate(0deg);transition:transform .2s}
|
||||
.event-row[open] .event-chevron{transform:rotate(90deg)}
|
||||
.event-detail{padding:3px 20px 19px 64px;font-size:11px;color:#9cb2c1}
|
||||
.event-meaning{margin:0 0 10px;color:#c2d0d9}
|
||||
.maintenance{background:#1d2b32;border-left:2px solid #6a879a;padding:10px 13px;margin:10px 0;color:#c5d4dc;line-height:1.9}
|
||||
.maintenance strong{color:var(--green);font-size:10px;display:block}
|
||||
.time-grid{display:grid;grid-template-columns:1fr 1fr;gap:6px 18px;font-size:10px;margin:12px 0}
|
||||
.time-grid span{color:#738d9f;display:block;font-size:9px}
|
||||
.time-grid strong{font-family:var(--mono);font-weight:400;overflow-wrap:anywhere}
|
||||
.time-warning{padding:9px 12px;border:1px solid #5d4a32;border-radius:4px;background:#2b251d;font-size:10px;color:#d9b67d;margin:10px 0}
|
||||
.raw-log{margin:10px 0 0;background:#0c151d;border:1px solid #293a47;border-radius:5px;padding:10px 12px;color:#9bb4c6;font:10px/1.8 var(--mono);white-space:pre-wrap;word-break:break-word}
|
||||
.pagination{display:flex;align-items:center;justify-content:space-between;padding:15px 20px;border-top:1px solid var(--line);font-size:10px;color:#869eac}
|
||||
.pagination>div{display:flex;gap:7px}
|
||||
.empty-events{padding:35px;text-align:center;color:var(--muted);font-size:12px}
|
||||
.empty-events strong{display:block;color:#c9d9e2;font-weight:500;margin-bottom:5px}
|
||||
.guide{margin-top:34px;border:1px solid #293944;border-radius:9px;padding:21px;background:#121d25;scroll-margin-top:25px}
|
||||
.guide>div:first-child{display:flex;align-items:center;gap:9px}
|
||||
.guide>div:first-child svg{color:#91b0c4;width:18px;height:18px}
|
||||
.guide h2{font-size:14px;font-weight:500;margin:0}
|
||||
.guide-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:28px;margin-top:20px}
|
||||
.guide-number{font:11px var(--mono);color:#8dad9b}
|
||||
.guide h3{font-size:12px;font-weight:500;margin:6px 0}
|
||||
.guide p{font-size:10px;color:#839baa;line-height:1.9;margin:0}
|
||||
footer{display:flex;justify-content:space-between;gap:20px;padding-top:25px;color:#607b8d;font-size:9px}
|
||||
.toast{position:fixed;bottom:24px;left:50%;transform:translateX(-50%);background:#304438;border:1px solid #6b886c;color:#e0f0de;padding:12px 20px;border-radius:7px;z-index:50;box-shadow:0 8px 20px #0006;font-size:12px}
|
||||
body:fullscreen main{margin-left:0}
|
||||
body:fullscreen .sidebar{display:none}
|
||||
@media(min-width:1700px){main{padding-left:50px;padding-right:50px}
|
||||
.chart{height:230px}
|
||||
.metric-card{padding:23px}
|
||||
.gpu-card{padding:23px}
|
||||
}
|
||||
@media(max-width:1200px){.sidebar{width:185px;padding-left:13px;padding-right:13px}
|
||||
.brand{font-size:17px;gap:9px}
|
||||
.brand-mark{width:33px;height:33px}
|
||||
.brand small{font-size:7px}
|
||||
main{margin-left:185px;padding:27px 23px}
|
||||
.metric-card{padding:15px}
|
||||
.metric-value{font-size:33px}
|
||||
.metric-bottom{font-size:9px;flex-wrap:wrap}
|
||||
.gpu-card{padding:17px}
|
||||
.gpu-content{gap:14px!important}
|
||||
.gpu-util{font-size:37px;min-width:95px}
|
||||
.gpu-meta{font-size:8px}
|
||||
.event-toolbar{gap:8px}
|
||||
.event-state{max-width:150px;white-space:normal}
|
||||
.compact>div>span{display:none}
|
||||
}
|
||||
@media(max-width:950px){.sidebar{width:66px;align-items:center;padding:23px 8px}
|
||||
.brand>span:not(.brand-mark),.nav-caption,.sidebar nav a:not(.active) .nav-dot,.sidebar-node,.sidebar-footer{display:none}
|
||||
.sidebar nav{margin-top:37px;width:100%}
|
||||
.sidebar nav a{font-size:0;padding:12px;gap:0;justify-content:center}
|
||||
.sidebar nav a svg{width:20px}
|
||||
.nav-dot{display:none}
|
||||
.nav-count{font-size:8px;position:absolute;margin:0 0 0 31px}
|
||||
.sidebar nav a{position:relative}
|
||||
main{margin-left:66px;padding:25px 22px}
|
||||
.overview-grid{gap:11px}
|
||||
.gpu-meta{flex-wrap:wrap;gap:4px}
|
||||
.gpu-numbers{gap:12px}
|
||||
.gpu-card h3{font-size:11px}
|
||||
.gpu-state{font-size:9px}
|
||||
.event-time{min-width:100px;font-size:9px}
|
||||
.event-state{font-size:8px}
|
||||
.event-row>summary{gap:9px;padding:14px}
|
||||
.guide-grid{gap:17px}
|
||||
.segmented button{padding:6px 8px}
|
||||
}
|
||||
@media(max-width:720px){main{padding:21px 15px}
|
||||
.page-header{margin-bottom:20px}
|
||||
h1{font-size:23px}
|
||||
.eyebrow{font-size:8px}
|
||||
.header-actions{gap:10px}
|
||||
.live-badge{font-size:9px}
|
||||
.page-header p{font-size:9px}
|
||||
.overview-grid{grid-template-columns:repeat(2,minmax(0,1fr))}
|
||||
.metric-value{font-size:34px;margin-top:12px}
|
||||
.metric-card{padding:15px 17px}
|
||||
.health-banner{padding:14px;gap:10px}
|
||||
.health-icon{width:30px;height:30px}
|
||||
.health-icon svg{width:20px}
|
||||
.health-text strong{font-size:13px}
|
||||
.health-text p{font-size:10px}
|
||||
.health-time{padding-left:10px;font-size:9px}
|
||||
.health-time strong{font-size:9px}
|
||||
.gpu-grid{grid-template-columns:1fr}
|
||||
.gpu-card h3{font-size:13px}
|
||||
.gpu-content{gap:25px!important}
|
||||
.gpu-meta{font-size:9px}
|
||||
.chart-grid{grid-template-columns:1fr}
|
||||
.section-heading{flex-wrap:wrap;margin-top:28px}
|
||||
.section-heading h2{font-size:18px}
|
||||
.section-heading p{font-size:10px}
|
||||
.segmented{width:100%;justify-content:space-around}
|
||||
.segmented button{flex:1}
|
||||
.section-heading.compact{flex-wrap:nowrap}
|
||||
.event-caption{font-size:9px;padding:12px}
|
||||
.event-caption span:last-child{display:none}
|
||||
.event-row>summary{flex-wrap:wrap;gap:8px}
|
||||
.event-main{min-width:150px}
|
||||
.event-state{order:5;margin-left:42px;font-size:9px}
|
||||
.event-time{margin-left:auto;min-width:130px;font-size:9px}
|
||||
.event-chevron{order:6;margin-left:auto}
|
||||
.event-detail{padding:5px 14px 17px}
|
||||
.time-grid{grid-template-columns:1fr}
|
||||
.event-toolbar{padding:14px}
|
||||
.event-toolbar label{flex:1;min-width:110px}
|
||||
.event-toolbar input,.event-toolbar select{max-width:none;width:100%;min-width:0}
|
||||
.event-toolbar input[type=date]{max-width:none}
|
||||
.event-toolbar .search-label{min-width:100%}
|
||||
.primary-button{width:100%}
|
||||
.guide-grid{grid-template-columns:1fr;gap:20px}
|
||||
.guide-number{float:left;margin-right:13px;margin-top:5px}
|
||||
.guide p{font-size:11px}
|
||||
.guide h3{font-size:13px}
|
||||
.guide h2{font-size:12px}
|
||||
footer{flex-direction:column;gap:4px;padding-bottom:10px}
|
||||
}
|
||||
@media(max-width:420px){.sidebar{display:none}
|
||||
main{margin-left:0;padding:20px 13px}
|
||||
h1{font-size:22px}
|
||||
.health-time{display:none}
|
||||
.gpu-numbers{gap:9px}
|
||||
.gpu-util{min-width:100px}
|
||||
.metric-card{padding:14px}
|
||||
.page-header{display:block;position:relative}
|
||||
.header-actions{position:absolute;right:0;top:0;gap:8px}
|
||||
.page-header .eyebrow{padding-right:130px;min-height:24px}
|
||||
.page-header h1{font-size:23px;margin-top:10px;white-space:nowrap}
|
||||
}
|
||||
@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}
|
||||
*{animation:none!important;transition:none!important}
|
||||
}
|
||||
|
||||
/* Readable incidents and scroll-linked navigation, revision 1.1. */
|
||||
main{max-width:none}
|
||||
.section-context{position:sticky;top:0;z-index:12;display:flex;align-items:center;justify-content:space-between;gap:14px;padding:11px 14px;margin:-8px 0 20px;background:#101b24f2;border:1px solid #2c404d;border-radius:7px;backdrop-filter:blur(12px)}
|
||||
.section-context>div:first-child{display:flex;gap:11px;align-items:center}
|
||||
.context-label{font-size:11px;color:#87a0b0}
|
||||
.section-context strong{font-size:13px;font-weight:500;color:#e3ecef}
|
||||
.context-actions{display:flex;gap:8px}
|
||||
.context-button{background:transparent;border:1px solid #3c505d;border-radius:5px;padding:4px 10px;color:#b9cdd8;font-size:11px}
|
||||
.context-button:hover{border-color:var(--green);color:var(--green)}
|
||||
.reading-progress{height:2px;position:absolute;bottom:-1px;left:0;right:0;overflow:hidden;border-radius:0 0 7px 7px}
|
||||
.reading-progress i{display:block;height:100%;width:0;background:var(--green);transition:width .1s linear}
|
||||
.sidebar nav a.active:after{content:'';width:5px;height:5px;border-radius:50%;background:var(--green);margin-left:auto}
|
||||
.sidebar nav a:not(.active):after{display:none}
|
||||
.event-explainer{margin:0 0 14px;padding:14px 17px;border:1px solid #425344;border-radius:7px;background:#1a2823;display:grid;gap:4px}
|
||||
.event-explainer strong{font-size:13px;color:#d0e3c8;font-weight:500}
|
||||
.event-explainer span{font-size:12px;color:#99b1a5}
|
||||
.event-row>summary{padding:18px 20px;gap:15px}
|
||||
.event-main strong{font-size:15px;line-height:1.7}
|
||||
.event-main p{font-size:12px;font-family:var(--sans);color:#91a9b9;margin-top:4px}
|
||||
.event-level{font-size:11px;padding:3px 7px}
|
||||
.event-state{font-size:11px;padding:4px 8px}
|
||||
.event-state.active.warning{color:var(--amber);background:#3a2c1a;border-color:#785a30}
|
||||
.event-time{font-size:12px;min-width:160px}
|
||||
.event-time small{font-size:10px}
|
||||
.event-detail{padding:4px 22px 23px 22px;font-size:13px;line-height:1.85}
|
||||
.plain-answer{padding:10px 13px;margin-bottom:5px;border-left:2px solid #3a5262}
|
||||
.plain-answer>strong{display:block;color:#cfdee7;font-weight:500;font-size:12px}
|
||||
.plain-answer p{margin:3px 0 0;color:#a5becd;font-size:13px}
|
||||
.maintenance{margin:12px 0 15px;padding:14px 17px}
|
||||
.maintenance>strong{font-size:12px}
|
||||
.maintenance-priority{margin:3px 0 7px;font-size:14px;color:#e6f1e3}
|
||||
.maintenance ol{margin:5px 0 0;padding-left:21px;font-size:13px}
|
||||
.maintenance li{margin:4px 0}
|
||||
.friendly-times{background:#111d25;padding:12px 15px;border:1px solid #2a3b47;border-radius:6px;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}
|
||||
.friendly-times span{font-size:11px;margin-bottom:3px}
|
||||
.friendly-times strong{font-size:12px;color:#bdd0dd}
|
||||
.record-summary{font-size:12px;color:#91a9b8;line-height:1.8}
|
||||
.time-warning{font-size:12px}
|
||||
.technical-details{border:1px solid #2d414f;border-radius:6px;margin-top:15px}
|
||||
.technical-details>summary{cursor:pointer;padding:10px 13px;font-size:12px;color:#91adbf;background:#121e27}
|
||||
.technical-details>summary:hover{color:var(--green)}
|
||||
.technical-body{padding:2px 13px 14px}
|
||||
.technical-body .time-grid{font-size:11px;grid-template-columns:repeat(2,minmax(0,1fr))}
|
||||
.technical-body .time-grid span{font-size:10px}
|
||||
.technical-body .raw-log{font-size:11px}
|
||||
.event-toolbar label{font-size:11px}
|
||||
.event-toolbar select,.event-toolbar input{font-size:12px;height:36px}
|
||||
.primary-button{height:36px;font-size:12px}
|
||||
.event-caption{font-size:11px;line-height:1.8}
|
||||
.mobile-section-nav{display:none}
|
||||
@media(min-width:1700px){.card-top{font-size:14px}.metric-bottom,.metric-bottom strong{font-size:12px}.gpu-card h3{font-size:15px}.gpu-number>span,.gpu-vram,.gpu-vram strong{font-size:12px}.system-strip,.system-strip strong{font-size:12px}.gpu-meta{font-size:11px}.chart-heading h3{font-size:15px}.chart-legend{font-size:11px}.chart{height:235px}}
|
||||
@media(max-width:1100px){.friendly-times{grid-template-columns:repeat(2,minmax(0,1fr))}.event-row>summary{gap:10px}.event-state{font-size:10px}.event-time{min-width:140px;font-size:10px}}
|
||||
@media(max-width:950px){.mobile-section-nav{position:fixed;display:grid;grid-template-columns:repeat(4,1fr);bottom:0;left:66px;right:0;background:#111d27f5;border-top:1px solid #3a4c57;z-index:25;padding:8px 6px calc(8px + env(safe-area-inset-bottom));backdrop-filter:blur(12px)}.mobile-section-nav a{display:flex;gap:7px;align-items:center;justify-content:center;font-size:12px;color:#8da8b9;padding:7px 4px;border-radius:5px}.mobile-section-nav a.active{color:var(--green);background:#293a2a}.mobile-section-nav svg{width:17px;height:17px}main{padding-bottom:90px}.sidebar nav a.active:after{position:absolute;right:3px;width:4px;height:4px}.context-button{font-size:10px}.section-context{margin-bottom:16px}}
|
||||
@media(max-width:720px){.event-main{min-width:130px}.event-main strong{font-size:14px}.event-row>summary{padding:15px}.event-state{margin-left:0}.event-time{margin-left:auto;min-width:140px}.event-detail{padding:3px 14px 17px}.event-caption{padding:12px 14px;font-size:11px}.plain-answer p,.maintenance ol{font-size:12px}.friendly-times strong{font-size:11px}.technical-body .time-grid{grid-template-columns:1fr}.context-label{display:none}.section-context{padding:9px 10px}.section-context strong{font-size:12px}.context-button{padding:4px 7px}.event-explainer strong{font-size:12px}.event-explainer span{font-size:11px}}
|
||||
@media(max-width:420px){.mobile-section-nav{left:0}.context-button{font-size:9px}.section-context strong{font-size:11px}.event-state{font-size:9px}.event-time{min-width:125px;font-size:9px}.friendly-times{grid-template-columns:1fr}.technical-body{padding-left:9px;padding-right:9px}}
|
||||
|
||||
/* Scheduled tasks */
|
||||
.task-panel{border:1px solid var(--line);border-radius:9px;background:var(--panel);overflow:hidden;min-height:max(380px,calc(100vh - 300px));display:flex;flex-direction:column}
|
||||
#task-manager,#task-list{display:flex;flex-direction:column;flex:1}
|
||||
#task-locked,#task-list>.task-empty{flex:1;display:flex;flex-direction:column;align-items:center;justify-content:center}
|
||||
.task-authbar{display:flex;justify-content:space-between;gap:18px;align-items:center;padding:18px 21px;background:#17252c;border-bottom:1px solid #30424d}
|
||||
.task-authbar strong{font-size:14px;font-weight:500;color:#d5e5ed}
|
||||
.task-authbar p{font-size:12px;color:#91aab9;margin:4px 0 0}
|
||||
.task-auth-actions{display:flex;gap:8px;flex-shrink:0}
|
||||
.task-empty{text-align:center;padding:44px 20px;color:#a5bdca}
|
||||
.task-empty>svg{width:31px;height:31px;color:var(--green);margin-bottom:7px}
|
||||
.task-empty h3{font-size:17px;font-weight:500;color:#d6e4ec;margin:6px 0}
|
||||
.task-empty p{font-size:13px;line-height:1.9;margin:7px 0}
|
||||
.task-empty>span{font-size:11px;color:#7993a4}
|
||||
.task-toolbar{display:flex;justify-content:space-between;align-items:center;padding:15px 20px;gap:15px}
|
||||
.task-toolbar>div{display:flex;align-items:center;gap:12px;flex-wrap:wrap}
|
||||
.task-toolbar strong{font-size:13px;font-weight:500;color:#d5e3eb}
|
||||
.task-toolbar span{font-size:10px;color:#7f99aa}
|
||||
.task-table-heading,.task-row{display:grid;grid-template-columns:minmax(150px,1.1fr) minmax(180px,1.2fr) minmax(145px,.9fr) minmax(200px,1.2fr);gap:18px;padding:13px 20px;align-items:center}
|
||||
.task-table-heading{background:#101b23;color:#7e9aaa;font-size:11px}
|
||||
.task-row{border-top:1px solid #283b47;padding-top:20px;padding-bottom:20px}
|
||||
.task-name-cell,.task-schedule-cell,.task-status-cell{min-width:0}
|
||||
.task-name-cell>strong{font-size:14px;font-weight:500;overflow-wrap:anywhere}
|
||||
.task-name-cell>p{font-size:11px;color:#829daa;line-height:1.8;margin:5px 0}
|
||||
.task-row-meta{display:flex;gap:7px;flex-wrap:wrap;margin-top:6px}
|
||||
.task-row-meta span{font-size:10px;color:#8eacbe;padding:1px 6px;background:#1d303c;border-radius:4px}
|
||||
.task-row-meta .task-danger-label{color:var(--amber);background:#3a2e1f}
|
||||
.task-schedule-cell>strong{font-size:12px;font-weight:400;overflow-wrap:anywhere;display:block;color:#c4d6e0}
|
||||
.task-schedule-cell>span{display:block;font:10px/1.8 var(--mono);color:#89a4b5;margin-top:6px}
|
||||
.task-status-cell{display:flex;align-items:flex-start;gap:6px;flex-wrap:wrap}
|
||||
.task-status-cell>small{font-size:10px;color:#809dac;flex-basis:100%;margin-top:3px}
|
||||
.task-state,.task-enabled{font-size:10px;border-radius:4px;padding:2px 7px;display:inline-flex;align-items:center;gap:4px;border:1px solid #3d5261;color:#a4bac9;background:#1a2a34;white-space:nowrap}
|
||||
.task-state.success{background:#253c2b;border-color:#466341;color:#b7ed97}
|
||||
.task-state.failed{background:#39282b;border-color:#734a49;color:#ffaaa1}
|
||||
.task-state.running{background:#22384a;border-color:#42657c;color:#96d0ff}
|
||||
.task-state.warning{background:#3a3020;border-color:#6c5734;color:#ebc181}
|
||||
.task-enabled.enabled{border-color:#3e6451;color:#a4dca8;background:#21382b}
|
||||
.task-row-actions{display:flex;gap:4px 9px;flex-wrap:wrap}
|
||||
.task-action{background:none;border:none;padding:4px 0;font-size:12px;color:#a9d88f;white-space:nowrap}
|
||||
.task-action:hover{color:#d2fabc;text-decoration:underline}
|
||||
.task-action.danger{color:#d9958d}
|
||||
.task-footnote{font-size:11px;color:#7996a7;background:#101b23;padding:13px 20px;border-top:1px solid #2b3b47;line-height:1.9}
|
||||
.task-dialog{width:min(810px,calc(100% - 26px));max-height:90vh;background:#14212a;color:var(--text);border:1px solid #4b6272;border-radius:12px;padding:0;box-shadow:0 30px 100px #000b;overflow:hidden}
|
||||
.task-dialog::backdrop{background:#030b12bf;backdrop-filter:blur(3px)}
|
||||
.task-dialog.small-dialog{max-width:510px}
|
||||
.task-dialog.log-dialog{width:min(1150px,calc(100% - 26px))}
|
||||
.task-dialog>form{display:flex;flex-direction:column;max-height:90vh}
|
||||
.dialog-heading{padding:21px 24px 17px;display:flex;justify-content:space-between;gap:16px;align-items:flex-start;border-bottom:1px solid #30414f;background:#192a34}
|
||||
.dialog-heading h2{font-size:19px;font-weight:500;margin:0;color:#e6eff4}
|
||||
.dialog-heading p{font-size:12px;color:#8eacbd;margin:5px 0 0;line-height:1.8}
|
||||
.dialog-close{border:1px solid #3b5160;border-radius:5px;background:#1a2a36;color:#9eb8c9;line-height:1;width:29px;height:29px;font-size:22px;flex-shrink:0}
|
||||
.dialog-close:hover{color:white;border-color:#7695aa}
|
||||
.dialog-content{padding:22px 24px;overflow:auto;line-height:1.9;min-height:0}
|
||||
.task-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:17px 20px}
|
||||
.task-form-grid label{display:grid;gap:7px;color:#b3c9d7;font-size:12px;align-content:start;min-width:0}
|
||||
.task-form-grid input:not([type=checkbox]):not([type=file]),.task-form-grid select,.task-form-grid textarea{width:100%;border:1px solid #3a5264;border-radius:5px;background:#0e1a23;color:#d8e6f0;padding:9px 11px;font:12px/1.65 var(--sans);height:38px;min-width:0}
|
||||
.task-form-grid textarea{height:auto;resize:vertical;font-family:var(--mono);line-height:1.75;font-size:12px;min-height:110px;tab-size:2}
|
||||
.task-form-grid input[type=checkbox]{accent-color:var(--green);width:15px;height:15px}
|
||||
.task-form-grid small{font-size:11px;color:#7e9aab;line-height:1.7}
|
||||
.full-field{grid-column:1/-1}
|
||||
.task-form-divider{display:flex;align-items:center;justify-content:space-between;border-top:1px solid #314651;padding-top:15px;margin-top:2px}
|
||||
.task-form-divider strong{font-size:13px;font-weight:500;color:#d3e4ee}
|
||||
.task-form-divider span{font:10px var(--mono);color:#7f9eaf}
|
||||
.field-label-row{display:flex;justify-content:space-between;align-items:center;gap:10px}
|
||||
.inline-button{border:none;background:none;color:var(--green);font-size:11px;padding:0}
|
||||
.weekday-field>span{font-size:12px;color:#b1c9d8}
|
||||
#task-weekdays{display:flex;gap:10px;margin-top:9px;flex-wrap:wrap}
|
||||
#task-weekdays label{display:flex;gap:7px;align-items:center;background:#1b303d;border:1px solid #385164;border-radius:5px;padding:7px 10px;min-width:54px}
|
||||
.schedule-preview{display:flex;gap:12px;align-items:center;background:#152a29;border:1px solid #365347;border-radius:5px;padding:11px 13px}
|
||||
.schedule-preview span{font-size:11px;color:#a5c2b0;line-height:1.9}
|
||||
.schedule-preview button{flex-shrink:0}
|
||||
.template-hint{padding:12px 14px;border:1px solid #705938;color:#e3c493;background:#332d21;border-radius:5px;font-size:12px;line-height:1.8}
|
||||
.task-advanced{border:1px solid #30495b;border-radius:5px}
|
||||
.task-advanced>summary{padding:10px 13px;color:#9ebacb;cursor:pointer;font-size:12px}
|
||||
.task-advanced .task-form-grid{padding:4px 13px 15px}
|
||||
.task-form-grid .task-enabled-field{display:flex;align-items:flex-start;gap:10px;background:#1f3028;padding:13px 14px;border:1px solid #3b5141;border-radius:5px}
|
||||
.task-enabled-field small{display:block;margin-top:3px;color:#8eae97}
|
||||
.task-enabled-field input{margin-top:4px}
|
||||
.dialog-footer{padding:15px 24px;display:flex;justify-content:space-between;gap:15px;align-items:center;border-top:1px solid #354a58;background:#182932}
|
||||
.dialog-footer>span{font-size:11px;color:#8ba8ba}
|
||||
.dialog-footer>div{display:flex;gap:8px}
|
||||
.form-error{background:#3b2529;border:1px solid #75474b;color:#ffb3ad;border-radius:5px;font-size:12px;line-height:1.8;padding:10px 12px;margin:0}
|
||||
#task-confirm-message{font-size:13px;color:#c4d8e5;margin:0;white-space:pre-wrap}
|
||||
#task-confirm-code{max-height:230px;overflow:auto;font-size:11px}
|
||||
.run-layout{display:grid;grid-template-columns:290px minmax(0,1fr);height:min(560px,70vh);min-height:300px}
|
||||
.task-runs-list{overflow:auto;background:#101d27;border-right:1px solid #304756;padding:10px}
|
||||
.run-item{background:none;border:1px solid transparent;display:grid;gap:5px;text-align:left;width:100%;color:#a5bdcc;border-radius:6px;padding:12px;margin-bottom:7px;font-size:11px}
|
||||
.run-item:hover,.run-item.selected{background:#203a45;border-color:#3f6271}
|
||||
.run-item>.task-state{justify-self:start}
|
||||
.run-item strong{font:11px var(--mono);color:#c7dce9}
|
||||
.run-item>span:not(.task-state){font-size:12px;overflow-wrap:anywhere}
|
||||
.run-item small{font-size:10px;color:#87a5b9}
|
||||
.task-no-runs{font-size:12px;color:#90aebe;line-height:1.9;padding:12px}
|
||||
.task-run-output{display:flex;flex-direction:column;min-width:0;background:#0b1620}
|
||||
.task-run-meta{padding:13px 17px;border-bottom:1px solid #2b4150;display:flex;gap:10px;align-items:center;flex-wrap:wrap;font-size:11px;color:#9fb9c9}
|
||||
#task-run-output{flex:1;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;font:12px/1.9 var(--mono);color:#c0d6e4;padding:17px;margin:0;tab-size:2}
|
||||
@media(max-width:1100px){.task-table-heading,.task-row{grid-template-columns:1fr 1.2fr 1fr}.task-table-heading>span:last-child{display:none}.task-row-actions{grid-column:1/-1;justify-content:flex-end;border-top:1px solid #293f4c;padding-top:10px}.task-row{gap:13px}.task-authbar{align-items:flex-start}.task-auth-actions{flex-wrap:wrap;justify-content:flex-end}}
|
||||
@media(max-width:950px){.mobile-section-nav{grid-template-columns:repeat(5,1fr)}.mobile-section-nav a{gap:5px;font-size:11px}}
|
||||
@media(max-width:720px){.task-authbar{flex-direction:column;padding:16px}.task-auth-actions{justify-content:flex-start}.task-toolbar{align-items:flex-start;padding:15px;flex-wrap:wrap}.task-table-heading{display:none}.task-row{grid-template-columns:1fr;padding:17px}.task-name-cell>strong{font-size:15px}.task-status-cell{align-items:center}.task-schedule-cell>strong{font-size:12px}.task-row-actions{justify-content:flex-start;gap:6px 17px}.task-action{font-size:13px}.task-status-cell>small{flex-basis:auto}.task-dialog{max-height:92vh}.task-dialog>form{max-height:92vh}.dialog-heading{padding:16px}.dialog-heading h2{font-size:17px}.dialog-heading p{font-size:11px}.dialog-content{padding:16px}.dialog-footer{padding:13px 16px;gap:9px}.dialog-footer>span{font-size:10px}.task-form-grid{grid-template-columns:1fr;gap:15px}.schedule-preview{flex-direction:column;align-items:flex-start}.task-form-divider{gap:10px}.run-layout{grid-template-columns:1fr;grid-template-rows:160px minmax(0,1fr);height:70vh}.task-runs-list{border-right:none;border-bottom:1px solid #2e4656}.run-item{grid-template-columns:auto 1fr;gap:6px 12px}.run-item small{grid-column:2}.task-empty{padding:30px 16px}.task-footnote{padding:12px 16px}}
|
||||
|
||||
176
static/tasks.js
Normal file
176
static/tasks.js
Normal file
@ -0,0 +1,176 @@
|
||||
/* Authenticated task management. Credentials are never saved in localStorage. */
|
||||
(()=>{
|
||||
'use strict';
|
||||
let auth={logged_in:false},tasks=[],listBusy=false,listRefreshPending=false,selectedRun=null,runsTask=null,confirmResolve=null,runReadSequence=0;
|
||||
const states={running:['执行中','running'],success:['执行成功','success'],failed:['执行失败','failed'],timeout:['超时已停止','failed'],cancelled:['已停止','neutral'],interrupted:['执行被中断','warning']};
|
||||
function taskError(id,message){$(id).textContent=message;$(id).hidden=!message;}
|
||||
async function request(path,body){
|
||||
const options={cache:'no-store',signal:AbortSignal.timeout(28000),credentials:'same-origin'};
|
||||
if(body!==undefined){options.method='POST';options.headers={'Content-Type':'application/json','X-Task-CSRF':auth.csrf||''};options.body=JSON.stringify(body);}
|
||||
const response=await fetch('/api/tasks'+path,options);
|
||||
const data=await response.json();
|
||||
if(!response.ok){
|
||||
if(response.status===401&&path!=='/login'){setAuth({logged_in:false});}
|
||||
const error=new Error(data.error||'任务操作未完成。');error.status=response.status;throw error;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
function setAuth(value){
|
||||
auth=value;$('task-manager').hidden=!value.logged_in;$('task-locked').hidden=!!value.logged_in;
|
||||
$('task-create').disabled=!value.logged_in;$('task-login-open').hidden=!!value.logged_in;
|
||||
$('task-change-password').hidden=!value.logged_in;$('task-logout').hidden=!value.logged_in;
|
||||
$('task-auth-title').textContent=value.logged_in?'管理员已登录':'定时任务管理未登录';
|
||||
$('task-auth-description').textContent=value.logged_in?'可新增、编辑和执行任务。登录最长有效 8 小时；执行身份为 root。':'设备监控无需登录；创建、修改或执行服务器任务需要管理员登录。';
|
||||
if(!value.logged_in){tasks=[];$('task-list').replaceChildren();}
|
||||
queueNavigation();
|
||||
}
|
||||
async function refreshAuth(){
|
||||
try{const value=await request('/auth');setAuth(value);if(value.logged_in)await refreshList();}
|
||||
catch(error){$('task-auth-title').textContent='任务管理服务暂不可用';$('task-auth-description').textContent=error.message;$('task-create').disabled=true;}
|
||||
}
|
||||
function openLogin(){taskError('task-login-error','');$('task-login-password').value='';$('task-login-dialog').showModal();$('task-login-password').focus();}
|
||||
$('task-login-open').addEventListener('click',openLogin);
|
||||
$('task-login-form').addEventListener('submit',async event=>{
|
||||
event.preventDefault();const button=event.submitter;button.disabled=true;taskError('task-login-error','');
|
||||
try{const result=await request('/login',{username:$('task-login-user').value.trim(),password:$('task-login-password').value});setAuth(result);$('task-login-password').value='';$('task-login-dialog').close();await refreshList();toast('已登录定时任务管理。');}
|
||||
catch(error){taskError('task-login-error',error.message);}finally{button.disabled=false;}
|
||||
});
|
||||
$('task-logout').addEventListener('click',async()=>{try{await request('/logout',{});setAuth({logged_in:false});document.querySelectorAll('.task-dialog[open]').forEach(dialog=>dialog.close());$('task-body').value='';$('task-run-output').textContent='';toast('已退出任务管理。');}catch(error){toast(error.message);}});
|
||||
document.querySelectorAll('[data-close-dialog]').forEach(button=>button.addEventListener('click',()=>$(button.dataset.closeDialog).close()));
|
||||
function confirmAction(title,message,code=''){
|
||||
return new Promise(resolve=>{confirmResolve=resolve;$('task-confirm-heading').textContent=title;$('task-confirm-message').textContent=message;$('task-confirm-code').textContent=code;$('task-confirm-code').hidden=!code;$('task-confirm-dialog').showModal();});
|
||||
}
|
||||
function finishConfirm(value){$('task-confirm-dialog').close();const resolve=confirmResolve;confirmResolve=null;resolve?.(value);}
|
||||
$('task-confirm-no').addEventListener('click',()=>finishConfirm(false));$('task-confirm-yes').addEventListener('click',()=>finishConfirm(true));$('task-confirm-dialog').addEventListener('cancel',event=>{event.preventDefault();finishConfirm(false);});
|
||||
function stateBadge(state){const [text,cls]=states[state]||['尚未执行','neutral'];return el('span','task-state '+cls,text);}
|
||||
function actionButton(label,action,task,cls=''){
|
||||
const button=el('button','task-action '+cls,label);button.type='button';button.addEventListener('click',()=>taskAction(action,task,button));return button;
|
||||
}
|
||||
function renderList(data){
|
||||
tasks=data.tasks||[];const holder=$('task-list');holder.replaceChildren();
|
||||
$('task-count').textContent=`共 ${tasks.length} 个任务 · ${tasks.filter(x=>x.enabled).length} 个已启用 · ${tasks.filter(x=>x.running).length} 个执行中`;
|
||||
$('task-server-time').textContent='服务器时间 '+when(data.server_time);
|
||||
if(!tasks.length){const empty=el('div','task-empty');empty.append(icon('clock'),el('h3','','还没有计划任务'),el('p','','点击“新增任务”，选择模板或填写自己的命令。任务默认停用，可先测试再启用。'));holder.append(empty);return;}
|
||||
tasks.forEach(task=>{
|
||||
const row=el('article','task-row');row.dataset.taskId=task.id;
|
||||
const name=el('div','task-name-cell');name.append(el('strong','',task.name));const meta=el('div','task-row-meta');meta.append(el('span','',({command:'命令',script:'脚本内容',file:'服务器脚本'})[task.mode]||task.mode),el('span','','root'));if(task.dangerous)meta.append(el('span','task-danger-label','重启／关机类'));name.append(meta);if(task.notes)name.append(el('p','',task.notes));
|
||||
const schedule=el('div','task-schedule-cell');schedule.append(el('strong','',task.schedule_label),el('span','',task.enabled?(task.next_run?'下次：'+when(task.next_run):'没有下次执行时间／单次任务已到期'):'计划已停用，不会自动执行'));
|
||||
const status=el('div','task-status-cell');status.append(el('span','task-enabled '+(task.enabled?'enabled':''),task.enabled?'计划已启用':'计划已停用'),stateBadge(task.running?'running':task.last_run?.state));if(task.last_run)status.append(el('small','','最近：'+when(task.last_run.started)));
|
||||
const actions=el('div','task-row-actions');
|
||||
if(task.running)actions.append(actionButton('停止本次','stop',task,'danger'));
|
||||
else actions.append(actionButton('执行一次','run',task));
|
||||
actions.append(actionButton('日志','logs',task),actionButton('编辑','edit',task),actionButton(task.enabled?'停用':'启用','enable',task));
|
||||
if(!task.running)actions.append(actionButton('删除','delete',task,'danger'));
|
||||
row.append(name,schedule,status,actions);holder.append(row);
|
||||
});queueNavigation();
|
||||
}
|
||||
async function refreshList(showMessage=false){if(!auth.logged_in)return;if(listBusy){listRefreshPending=true;return;}listBusy=true;try{renderList(await request(''));if(showMessage)toast('任务状态已更新。');}catch(error){if(error.status!==401)toast(error.message);}finally{listBusy=false;if(listRefreshPending){listRefreshPending=false;refreshList();}}}
|
||||
$('task-refresh').addEventListener('click',()=>refreshList(true));
|
||||
async function taskAction(action,task,button){
|
||||
button.disabled=true;
|
||||
try{
|
||||
if(action==='edit'){await openEditor(task.id);return;}
|
||||
if(action==='logs'){await openRuns(task);return;}
|
||||
if(action==='run'){
|
||||
const detail=await request('/item?id='+encodeURIComponent(task.id));
|
||||
const code=detail.definition.mode==='file'?detail.definition.path+' '+detail.definition.arguments:detail.definition.body;
|
||||
if(!await confirmAction('立即执行“'+task.name+'”？',task.dangerous?'这可能会立刻重启或关闭服务器，中断计算任务和网页连接。只有确认现在可以中断业务时才继续。':'会立刻以 root 身份执行以下内容。手动执行不会自动启用这个任务的计划。',code.slice(0,1200)))return;
|
||||
const result=await request('/run',{id:task.id});toast(result.message);await refreshList();await openRuns(task);return;
|
||||
}
|
||||
if(action==='enable'){
|
||||
if(!await confirmAction(task.enabled?'停用这项计划？':'启用这项计划？',task.enabled?'只停止以后的定时触发，已经开始的执行不会被强制中断。':`执行周期：${task.schedule_label}。${task.dangerous?'此任务可能重启／关闭服务器，请确认执行时段允许中断业务。':'到计划时间会以 root 身份执行。'}`))return;
|
||||
toast((await request('/enable',{id:task.id,enabled:!task.enabled})).message);
|
||||
}
|
||||
if(action==='delete'){
|
||||
if(!await confirmAction('删除“'+task.name+'”？','将停止后续调度并删除任务定义，已有执行记录仍保留。'))return;
|
||||
toast((await request('/delete',{id:task.id})).message);
|
||||
}
|
||||
if(action==='stop'){
|
||||
if(!await confirmAction('停止本次执行？','将终止本任务的执行进程。脚本可能尚未完成修改，请先确认可以中断。'))return;
|
||||
toast((await request('/stop',{id:task.id})).message);
|
||||
}
|
||||
await refreshList();
|
||||
}catch(error){toast(error.message);}finally{button.disabled=false;}
|
||||
}
|
||||
function localDateValue(timestamp){const parts=new Intl.DateTimeFormat('sv-SE',{timeZone:'Asia/Shanghai',year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',hour12:false}).format(new Date(timestamp*1000));return parts.replace(' ','T');}
|
||||
function updateEditorFields(){
|
||||
const mode=$('task-mode').value,cycle=$('task-cycle').value;
|
||||
$('task-body-field').hidden=mode==='file';$('task-body').required=mode!=='file';$('task-body').disabled=mode==='file';
|
||||
$('task-path-field').hidden=mode!=='file';$('task-path').required=mode==='file';$('task-path').disabled=mode!=='file';$('task-arguments-field').hidden=mode!=='file';
|
||||
$('task-language-field').hidden=mode==='command';$('task-body-label').textContent=mode==='command'?'执行命令':'脚本内容';$('task-body').rows=mode==='command'?5:10;
|
||||
for(const option of $('task-interpreter').options)option.disabled=option.value==='auto'&&mode!=='file';
|
||||
if(mode==='script'&&$('task-interpreter').value==='auto')$('task-interpreter').value='bash';
|
||||
document.querySelectorAll('[data-task-cycle]').forEach(wrapper=>{const shown=wrapper.dataset.taskCycle.split(' ').includes(cycle);wrapper.hidden=!shown;wrapper.querySelectorAll('input,select').forEach(input=>input.disabled=!shown);});
|
||||
$('task-every').max=cycle==='hours'?'168':'1440';
|
||||
$('task-save').textContent=$('task-enabled').checked?'保存并启用':'保存任务';
|
||||
$('task-preview-text').textContent='修改后可检查执行周期；错过时间不补跑。';
|
||||
}
|
||||
function formDefinition(){
|
||||
const kind=$('task-cycle').value;const schedule={kind};
|
||||
if(['daily','weekly','monthly'].includes(kind))schedule.time=$('task-time').value;
|
||||
if(kind==='weekly')schedule.weekdays=[...$('task-weekdays').querySelectorAll('input:checked')].map(x=>Number(x.value));
|
||||
if(kind==='monthly')schedule.day=Number($('task-day').value);
|
||||
if(kind==='minutes'||kind==='hours')schedule.every=Number($('task-every').value);
|
||||
if(kind==='once')schedule.at=$('task-once').value;
|
||||
if(kind==='cron')schedule.expression=$('task-cron').value;
|
||||
return {id:$('task-id').value||undefined,name:$('task-name').value.trim(),template:$('task-template').value,
|
||||
mode:$('task-mode').value,interpreter:$('task-interpreter').value,body:$('task-body').value,
|
||||
path:$('task-path').value,arguments:$('task-arguments').value,cwd:$('task-cwd').value,
|
||||
timeout:Number($('task-timeout').value),notes:$('task-notes').value,enabled:$('task-enabled').checked,schedule};
|
||||
}
|
||||
async function openEditor(id=null){
|
||||
$('task-editor-form').reset();$('task-id').value='';$('task-template-hint').hidden=true;taskError('task-editor-error','');
|
||||
$('task-once').value=localDateValue(Date.now()/1000+3600);$('task-enabled').checked=false;
|
||||
$('task-editor-heading').textContent=id?'编辑定时任务':'新增定时任务';
|
||||
if(id){const task=await request('/item?id='+encodeURIComponent(id)),d=task.definition,s=d.schedule;
|
||||
$('task-id').value=id;$('task-name').value=d.name;$('task-template').value=d.template||'custom';$('task-mode').value=d.mode;
|
||||
$('task-interpreter').value=d.interpreter;$('task-body').value=d.body;$('task-path').value=d.path;$('task-arguments').value=d.arguments;
|
||||
$('task-cwd').value=d.cwd;$('task-timeout').value=d.timeout;$('task-notes').value=d.notes;$('task-enabled').checked=!!task.enabled;
|
||||
$('task-cycle').value=s.kind;$('task-time').value=s.time||'03:00';$('task-day').value=s.day||1;$('task-every').value=s.every||60;$('task-cron').value=s.expression||'0 3 * * *';if(s.at)$('task-once').value=s.at.slice(0,16);
|
||||
$('task-weekdays').querySelectorAll('input').forEach(input=>input.checked=(s.weekdays||[0]).includes(Number(input.value)));
|
||||
}
|
||||
updateEditorFields();$('task-editor-dialog').showModal();$('task-name').focus();
|
||||
}
|
||||
$('task-create').addEventListener('click',()=>openEditor().catch(error=>toast(error.message)));
|
||||
['task-mode','task-cycle','task-enabled'].forEach(id=>$(id).addEventListener('change',updateEditorFields));
|
||||
$('task-preview').addEventListener('click',async()=>{if(!$('task-editor-form').reportValidity())return;const button=$('task-preview');button.disabled=true;try{const result=await request('/preview',formDefinition());$('task-preview-text').textContent=result.schedule_label+'；预计下次：'+(result.next_run?when(result.next_run):'没有可计算的将来时间');}catch(error){$('task-preview-text').textContent=error.message;}finally{button.disabled=false;}});
|
||||
$('task-editor-form').addEventListener('submit',async event=>{
|
||||
event.preventDefault();taskError('task-editor-error','');const definition=formDefinition();
|
||||
const dangerous=/\b(reboot|shutdown|poweroff|halt)\b|\binit\s+[06]\b/i.test(definition.body+' '+definition.path);
|
||||
if(definition.enabled&&!await confirmAction(dangerous?'保存并启用重启／关机类任务？':'保存并启用任务？',dangerous?'此任务将在计划时间中断服务器业务。请确认脚本和执行时段已核对。':'保存后会按设置的周期执行，不会立即运行。',definition.mode==='file'?definition.path:definition.body.slice(0,800)))return;
|
||||
$('task-save').disabled=true;
|
||||
try{const result=await request('/save',definition);$('task-editor-dialog').close();toast(result.message);await refreshList();}
|
||||
catch(error){taskError('task-editor-error',error.message);}finally{$('task-save').disabled=false;}
|
||||
});
|
||||
$('task-template').addEventListener('change',()=>{
|
||||
const type=$('task-template').value;$('task-enabled').checked=false;$('task-template-hint').hidden=type==='custom';
|
||||
if(type==='hardware'){$('task-name').value='检查显卡状态';$('task-mode').value='command';$('task-body').value='/usr/bin/nvidia-smi';$('task-cycle').value='daily';$('task-time').value='08:00';$('task-template-hint').textContent='只读取显卡状态，结果会出现在执行日志。可先保存为停用，再执行一次测试。';}
|
||||
if(type==='reboot'){$('task-name').value='服务器定时重启';$('task-mode').value='command';$('task-body').value='/usr/bin/systemctl reboot';$('task-cycle').value='weekly';$('task-time').value='03:00';$('task-weekdays').querySelectorAll('input').forEach(input=>input.checked=input.value==='6');$('task-template-hint').textContent='这条命令会重启整台服务器，正在运行的 GPU 任务会中断。模板默认停用，请先确认允许重启的时间。';}
|
||||
if(type==='certificate'){$('task-name').value='SSL 证书续签';$('task-mode').value='script';$('task-interpreter').value='bash';$('task-cycle').value='daily';$('task-time').value='03:10';$('task-body').value='#!/usr/bin/env bash\nset -euo pipefail\n# 先确认已有域名和续签配置，且没有其他重复的自动续签计划。\nif command -v certbot >/dev/null 2>&1; then\n certbot renew --quiet\nelif [ -x /root/.acme.sh/acme.sh ]; then\n /root/.acme.sh/acme.sh --cron --home /root/.acme.sh\nelse\n echo "未检测到 Certbot 或 acme.sh，请先配置证书续签工具。" >&2\n exit 1\nfi\n# 如需重新加载 Web 服务，请按实际服务另行配置 deploy-hook。';$('task-template-hint').textContent='这是续签示例，不会安装或配置域名。请先确认 Certbot／acme.sh 的现有配置，并检查是否已有续签任务。';}
|
||||
updateEditorFields();
|
||||
});
|
||||
$('task-import-file').addEventListener('click',()=>$('task-upload').click());
|
||||
$('task-upload').addEventListener('change',async()=>{const file=$('task-upload').files[0];if(!file)return;if(file.size>65536){toast('脚本文件不能超过 64 KiB。');return;}$('task-mode').value='script';$('task-template').value='custom';$('task-interpreter').value=file.name.endsWith('.py')?'python3':'bash';$('task-body').value=await file.text();if(!$('task-name').value)$('task-name').value=file.name;updateEditorFields();$('task-upload').value='';});
|
||||
async function openRuns(task=null){runsTask=task;selectedRun=null;$('task-runs-heading').textContent=task?'执行记录 · '+task.name:'全部任务执行记录';$('task-run-meta').textContent='选择一次执行，查看输出和错误。';$('task-run-output').textContent='尚未选择执行记录。';$('task-runs-dialog').showModal();await refreshRuns();}
|
||||
async function refreshRuns(){
|
||||
if(!auth.logged_in||!$('task-runs-dialog').open)return;
|
||||
try{const data=await request(runsTask?'/runs?id='+encodeURIComponent(runsTask.id):'/history');const holder=$('task-runs-list');holder.replaceChildren();
|
||||
if(!data.runs.length){holder.append(el('p','task-no-runs','还没有执行记录。可以等待计划时间，或手动执行一次。'));return;}
|
||||
data.runs.forEach(run=>{const button=el('button','run-item'+(selectedRun===run.id?' selected':''));button.type='button';button.dataset.runId=run.id;button.append(stateBadge(run.state),el('strong','',when(run.started)),el('span','',run.task_name),el('small','',(run.trigger==='manual'?'手动执行':'定时执行')+' · '+(run.duration!=null?num(run.duration,1)+' 秒':run.state==='running'?'执行中':'耗时未记录')));button.addEventListener('click',()=>loadRun(run.id));holder.append(button);});
|
||||
if(!selectedRun)selectedRun=data.runs[0].id;await loadRun(selectedRun);
|
||||
}catch(error){if(error.status!==401)toast(error.message);}
|
||||
}
|
||||
async function loadRun(id){
|
||||
selectedRun=id;const sequence=++runReadSequence;
|
||||
try{const data=await request('/runlog?run_id='+encodeURIComponent(id));if(sequence!==runReadSequence)return;
|
||||
$('task-run-meta').replaceChildren(stateBadge(data.run.state),el('span','',`开始 ${when(data.run.started)} · 退出码 ${data.run.exit_code??'尚未取得'} · 修订 ${data.run.revision}`));
|
||||
const output=$('task-run-output'),stick=output.scrollHeight-output.scrollTop-output.clientHeight<50;output.textContent=(data.tail_only?'[仅显示最后 128 KiB]\n':'')+data.output;if(stick)output.scrollTop=output.scrollHeight;
|
||||
$('task-runs-list').querySelectorAll('.run-item').forEach(button=>button.classList.toggle('selected',button.dataset.runId===id));
|
||||
}catch(error){if(error.status!==401)$('task-run-output').textContent=error.message;}
|
||||
}
|
||||
$('task-all-runs').addEventListener('click',()=>openRuns());
|
||||
$('task-change-password').addEventListener('click',()=>{$('task-password-form').reset();taskError('task-password-error','');$('task-password-dialog').showModal();});
|
||||
$('task-password-form').addEventListener('submit',async event=>{event.preventDefault();taskError('task-password-error','');if($('task-new-password').value!==$('task-repeat-password').value){taskError('task-password-error','两次新密码输入不一致。');return;}const button=event.submitter;button.disabled=true;try{const result=await request('/password',{old_password:$('task-old-password').value,new_password:$('task-new-password').value});$('task-password-dialog').close();setAuth({logged_in:false});toast(result.message);}catch(error){taskError('task-password-error',error.message);}finally{button.disabled=false;}});
|
||||
refreshAuth();setInterval(()=>{if(!document.hidden&&auth.logged_in)refreshList();},15000);setInterval(()=>{if(!document.hidden&&$('task-runs-dialog').open)refreshRuns();},3000);
|
||||
document.addEventListener('visibilitychange',()=>{if(!document.hidden)refreshAuth();});
|
||||
})();
|
||||
386
task_core.py
Normal file
386
task_core.py
Normal file
@ -0,0 +1,386 @@
|
||||
"""Root-owned task store and native systemd schedules; no third-party modules."""
|
||||
from contextlib import contextmanager
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shlex
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
BASE=Path(os.environ.get('HARDWARE_TASK_DATA','/var/lib/hardware-taskd'))
|
||||
LOGS=Path(os.environ.get('HARDWARE_TASK_LOGS','/var/log/hardware-taskd'))
|
||||
UNITS=Path(os.environ.get('HARDWARE_TASK_UNITS','/etc/systemd/system'))
|
||||
APP=Path(__file__).resolve().parent
|
||||
TZ=dt.timezone(dt.timedelta(hours=8))
|
||||
WEEKDAYS=['Mon','Tue','Wed','Thu','Fri','Sat','Sun']
|
||||
|
||||
class TaskError(Exception):
|
||||
def __init__(self,message,status=400):
|
||||
self.status=status
|
||||
super().__init__(message)
|
||||
|
||||
def iso(value=None):
|
||||
return dt.datetime.fromtimestamp(time.time() if value is None else value,TZ).isoformat(timespec='seconds')
|
||||
|
||||
@contextmanager
|
||||
def db():
|
||||
con=sqlite3.connect(BASE/'tasks.sqlite3',timeout=10)
|
||||
con.row_factory=sqlite3.Row
|
||||
try:
|
||||
yield con;con.commit()
|
||||
except Exception:
|
||||
con.rollback();raise
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
def init_store():
|
||||
BASE.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
LOGS.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
(BASE/'scripts').mkdir(exist_ok=True,mode=0o700)
|
||||
with db() as con:
|
||||
con.execute('PRAGMA journal_mode=WAL')
|
||||
con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS tasks(id TEXT PRIMARY KEY,name TEXT,definition TEXT,
|
||||
enabled INTEGER,deleted INTEGER DEFAULT 0,created REAL,updated REAL,revision INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS runs(id TEXT PRIMARY KEY,task_id TEXT,task_name TEXT,
|
||||
started REAL,finished REAL,state TEXT,exit_code INTEGER,trigger TEXT,boot TEXT,
|
||||
script_hash TEXT,log_bytes INTEGER DEFAULT 0,truncated INTEGER DEFAULT 0,revision INTEGER,duration REAL,
|
||||
log_purged INTEGER DEFAULT 0);
|
||||
CREATE INDEX IF NOT EXISTS run_task ON runs(task_id,started DESC);
|
||||
CREATE TABLE IF NOT EXISTS audit(id INTEGER PRIMARY KEY AUTOINCREMENT,at REAL,
|
||||
action TEXT,task_id TEXT,actor TEXT,client TEXT,detail TEXT);
|
||||
CREATE TABLE IF NOT EXISTS requests(task_id TEXT PRIMARY KEY,trigger TEXT,expires REAL);
|
||||
''')
|
||||
|
||||
def atom_write(path,text,mode=0o600):
|
||||
temporary=Path(str(path)+'.tmp')
|
||||
temporary.write_text(text,encoding='utf-8');os.chmod(temporary,mode);os.replace(temporary,path)
|
||||
|
||||
def validate_id(value):
|
||||
if not isinstance(value,str) or not re.fullmatch('[0-9a-f]{32}',value):
|
||||
raise TaskError('任务编号不正确。')
|
||||
return value
|
||||
|
||||
def execute(args,timeout=15,check=True):
|
||||
try:
|
||||
p=subprocess.run(args,capture_output=True,text=True,timeout=timeout,env={**os.environ,'LC_ALL':'C','TZ':'Asia/Shanghai'})
|
||||
except (OSError,subprocess.TimeoutExpired) as exc:
|
||||
raise TaskError('系统调度操作未完成：'+str(exc),503)
|
||||
if check and p.returncode:
|
||||
raise TaskError('系统调度操作失败：'+(p.stderr or p.stdout)[-1200:],503)
|
||||
return p
|
||||
|
||||
def number(value,minimum,maximum,label):
|
||||
try:
|
||||
if isinstance(value,bool):raise ValueError()
|
||||
n=int(str(value))
|
||||
except (ValueError,TypeError):
|
||||
raise TaskError(label+'必须是整数。')
|
||||
if not minimum<=n<=maximum:
|
||||
raise TaskError(f'{label}必须在 {minimum} 至 {maximum} 之间。')
|
||||
return n
|
||||
|
||||
def clock(value):
|
||||
if not isinstance(value,str) or not re.fullmatch(r'\d{2}:\d{2}',value):
|
||||
raise TaskError('执行时间格式应为 03:00。')
|
||||
h,m=map(int,value.split(':'))
|
||||
if h>23 or m>59:raise TaskError('执行时间不正确。')
|
||||
return value+':00'
|
||||
|
||||
def cron_field(value,low,high,names=None):
|
||||
value=value.upper()
|
||||
if names:
|
||||
value=re.sub('[A-Z]{3}',lambda m:str(names[m[0]]) if m[0] in names else m[0],value)
|
||||
result=set()
|
||||
for part in value.split(','):
|
||||
pieces=part.split('/')
|
||||
if len(pieces)>2:raise TaskError('Cron 的步长格式不正确。')
|
||||
step=number(pieces[1],1,high-low+1,'Cron 步长') if len(pieces)==2 else 1
|
||||
base=pieces[0]
|
||||
if base=='*':start,end=low,high
|
||||
elif re.fullmatch(r'\d+-\d+',base):start,end=map(int,base.split('-'))
|
||||
elif base.isdigit():start=int(base);end=high if len(pieces)==2 else start
|
||||
else:raise TaskError('Cron 仅支持数字、*、范围、逗号和步长；不支持 L、W、# 或 ?。')
|
||||
if not low<=start<=end<=high:raise TaskError('Cron 字段数值超出允许范围。')
|
||||
result.update(range(start,end+1,step))
|
||||
return sorted(result)
|
||||
|
||||
def cron_calendars(expression):
|
||||
parts=expression.split()
|
||||
if len(parts)!=5:raise TaskError('Cron 需要五段：分 时 日 月 周，例如 0 3 * * *。')
|
||||
minute=cron_field(parts[0],0,59);hour=cron_field(parts[1],0,23)
|
||||
day=cron_field(parts[2],1,31)
|
||||
month=cron_field(parts[3],1,12,{x:i+1 for i,x in enumerate(['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC'])})
|
||||
dow=cron_field(parts[4],0,7,{'SUN':0,'MON':1,'TUE':2,'WED':3,'THU':4,'FRI':5,'SAT':6})
|
||||
join=lambda values:','.join(map(str,values))
|
||||
months='*' if parts[3]=='*' else join(month)
|
||||
hours='*' if parts[1]=='*' else join(hour)
|
||||
minutes='*' if parts[0]=='*' else join(minute)
|
||||
dates='*' if parts[2]=='*' else join(day)
|
||||
days=','.join(['Sun','Mon','Tue','Wed','Thu','Fri','Sat'][v] for v in sorted({v%7 for v in dow}))
|
||||
timepart=f'{hours}:{minutes}:00 Asia/Shanghai'
|
||||
if parts[4]=='*':return [f'*-{months}-{dates} {timepart}']
|
||||
if parts[2]=='*':return [f'{days} *-{months}-* {timepart}']
|
||||
# Traditional cron: restricted day-of-month and weekday are joined with OR.
|
||||
return [f'*-{months}-{dates} {timepart}',f'{days} *-{months}-* {timepart}']
|
||||
|
||||
def schedule_definition(schedule):
|
||||
if not isinstance(schedule,dict):raise TaskError('请设置执行周期。')
|
||||
kind=schedule.get('kind','daily')
|
||||
if kind=='daily':
|
||||
value=clock(schedule.get('time','03:00'))
|
||||
return [f'OnCalendar=*-*-* {value} Asia/Shanghai'],f'每天 {value[:5]}'
|
||||
if kind=='weekly':
|
||||
chosen=schedule.get('weekdays',[0])
|
||||
if not isinstance(chosen,list) or not chosen:raise TaskError('至少选择一个星期。')
|
||||
selected=sorted({number(v,0,6,'星期') for v in chosen})
|
||||
value=clock(schedule.get('time','03:00'))
|
||||
days=','.join(WEEKDAYS[v] for v in selected)
|
||||
label='、'.join('周'+['一','二','三','四','五','六','日'][v] for v in selected)
|
||||
return [f'OnCalendar={days} *-*-* {value} Asia/Shanghai'],f'每{label} {value[:5]}'
|
||||
if kind=='monthly':
|
||||
day=number(schedule.get('day',1),1,31,'每月日期');value=clock(schedule.get('time','03:00'))
|
||||
return [f'OnCalendar=*-*-{day} {value} Asia/Shanghai'],f'每月 {day} 日 {value[:5]}'+('（当月没有该日则跳过）' if day>28 else '')
|
||||
if kind in ('minutes','hours'):
|
||||
every=number(schedule.get('every',1),1,1440 if kind=='minutes' else 168,'间隔')
|
||||
seconds=every*(60 if kind=='minutes' else 3600)
|
||||
return [f'OnActiveSec={seconds}s',f'OnUnitInactiveSec={seconds}s'],f'每 {every} '+('分钟' if kind=='minutes' else '小时')+'（上次执行结束后重新计时）'
|
||||
if kind=='once':
|
||||
try:
|
||||
at=dt.datetime.fromisoformat(schedule.get('at',''))
|
||||
if at.tzinfo is None:at=at.replace(tzinfo=TZ)
|
||||
at=at.replace(second=0,microsecond=0)
|
||||
except (ValueError,TypeError):raise TaskError('请填写正确的指定日期与时间。')
|
||||
if at.timestamp()<time.time()+30:raise TaskError('指定执行时间必须至少晚于现在 30 秒。')
|
||||
return [f'OnCalendar={at.astimezone(TZ).strftime("%Y-%m-%d %H:%M:00")} Asia/Shanghai'],f'仅执行一次：{at.astimezone(TZ).strftime("%Y-%m-%d %H:%M")}'
|
||||
if kind=='cron':
|
||||
expression=str(schedule.get('expression','')).strip()
|
||||
if len(expression)>150:raise TaskError('Cron 表达式过长。')
|
||||
return ['OnCalendar='+x for x in cron_calendars(expression)],'Cron：'+expression
|
||||
raise TaskError('不支持的执行周期。')
|
||||
|
||||
def validate_definition(payload,check_paths=True):
|
||||
name=str(payload.get('name','')).strip()
|
||||
if not 1<=len(name)<=80 or any(ord(c)<32 for c in name):raise TaskError('任务名称需要 1 至 80 个字符，不能包含换行。')
|
||||
mode=payload.get('mode','command');interpreter=payload.get('interpreter','bash')
|
||||
if mode not in ('command','script','file'):raise TaskError('请选择命令、脚本内容或服务器脚本文件。')
|
||||
if interpreter not in ('bash','python3','auto'):raise TaskError('脚本语言不正确。')
|
||||
body=str(payload.get('body','')).replace('\r\n','\n').replace('\r','\n').lstrip('\ufeff').strip()
|
||||
path=str(payload.get('path','')).strip()
|
||||
arguments=str(payload.get('arguments','')).strip()
|
||||
if mode in ('command','script') and not body:raise TaskError('请输入要执行的命令或脚本。')
|
||||
if len(body.encode())>65536:raise TaskError('脚本内容不能超过 64 KiB。')
|
||||
if mode=='script' and interpreter=='auto':raise TaskError('脚本内容请选择 Bash 或 Python 3。')
|
||||
cwd=str(payload.get('cwd','/root')).strip() or '/root'
|
||||
if not PurePosixPath(cwd).is_absolute() or '\x00' in cwd:raise TaskError('工作目录必须是绝对路径。')
|
||||
if check_paths and not Path(cwd).is_dir():raise TaskError('工作目录不存在。')
|
||||
if mode=='file':
|
||||
if not PurePosixPath(path).is_absolute() or '\x00' in path:raise TaskError('脚本文件需要填写服务器上的绝对路径。')
|
||||
if check_paths and not Path(path).is_file():raise TaskError('服务器上找不到这个脚本文件。')
|
||||
if check_paths and interpreter=='auto' and not os.access(path,os.X_OK):raise TaskError('该文件没有执行权限，请选择 Bash／Python 3 解释器，或先设置文件执行权限。')
|
||||
try:shlex.split(arguments)
|
||||
except ValueError:raise TaskError('脚本参数的引号不匹配。')
|
||||
lines,label=schedule_definition(payload.get('schedule',{}))
|
||||
notes=str(payload.get('notes','')).strip()[:400]
|
||||
timeout=number(payload.get('timeout',1800),10,86400,'最长执行秒数')
|
||||
if not isinstance(payload.get('enabled',False),bool):raise TaskError('启用状态必须为是或否。')
|
||||
dangerous=bool(re.search(r'\b(reboot|shutdown|poweroff|halt)\b|\binit\s+[06]\b',body+' '+path,re.I))
|
||||
return {'name':name,'mode':mode,'interpreter':interpreter,'body':body,'path':path,'arguments':arguments,
|
||||
'cwd':cwd,'schedule':payload.get('schedule',{}),'schedule_lines':lines,'schedule_label':label,
|
||||
'timeout':timeout,'notes':notes,'enabled':bool(payload.get('enabled',False)),
|
||||
'dangerous':dangerous,'template':str(payload.get('template','custom'))[:40]}
|
||||
|
||||
def calendar_preview(definition):
|
||||
result=[]
|
||||
for line in definition['schedule_lines']:
|
||||
if line.startswith('OnCalendar='):
|
||||
p=execute(['systemd-analyze','calendar','--iterations=1',line.split('=',1)[1]],timeout=5,check=False)
|
||||
if p.returncode:raise TaskError('执行周期无效：'+(p.stderr or p.stdout)[-600:])
|
||||
result.append(p.stdout.strip())
|
||||
if result and all(re.search(r'Next elapse:\s*n/a',output) for output in result):
|
||||
raise TaskError('这个周期没有可以执行的将来时间，请检查日期或 Cron 表达式。')
|
||||
return result
|
||||
|
||||
def get_task(task_id,include_deleted=False):
|
||||
validate_id(task_id)
|
||||
with db() as con:
|
||||
row=con.execute('SELECT * FROM tasks WHERE id=?'+('' if include_deleted else ' AND deleted=0'),(task_id,)).fetchone()
|
||||
if not row:raise TaskError('找不到这个任务。',404)
|
||||
item=dict(row);item['definition']=json.loads(item['definition']);return item
|
||||
|
||||
def unit_name(task_id):return 'hwjob-'+validate_id(task_id)
|
||||
|
||||
def running(task_id):
|
||||
p=execute(['systemctl','is-active',unit_name(task_id)+'.service'],check=False,timeout=5)
|
||||
return p.stdout.strip() in ('active','activating','deactivating')
|
||||
|
||||
def render_units(task_id,definition):
|
||||
name=unit_name(task_id)
|
||||
service=f'''[Unit]
|
||||
Description=Dashboard scheduled task {task_id}
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
ExecStart=/usr/bin/python3 {APP}/task_exec.py {task_id}
|
||||
RuntimeMaxSec={definition['timeout']+10}
|
||||
TimeoutStopSec=15
|
||||
KillMode=mixed
|
||||
Restart=no
|
||||
UMask=0077
|
||||
Environment=TZ=Asia/Shanghai
|
||||
'''
|
||||
timer=f'''[Unit]
|
||||
Description=Dashboard task schedule {task_id}
|
||||
|
||||
[Timer]
|
||||
{chr(10).join(definition['schedule_lines'])}
|
||||
AccuracySec=1s
|
||||
Persistent=false
|
||||
Unit={name}.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
'''
|
||||
return service,timer
|
||||
|
||||
def install_units(task_id,definition):
|
||||
service,timer=render_units(task_id,definition)
|
||||
name=unit_name(task_id)
|
||||
atom_write(UNITS/(name+'.service'),service,0o644)
|
||||
atom_write(UNITS/(name+'.timer'),timer,0o644)
|
||||
execute(['systemctl','daemon-reload'])
|
||||
|
||||
def audit(action,task_id='',actor='admin',client='',detail=''):
|
||||
with db() as con:
|
||||
con.execute('INSERT INTO audit(at,action,task_id,actor,client,detail) VALUES(?,?,?,?,?,?)',
|
||||
(time.time(),action,task_id,actor,client,detail[:2000]))
|
||||
|
||||
def save_task(payload,client=''):
|
||||
task_id=payload.get('id')
|
||||
old=get_task(task_id) if task_id else None
|
||||
if old and running(task_id):raise TaskError('任务正在执行，请等待结束后再修改。',409)
|
||||
definition=validate_definition(payload);calendar_preview(definition)
|
||||
if not old:
|
||||
with db() as con:
|
||||
if con.execute('SELECT COUNT(*) FROM tasks WHERE deleted=0').fetchone()[0]>=100:raise TaskError('最多管理 100 个任务。')
|
||||
task_id=uuid.uuid4().hex
|
||||
if old:
|
||||
execute(['systemctl','disable','--now',unit_name(task_id)+'.timer'],check=False)
|
||||
if running(task_id):
|
||||
if old['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer'],check=False)
|
||||
raise TaskError('任务刚刚开始执行，请等待完成后再修改。',409)
|
||||
now=time.time();revision=old['revision']+1 if old else 1
|
||||
try:
|
||||
install_units(task_id,definition)
|
||||
with db() as con:
|
||||
con.execute('''INSERT INTO tasks VALUES(?,?,?,?,0,?,?,?) ON CONFLICT(id) DO UPDATE SET
|
||||
name=excluded.name,definition=excluded.definition,enabled=excluded.enabled,
|
||||
updated=excluded.updated,revision=excluded.revision''',
|
||||
(task_id,definition['name'],json.dumps(definition,ensure_ascii=False),int(definition['enabled']),
|
||||
old['created'] if old else now,now,revision))
|
||||
if definition['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer'])
|
||||
except Exception:
|
||||
if old:
|
||||
install_units(task_id,old['definition'])
|
||||
with db() as con:
|
||||
con.execute('UPDATE tasks SET name=?,definition=?,enabled=?,revision=? WHERE id=?',
|
||||
(old['name'],json.dumps(old['definition'],ensure_ascii=False),old['enabled'],old['revision'],task_id))
|
||||
if old['enabled']:execute(['systemctl','enable','--now',unit_name(task_id)+'.timer'],check=False)
|
||||
else:
|
||||
execute(['systemctl','disable','--now',unit_name(task_id)+'.timer'],check=False)
|
||||
with db() as con:con.execute('DELETE FROM tasks WHERE id=?',(task_id,))
|
||||
for suffix in ('.service','.timer'):(UNITS/(unit_name(task_id)+suffix)).unlink(missing_ok=True)
|
||||
execute(['systemctl','daemon-reload'],check=False)
|
||||
raise
|
||||
audit('update' if old else 'create',task_id,client=client,detail='revision '+str(revision))
|
||||
return {'id':task_id,'message':'任务已保存。'+('已启用，将按计划执行。' if definition['enabled'] else '当前停用，不会自动执行。')}
|
||||
|
||||
def set_enabled(task_id,enabled,client=''):
|
||||
task=get_task(task_id)
|
||||
if enabled:
|
||||
schedule_definition(task['definition']['schedule'])
|
||||
calendar_preview(task['definition'])
|
||||
execute(['systemctl','enable' if enabled else 'disable','--now',unit_name(task_id)+'.timer'])
|
||||
with db() as con:
|
||||
definition=task['definition'];definition['enabled']=bool(enabled)
|
||||
con.execute('UPDATE tasks SET enabled=?,definition=?,updated=? WHERE id=?',
|
||||
(int(bool(enabled)),json.dumps(definition,ensure_ascii=False),time.time(),task_id))
|
||||
audit('enable' if enabled else 'disable',task_id,client=client)
|
||||
return {'message':'已启用。' if enabled else '已停用后续调度；已经开始的执行不会被强制中断。'}
|
||||
|
||||
def delete_task(task_id,client=''):
|
||||
get_task(task_id)
|
||||
if running(task_id):raise TaskError('任务正在执行，请先等待完成或停止本次执行。',409)
|
||||
execute(['systemctl','disable','--now',unit_name(task_id)+'.timer'])
|
||||
for suffix in ('.service','.timer'):
|
||||
(UNITS/(unit_name(task_id)+suffix)).unlink(missing_ok=True)
|
||||
execute(['systemctl','daemon-reload'])
|
||||
with db() as con:con.execute('UPDATE tasks SET deleted=1,enabled=0,updated=? WHERE id=?',(time.time(),task_id))
|
||||
audit('delete',task_id,client=client)
|
||||
return {'message':'任务已删除，已有执行记录仍保留在服务器。'}
|
||||
|
||||
def run_now(task_id,client=''):
|
||||
task=get_task(task_id)
|
||||
if running(task_id):raise TaskError('这个任务正在执行，不会重复启动。',409)
|
||||
with db() as con:
|
||||
con.execute('INSERT OR REPLACE INTO requests VALUES(?,?,?)',(task_id,'manual',time.time()+30))
|
||||
execute(['systemctl','reset-failed',unit_name(task_id)+'.service'],check=False)
|
||||
execute(['systemctl','start','--no-block',unit_name(task_id)+'.service'])
|
||||
audit('run_now',task_id,client=client,detail='revision '+str(task['revision']))
|
||||
return {'message':'已提交执行，请在执行日志中查看结果。'}
|
||||
|
||||
def stop_task(task_id,client=''):
|
||||
get_task(task_id)
|
||||
execute(['systemctl','stop',unit_name(task_id)+'.service'])
|
||||
audit('stop_run',task_id,client=client)
|
||||
return {'message':'已请求停止本次执行。'}
|
||||
|
||||
def list_tasks():
|
||||
p=execute(['systemctl','list-timers','--all','--output=json','--no-pager','hwjob-*.timer'],check=False)
|
||||
try:timers={x['unit']:x for x in json.loads(p.stdout or '[]')}
|
||||
except (ValueError,TypeError):timers={}
|
||||
p=execute(['systemctl','list-units','--all','--type=service','--output=json','--no-pager','hwjob-*.service'],check=False)
|
||||
try:active={x['unit']:x for x in json.loads(p.stdout or '[]')}
|
||||
except (ValueError,TypeError):active={}
|
||||
with db() as con:
|
||||
tasks=con.execute('SELECT * FROM tasks WHERE deleted=0 ORDER BY created DESC').fetchall()
|
||||
result=[]
|
||||
for row in tasks:
|
||||
item=dict(row);definition=json.loads(item.pop('definition'))
|
||||
timer=timers.get(unit_name(item['id'])+'.timer',{})
|
||||
next_us=timer.get('next')
|
||||
item['next_run']=next_us/1e6 if isinstance(next_us,(int,float)) and 0<next_us<2**63 else None
|
||||
item['running']=active.get(unit_name(item['id'])+'.service',{}).get('active') in ('active','activating','deactivating')
|
||||
item['schedule_label']=definition['schedule_label'];item['mode']=definition['mode']
|
||||
item['dangerous']=definition.get('dangerous',False);item['notes']=definition.get('notes','')
|
||||
last=con.execute('SELECT * FROM runs WHERE task_id=? ORDER BY started DESC LIMIT 1',(item['id'],)).fetchone()
|
||||
item['last_run']=dict(last) if last else None
|
||||
result.append(item)
|
||||
return {'tasks':result,'server_time':iso(),'timezone':'Asia/Shanghai'}
|
||||
|
||||
def runs_for(task_id):
|
||||
get_task(task_id,True)
|
||||
with db() as con:
|
||||
rows=con.execute('SELECT * FROM runs WHERE task_id=? ORDER BY started DESC LIMIT 40',(task_id,)).fetchall()
|
||||
return {'runs':[dict(row) for row in rows]}
|
||||
|
||||
def read_run(run_id):
|
||||
validate_id(run_id)
|
||||
with db() as con:row=con.execute('SELECT * FROM runs WHERE id=?',(run_id,)).fetchone()
|
||||
if not row:raise TaskError('找不到这次执行记录。',404)
|
||||
path=LOGS/(run_id+'.log')
|
||||
try:
|
||||
with path.open('rb') as f:
|
||||
f.seek(0,2);size=f.tell();f.seek(max(0,size-131072));content=f.read().decode('utf-8',errors='replace')
|
||||
except OSError:
|
||||
content='本次输出已按保留规则清理，执行结果仍保留。' if row['state']!='running' else '日志尚未写入。'
|
||||
size=0
|
||||
return {'run':dict(row),'output':content,'tail_only':size>131072}
|
||||
104
task_exec.py
Normal file
104
task_exec.py
Normal file
@ -0,0 +1,104 @@
|
||||
#!/usr/bin/python3
|
||||
"""Run one approved, root-owned task definition; record bounded output and result."""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import task_core as core
|
||||
|
||||
LIMIT=512*1024
|
||||
|
||||
def main(task_id):
|
||||
task=core.get_task(task_id)
|
||||
definition=task['definition']
|
||||
lock=(core.BASE/(task_id+'.lock')).open('a')
|
||||
try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
except BlockingIOError:return 0
|
||||
run_id=uuid.uuid4().hex;started=time.time();start_mono=time.monotonic()
|
||||
boot=core.read_boot() if hasattr(core,'read_boot') else open('/proc/sys/kernel/random/boot_id').read().strip()
|
||||
with core.db() as con:
|
||||
requested=con.execute('SELECT * FROM requests WHERE task_id=?',(task_id,)).fetchone()
|
||||
trigger='manual' if requested and requested['expires']>=started else 'scheduled'
|
||||
con.execute('DELETE FROM requests WHERE task_id=?',(task_id,))
|
||||
body=definition['body']
|
||||
fingerprint=hashlib.sha256((body if definition['mode']!='file' else definition['path']+' '+definition['arguments']).encode()).hexdigest()
|
||||
con.execute('INSERT INTO runs(id,task_id,task_name,started,state,trigger,boot,script_hash,revision) VALUES(?,?,?,?,?,?,?,?,?)',
|
||||
(run_id,task_id,task['name'],started,'running',trigger,boot,fingerprint,task['revision']))
|
||||
cancelled=[False]
|
||||
def stop(*_):cancelled[0]=True
|
||||
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
||||
state='failed';exit_code=None;proc=None;written=0;truncated=False
|
||||
log_path=core.LOGS/(run_id+'.log')
|
||||
with log_path.open('wb',buffering=0) as output:
|
||||
os.chmod(log_path,0o600)
|
||||
def write(data):
|
||||
nonlocal written,truncated
|
||||
if isinstance(data,str):data=data.encode('utf-8')
|
||||
available=max(0,LIMIT-written)
|
||||
if available:
|
||||
piece=data[:available];output.write(piece);written+=len(piece)
|
||||
if len(data)>available:truncated=True
|
||||
write(f'开始：{core.iso(started)}\n任务：{task["name"]}\n触发：{"手动" if trigger=="manual" else "定时"}\n执行身份：root\n修订：{task["revision"]}\n'+'─'*32+'\n')
|
||||
try:
|
||||
if definition['mode']=='command':
|
||||
argv=['/bin/bash','-e','-o','pipefail','-c',body]
|
||||
elif definition['mode']=='script':
|
||||
script=core.BASE/'scripts'/(run_id+('.py' if definition['interpreter']=='python3' else '.sh'))
|
||||
core.atom_write(script,body+'\n')
|
||||
argv=['/usr/bin/python3',str(script)] if definition['interpreter']=='python3' else ['/bin/bash','-e','-o','pipefail',str(script)]
|
||||
else:
|
||||
argv=[definition['path'],*shlex.split(definition['arguments'])]
|
||||
if definition['interpreter']=='bash':argv=['/bin/bash','-e','-o','pipefail',*argv]
|
||||
elif definition['interpreter']=='python3':argv=['/usr/bin/python3',*argv]
|
||||
env={'HOME':'/root','USER':'root','LOGNAME':'root','LANG':'C.UTF-8','TZ':'Asia/Shanghai',
|
||||
'PATH':'/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin',
|
||||
'HARDWARE_TASK_ID':task_id,'HARDWARE_TASK_RUN_ID':run_id}
|
||||
proc=subprocess.Popen(argv,cwd=definition['cwd'],env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
selector=selectors.DefaultSelector();selector.register(proc.stdout,selectors.EVENT_READ)
|
||||
deadline=start_mono+definition['timeout'];exited_at=None
|
||||
while selector.get_map() or proc.poll() is None:
|
||||
if cancelled[0] or time.monotonic()>=deadline:
|
||||
state='cancelled' if cancelled[0] else 'timeout'
|
||||
try:os.killpg(proc.pid,signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
try:proc.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:os.killpg(proc.pid,signal.SIGKILL)
|
||||
except ProcessLookupError:pass
|
||||
proc.wait(timeout=3)
|
||||
break
|
||||
for key,_ in selector.select(.25):
|
||||
chunk=os.read(key.fileobj.fileno(),32768)
|
||||
if chunk:write(chunk)
|
||||
else:selector.unregister(key.fileobj)
|
||||
if proc.poll() is not None:
|
||||
exited_at=exited_at or time.monotonic()
|
||||
if time.monotonic()-exited_at>1:break
|
||||
selector.close();proc.stdout.close()
|
||||
exit_code=proc.wait(timeout=3)
|
||||
if state not in ('cancelled','timeout'):state='success' if exit_code==0 else 'failed'
|
||||
except Exception as exc:
|
||||
write('\n执行错误：'+str(exc)+'\n');state='failed'
|
||||
if proc and proc.poll() is None:
|
||||
try:os.killpg(proc.pid,signal.SIGKILL);proc.wait(timeout=3)
|
||||
except (ProcessLookupError,subprocess.TimeoutExpired):pass
|
||||
elapsed=time.monotonic()-start_mono
|
||||
tail=f'\n'+('─'*32)+f'\n结束：{core.iso()}\n结果：{state}\n退出码：{exit_code}\n耗时：{elapsed:.2f} 秒\n'
|
||||
if truncated:output.write('\n[输出超过 512 KiB，后续内容已省略。]\n'.encode())
|
||||
output.write(tail.encode())
|
||||
os.fsync(output.fileno())
|
||||
with core.db() as con:
|
||||
con.execute('UPDATE runs SET finished=?,state=?,exit_code=?,log_bytes=?,truncated=?,duration=? WHERE id=?',
|
||||
(time.time(),state,exit_code,log_path.stat().st_size,int(truncated),elapsed,run_id))
|
||||
lock.close()
|
||||
return 0 if state=='success' else 1
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(main(core.validate_id(sys.argv[1])))
|
||||
193
taskd.py
Normal file
193
taskd.py
Normal file
@ -0,0 +1,193 @@
|
||||
#!/usr/bin/python3
|
||||
"""Authenticated local task broker. Only the web service UID may connect to its socket."""
|
||||
import argparse
|
||||
import getpass
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
try:import pwd
|
||||
except ImportError:pwd=None
|
||||
import secrets
|
||||
import socket
|
||||
import socketserver
|
||||
import struct
|
||||
import re
|
||||
import datetime as dt
|
||||
import threading
|
||||
import time
|
||||
import task_core as core
|
||||
|
||||
ADMIN=Path('/etc/hardware-task-admin.json')
|
||||
SOCKET=Path('/run/hardware-taskd/control.sock')
|
||||
SESSIONS={};ATTEMPTS={};GUARD=threading.RLock();MUTATE=threading.RLock()
|
||||
ALLOWED_UID=None
|
||||
|
||||
def password_record(password):
|
||||
salt=secrets.token_hex(16)
|
||||
return {'username':'admin','salt':salt,'hash':hashlib.pbkdf2_hmac('sha256',password.encode(),bytes.fromhex(salt),240000).hex()}
|
||||
|
||||
def verify(password):
|
||||
try:
|
||||
record=json.loads(ADMIN.read_text())
|
||||
hashed=hashlib.pbkdf2_hmac('sha256',password.encode(),bytes.fromhex(record['salt']),240000).hex()
|
||||
return hmac.compare_digest(record['hash'],hashed)
|
||||
except (OSError,ValueError,KeyError):return False
|
||||
|
||||
def create_admin():
|
||||
if ADMIN.exists():return
|
||||
password=secrets.token_urlsafe(18)
|
||||
core.atom_write(ADMIN,json.dumps(password_record(password)))
|
||||
core.atom_write(Path('/root/hardware-task-admin-login.txt'),
|
||||
'定时任务管理员登录信息\n\n网页：http://192.168.1.37:8090/#tasks\n账号：admin\n初始密码：'+password+
|
||||
'\n\n此账号可以管理以 root 身份执行的服务器任务，请妥善保存。登录后可修改管理密码。\nSSH root 密码未被修改，也没有作为网页管理密码保存。\n')
|
||||
print('初始登录信息已保存到 /root/hardware-task-admin-login.txt（仅 root 可读）。')
|
||||
|
||||
def session(token,csrf=None,write=False):
|
||||
with GUARD:
|
||||
record=SESSIONS.get(token)
|
||||
if not record or record['expires']<time.monotonic():
|
||||
SESSIONS.pop(token,None);raise core.TaskError('请先登录定时任务管理。',401)
|
||||
if write and (not isinstance(csrf,str) or not hmac.compare_digest(record['csrf'],csrf)):
|
||||
raise core.TaskError('操作验证已失效，请重新登录后再试。',403)
|
||||
return record
|
||||
|
||||
def login(payload,client):
|
||||
now=time.monotonic()
|
||||
with GUARD:
|
||||
recent=[x for x in ATTEMPTS.get(client,[]) if now-x<300]
|
||||
if len(recent)>=6:raise core.TaskError('登录失败次数较多，请五分钟后重试。',429)
|
||||
if payload.get('username')!='admin' or not verify(str(payload.get('password',''))):
|
||||
ATTEMPTS[client]=recent+[now]
|
||||
core.audit('login_failed',client=client)
|
||||
raise core.TaskError('账号或密码不正确。',401)
|
||||
ATTEMPTS.pop(client,None)
|
||||
if len(SESSIONS)>200:
|
||||
for key in list(SESSIONS):
|
||||
if SESSIONS[key]['expires']<now:SESSIONS.pop(key,None)
|
||||
token=secrets.token_urlsafe(32);csrf=secrets.token_urlsafe(24)
|
||||
SESSIONS[token]={'username':'admin','csrf':csrf,'expires':now+8*3600}
|
||||
core.audit('login',client=client)
|
||||
return {'token':token,'csrf':csrf,'logged_in':True,'username':'admin','max_age':8*3600}
|
||||
|
||||
def dispatch(request):
|
||||
action=request.get('action','');payload=request.get('payload') or {};token=request.get('token','');client=str(request.get('client','local'))[:100]
|
||||
if not isinstance(payload,dict):raise core.TaskError('请求格式不正确。')
|
||||
if action=='auth.login':return login(payload,client)
|
||||
if action=='auth.status':
|
||||
try:record=session(token)
|
||||
except core.TaskError:return {'logged_in':False,'configured':ADMIN.exists()}
|
||||
return {'logged_in':True,'username':record['username'],'csrf':record['csrf']}
|
||||
write=action not in ('tasks.list','tasks.get','tasks.runs','tasks.runlog','tasks.history')
|
||||
session(token,request.get('csrf'),write)
|
||||
if action=='auth.logout':
|
||||
with GUARD:SESSIONS.pop(token,None)
|
||||
return {'message':'已退出任务管理。'}
|
||||
if action=='auth.password':
|
||||
new=str(payload.get('new_password',''))
|
||||
if not 12<=len(new)<=200:raise core.TaskError('新密码长度需要 12 至 200 个字符。')
|
||||
if not verify(str(payload.get('old_password',''))):raise core.TaskError('原管理密码不正确。',400)
|
||||
core.atom_write(ADMIN,json.dumps(password_record(new)))
|
||||
with GUARD:SESSIONS.clear()
|
||||
core.audit('password_changed',client=client)
|
||||
return {'message':'管理密码已修改，请重新登录。'}
|
||||
if action=='tasks.list':return core.list_tasks()
|
||||
if action=='tasks.get':return core.get_task(payload.get('id'))
|
||||
if action=='tasks.runs':return core.runs_for(payload.get('id'))
|
||||
if action=='tasks.runlog':return core.read_run(payload.get('run_id'))
|
||||
if action=='tasks.history':
|
||||
with core.db() as con:rows=con.execute('SELECT * FROM runs ORDER BY started DESC LIMIT 80').fetchall()
|
||||
return {'runs':[dict(row) for row in rows]}
|
||||
if action=='tasks.preview':
|
||||
definition=core.validate_definition(payload)
|
||||
calendars=core.calendar_preview(definition)
|
||||
next_times=[]
|
||||
for output in calendars:
|
||||
match=re.search(r'Next elapse:.*?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})',output)
|
||||
if match:next_times.append(dt.datetime.strptime(match[1],'%Y-%m-%d %H:%M:%S').replace(tzinfo=core.TZ).timestamp())
|
||||
if not calendars:
|
||||
next_times=[time.time()+int(definition['schedule_lines'][0].split('=')[1][:-1])]
|
||||
return {'schedule_label':definition['schedule_label'],'next_run':min(next_times) if next_times else None,'dangerous':definition['dangerous']}
|
||||
with MUTATE:
|
||||
if action=='tasks.save':return core.save_task(payload,client)
|
||||
if action=='tasks.enable':
|
||||
if not isinstance(payload.get('enabled'),bool):raise core.TaskError('启用状态格式不正确。')
|
||||
return core.set_enabled(payload.get('id'),payload['enabled'],client)
|
||||
if action=='tasks.delete':return core.delete_task(payload.get('id'),client)
|
||||
if action=='tasks.run':return core.run_now(payload.get('id'),client)
|
||||
if action=='tasks.stop':return core.stop_task(payload.get('id'),client)
|
||||
raise core.TaskError('不支持这个任务操作。',404)
|
||||
|
||||
class Handler(socketserver.StreamRequestHandler):
|
||||
def handle(self):
|
||||
self.request.settimeout(25)
|
||||
try:
|
||||
_,uid,_=struct.unpack('3i',self.request.getsockopt(socket.SOL_SOCKET,socket.SO_PEERCRED,12))
|
||||
if uid not in (0,ALLOWED_UID):raise core.TaskError('不允许连接任务服务。',403)
|
||||
raw=self.rfile.readline(131073)
|
||||
if len(raw)>131072:raise core.TaskError('请求内容过大。',413)
|
||||
request=json.loads(raw)
|
||||
response={'ok':True,'data':dispatch(request)}
|
||||
except core.TaskError as exc:response={'ok':False,'error':str(exc),'status':exc.status}
|
||||
except Exception as exc:
|
||||
print('task broker error:',type(exc).__name__,str(exc),flush=True)
|
||||
response={'ok':False,'error':'任务操作未完成，请查看任务管理服务日志。','status':500}
|
||||
try:self.wfile.write(json.dumps(response,ensure_ascii=False).encode()+b'\n')
|
||||
except (BrokenPipeError,ConnectionResetError):pass
|
||||
|
||||
class Server(getattr(socketserver,'ThreadingUnixStreamServer',socketserver.ThreadingTCPServer)):
|
||||
daemon_threads=True
|
||||
|
||||
def maintenance():
|
||||
while True:
|
||||
try:
|
||||
with core.db() as con:
|
||||
rows=con.execute("SELECT * FROM runs WHERE state='running'").fetchall()
|
||||
boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
for row in rows:
|
||||
if row['boot']!=boot or (time.time()-row['started']>30 and not core.running(row['task_id'])):
|
||||
with core.db() as con:
|
||||
con.execute("UPDATE runs SET state='interrupted',finished=? WHERE id=? AND state='running'",(time.time(),row['id']))
|
||||
with (core.LOGS/(row['id']+'.log')).open('ab') as f:
|
||||
f.write(('\n[执行被系统重启或进程中断，没有取得完整退出结果。]\n').encode())
|
||||
cutoff=time.time()-90*86400
|
||||
with core.db() as con:
|
||||
old=con.execute('SELECT id FROM runs WHERE started<? AND state!=?',(cutoff,'running')).fetchall()
|
||||
for row in old:
|
||||
(core.LOGS/(row['id']+'.log')).unlink(missing_ok=True)
|
||||
for suffix in ('.sh','.py'):(core.BASE/'scripts'/(row['id']+suffix)).unlink(missing_ok=True)
|
||||
con.execute("DELETE FROM runs WHERE started<? AND state!='running'",(cutoff,))
|
||||
con.execute('DELETE FROM audit WHERE at<?',(cutoff,))
|
||||
protected={x['id'] for x in con.execute("SELECT id FROM runs WHERE state='running'")}
|
||||
files=[p for folder in (core.LOGS,core.BASE/'scripts') for p in folder.iterdir()
|
||||
if p.is_file() and re.fullmatch(r'[0-9a-f]{32}\.(log|sh|py)',p.name)]
|
||||
files.sort(key=lambda p:p.stat().st_mtime)
|
||||
total=sum(p.stat().st_size for p in files)
|
||||
for path in files:
|
||||
if total<=256*1024*1024:break
|
||||
if path.stem in protected:continue
|
||||
total-=path.stat().st_size;path.unlink()
|
||||
if path.suffix=='.log':con.execute('UPDATE runs SET log_purged=1 WHERE id=?',(path.stem,))
|
||||
except Exception as exc:print('task maintenance:',type(exc).__name__,str(exc),flush=True)
|
||||
time.sleep(60)
|
||||
|
||||
def main():
|
||||
global ALLOWED_UID
|
||||
parser=argparse.ArgumentParser();parser.add_argument('mode',choices=['serve','init-admin','password'],default='serve',nargs='?');args=parser.parse_args()
|
||||
core.init_store()
|
||||
if args.mode=='init-admin':create_admin();return
|
||||
if args.mode=='password':
|
||||
password=getpass.getpass('新管理密码（至少12字符）：')
|
||||
if len(password)<12 or password!=getpass.getpass('再次输入：'):raise SystemExit('密码太短或两次输入不一致。')
|
||||
core.atom_write(ADMIN,json.dumps(password_record(password)));print('密码已重置；请重启 hardware-taskd 服务使旧登录失效。');return
|
||||
if not ADMIN.exists():raise SystemExit('请先运行 taskd.py init-admin。')
|
||||
ALLOWED_UID=pwd.getpwnam('hardware-dashboard').pw_uid
|
||||
SOCKET.parent.mkdir(parents=True,exist_ok=True)
|
||||
SOCKET.unlink(missing_ok=True)
|
||||
server=Server(str(SOCKET),Handler);os.chmod(SOCKET,0o660)
|
||||
threading.Thread(target=maintenance,daemon=True).start()
|
||||
print('Task broker ready on local Unix socket.',flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
if __name__=='__main__':main()
|
||||
147
tests/test_dashboard.py
Normal file
147
tests/test_dashboard.py
Normal file
@ -0,0 +1,147 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
spec=importlib.util.spec_from_file_location('dashboard',Path(__file__).resolve().parents[1]/'server.py')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
|
||||
class DashboardTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory(prefix='dashboard-qa-')
|
||||
self.old=(m.DATA,m.LOGS)
|
||||
m.DATA=Path(self.temp.name)/'state';m.LOGS=Path(self.temp.name)/'logs';m.LOGS.mkdir()
|
||||
m.initialize()
|
||||
m.LATEST.clear();m.LATEST.update({'ts':time.time(),'boot_id':'test-current','monotonic_s':1000,'issues':[]})
|
||||
|
||||
def tearDown(self):
|
||||
m.DATA,m.LOGS=self.old;self.temp.cleanup()
|
||||
|
||||
def test_import_replay_and_clock_adjustment(self):
|
||||
now=int(time.time())
|
||||
row={'observed_at':m.iso(now),'first_at':m.iso(now+3600),'last_at':m.iso(now-3600),
|
||||
'first_monotonic_us':'11000000','last_monotonic_us':'487000000',
|
||||
'boot_id':'oldboot','source':'kernel','category':'power','severity':'ERROR','count':93,
|
||||
'device':'0000:84:00.0','detail':'GPU does not have the necessary power cables connected.'}
|
||||
p=m.LOGS/'events-test.jsonl';p.write_text(json.dumps(row)+'\n'+json.dumps(row)+'\n')
|
||||
with m.connect() as db:
|
||||
m.import_logs(db);m.import_logs(db)
|
||||
self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],1)
|
||||
result=m.events({'days':['7']})
|
||||
self.assertEqual(result['total'],1)
|
||||
self.assertEqual(result['items'][0]['state'],'history')
|
||||
self.assertAlmostEqual(result['items'][0]['first_ts'],now-3600-476,delta=1)
|
||||
self.assertTrue(result['items'][0]['time_note'])
|
||||
self.assertEqual(result['items'][0]['raw']['first_at'],row['first_at'])
|
||||
|
||||
def test_history_does_not_invent_old_cpu_utilization(self):
|
||||
now=int(time.time())-40
|
||||
with m.connect() as db:
|
||||
db.execute('INSERT INTO metrics VALUES(?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,?,?)',
|
||||
(now-1500,json.dumps([{'uuid':'gpu-a','name':'P100','temperature_c':40,'power_w':30}]),'legacy'))
|
||||
for offset,cpu in [(0,40),(10,80)]:
|
||||
db.execute('INSERT INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)',
|
||||
(now+offset,cpu,60,45,1,2,3,4,json.dumps([{'uuid':'gpu-a','name':'P100','utilization':90,'temperature_c':50,'power_w':100}]),'live'))
|
||||
data=m.history('1h')
|
||||
legacy=[p for p in data['points'] if not p['live']]
|
||||
self.assertTrue(legacy)
|
||||
self.assertIsNone(legacy[0]['cpu'])
|
||||
self.assertIsNone(legacy[0]['gpus']['gpu-a']['utilization'])
|
||||
self.assertEqual(legacy[0]['gpus']['gpu-a']['temperature_c'],40)
|
||||
self.assertEqual(data['devices']['gpu-a'],'P100')
|
||||
self.assertTrue(any(p['cpu'] is not None for p in data['points']))
|
||||
|
||||
def test_filters_and_partial_tail(self):
|
||||
now=int(time.time())
|
||||
row={'observed_at':m.iso(now),'category':'storage','severity':'ERROR','detail':'nvme timeout'}
|
||||
p=m.LOGS/'events-tail.jsonl';p.write_text(json.dumps(row))
|
||||
with m.connect() as db:
|
||||
m.import_logs(db)
|
||||
self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],0)
|
||||
with p.open('a') as f:f.write('\n')
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
self.assertEqual(m.events({'category':['storage']})['total'],1)
|
||||
self.assertEqual(m.events({'category':['power']})['total'],0)
|
||||
self.assertEqual(m.events({'q':['timeout']})['total'],1)
|
||||
|
||||
def test_stale_status_is_unknown_not_recovered(self):
|
||||
m.LATEST.update({'ts':time.time()-300})
|
||||
now=int(time.time())
|
||||
row={'observed_at':m.iso(now),'category':'power','severity':'ERROR','detail':'power error'}
|
||||
(m.LOGS/'events-stale.jsonl').write_text(json.dumps(row)+'\n')
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
self.assertEqual(m.events({})['items'][0]['state'],'unknown')
|
||||
|
||||
def test_link_reports_merge_and_keep_two_cards_separate(self):
|
||||
now=int(time.time())
|
||||
a={'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'}
|
||||
b={'id':'gpu-b','label':'第 2 张 P100','pci':'0000:84:00.0'}
|
||||
m.LATEST.update({'device_map':{'0000:00:03.0':a,'0000:03:00.0':a,'0000:80:03.0':b},
|
||||
'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2})
|
||||
def row(device,detail,timestamp,mono):
|
||||
return {'observed_at':m.iso(now),'first_at':m.iso(timestamp),'last_at':m.iso(timestamp),
|
||||
'first_monotonic_us':str(mono),'last_monotonic_us':str(mono),
|
||||
'boot_id':'test-current','source':'kernel','category':'pcie_correctable','severity':'WARN',
|
||||
'device':device,'count':1,'detail':detail}
|
||||
rows=[row('0000:00:03.0','PCIe Bus Error: severity=Correctable',now-50,950000000),
|
||||
row('0000:00:03.0','[6] BadTLP',now-50,950000001),
|
||||
row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-50,950000002),
|
||||
row('0000:80:03.0','PCIe Bus Error: severity=Correctable',now-40,960000000)]
|
||||
path=m.LOGS/'events-grouped.jsonl';path.write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
result=m.events({})
|
||||
self.assertEqual(result['raw_total'],4)
|
||||
self.assertEqual(result['total'],2)
|
||||
first=next(x for x in result['items'] if x['component_id']=='gpu-a')
|
||||
self.assertEqual(first['record_count'],3)
|
||||
self.assertEqual(first['signal_reports'],2)
|
||||
self.assertEqual(m.events({'view':['raw']})['total'],4)
|
||||
original_id=first['id']
|
||||
with path.open('a') as f:f.write(json.dumps(row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-10,990000000))+'\n')
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
new=next(x for x in m.events({})['items'] if x['component_id']=='gpu-a')
|
||||
self.assertEqual(new['id'],original_id)
|
||||
self.assertEqual(new['signal_reports'],3)
|
||||
|
||||
def test_five_minute_quiet_gap_starts_another_incident(self):
|
||||
now=int(time.time())
|
||||
rows=[{'observed_at':m.iso(now),'first_at':m.iso(now-age),'last_at':m.iso(now-age),
|
||||
'source':'kernel','boot_id':'oldboot','first_monotonic_us':str(1000000+i),
|
||||
'last_monotonic_us':str(1000000+i),'device':'0000:03:00.0',
|
||||
'category':'pcie_correctable','severity':'WARN','count':1,
|
||||
'detail':'PCIe Bus Error: severity=Correctable'} for i,age in enumerate([1000,10])]
|
||||
(m.LOGS/'events-gaps.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
self.assertEqual(m.events({})['total'],2)
|
||||
|
||||
def test_current_retry_is_warning_and_has_plain_action(self):
|
||||
now=int(time.time())
|
||||
m.LATEST.update({'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2,
|
||||
'issues':[{'category':'pcie_correctable','device':'0000:00:03.0'}],
|
||||
'device_map':{'0000:00:03.0':{'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'}}})
|
||||
row={'observed_at':m.iso(now),'first_at':m.iso(now-10),'last_at':m.iso(now-10),
|
||||
'source':'kernel','boot_id':'test-current','first_monotonic_us':'990000000',
|
||||
'last_monotonic_us':'990000000','device':'0000:00:03.0',
|
||||
'category':'pcie_correctable','severity':'WARN','count':1,
|
||||
'detail':'PCIe Bus Error: severity=Correctable'}
|
||||
(m.LOGS/'events-active.jsonl').write_text(json.dumps(row)+'\n')
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
item=m.events({})['items'][0]
|
||||
self.assertEqual(item['severity'],'WARN')
|
||||
self.assertEqual(item['state'],'active')
|
||||
self.assertIn('显卡可用',item['state_label'])
|
||||
self.assertIn('不能证明',item['meaning'])
|
||||
self.assertEqual(len(item['next_steps']),3)
|
||||
|
||||
def test_problem_filter_hides_routine_information(self):
|
||||
now=int(time.time())
|
||||
rows=[{'observed_at':m.iso(now),'category':cat,'severity':severity,'detail':cat}
|
||||
for cat,severity in [('boot','INFO'),('power','ERROR')]]
|
||||
(m.LOGS/'events-levels.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
||||
with m.connect() as db:m.import_logs(db)
|
||||
self.assertEqual(m.events({'severity':['problem']})['total'],1)
|
||||
self.assertEqual(m.events({'severity':['INFO']})['total'],1)
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
99
tests/test_tasks.py
Normal file
99
tests/test_tasks.py
Normal file
@ -0,0 +1,99 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import task_core as core
|
||||
import taskd
|
||||
|
||||
class TaskTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory(prefix='scheduled-task-qa-')
|
||||
self.original=(core.BASE,core.LOGS,core.UNITS,taskd.ADMIN)
|
||||
folder=Path(self.temp.name)
|
||||
core.BASE=folder/'data';core.LOGS=folder/'logs';core.UNITS=folder/'units';core.UNITS.mkdir()
|
||||
taskd.ADMIN=folder/'admin.json';taskd.SESSIONS.clear();taskd.ATTEMPTS.clear();core.init_store()
|
||||
core.atom_write(taskd.ADMIN,json.dumps(taskd.password_record('only-a-test-password')))
|
||||
|
||||
def tearDown(self):
|
||||
core.BASE,core.LOGS,core.UNITS,taskd.ADMIN=self.original
|
||||
taskd.SESSIONS.clear();taskd.ATTEMPTS.clear();self.temp.cleanup()
|
||||
|
||||
def definition(self,**values):
|
||||
return {'name':'测试任务','mode':'command','body':'printf hello','enabled':False,
|
||||
'schedule':{'kind':'daily','time':'03:00'},**values}
|
||||
|
||||
def test_daily_weekly_monthly_and_interval(self):
|
||||
self.assertEqual(core.schedule_definition({'kind':'daily','time':'03:15'})[0],['OnCalendar=*-*-* 03:15:00 Asia/Shanghai'])
|
||||
self.assertIn('Mon,Sun',core.schedule_definition({'kind':'weekly','weekdays':[0,6],'time':'04:00'})[0][0])
|
||||
self.assertIn('跳过',core.schedule_definition({'kind':'monthly','day':31,'time':'02:00'})[1])
|
||||
interval=core.schedule_definition({'kind':'minutes','every':10})[0]
|
||||
self.assertEqual(interval,['OnActiveSec=600s','OnUnitInactiveSec=600s'])
|
||||
self.assertFalse(any('OnBootSec' in x for x in interval))
|
||||
|
||||
def test_cron_lists_ranges_steps_and_day_or(self):
|
||||
output=core.cron_calendars('*/15 3 * * MON-FRI')
|
||||
self.assertEqual(len(output),1)
|
||||
self.assertIn('0,15,30,45:00',output[0])
|
||||
self.assertIn('Mon,Tue,Wed,Thu,Fri',output[0])
|
||||
output=core.cron_calendars('0 3 1 * 1')
|
||||
self.assertEqual(len(output),2)
|
||||
self.assertIn('*-*-1',output[0]);self.assertIn('Mon',output[1])
|
||||
for bad in ['0 3 * *','*/0 * * * *','0 24 * * *','0 3 32 * *','0 3 * * 9','0 3 L * *']:
|
||||
with self.subTest(bad=bad),self.assertRaises(core.TaskError):core.cron_calendars(bad)
|
||||
|
||||
def test_task_text_never_becomes_unit_instructions(self):
|
||||
definition=core.validate_definition(self.definition(body='echo test\n# ExecStart=/bad/example'),check_paths=False)
|
||||
service,timer=core.render_units('a'*32,definition)
|
||||
self.assertNotIn('echo test',service);self.assertNotIn('/bad/example',service)
|
||||
self.assertIn('task_exec.py '+'a'*32,service)
|
||||
self.assertIn('Persistent=false',timer)
|
||||
with self.assertRaises(core.TaskError):core.validate_definition(self.definition(name='name\nExecStart=bad'),check_paths=False)
|
||||
with self.assertRaises(core.TaskError):core.validate_id('../../etc/shadow')
|
||||
|
||||
def test_validation_limits_and_past_dates(self):
|
||||
for changes in [{'body':''},{'timeout':1},{'timeout':90000},{'body':'x'*65537},{'enabled':'false'},
|
||||
{'schedule':{'kind':'once','at':'2020-01-01T03:00'}}]:
|
||||
with self.subTest(changes=list(changes)),self.assertRaises(core.TaskError):core.validate_definition(self.definition(**changes),check_paths=False)
|
||||
self.assertTrue(core.validate_definition(self.definition(body='systemctl reboot'),check_paths=False)['dangerous'])
|
||||
|
||||
def test_session_and_csrf_gate_privileged_operations(self):
|
||||
with patch.object(core,'save_task') as save:
|
||||
with self.assertRaises(core.TaskError) as error:taskd.dispatch({'action':'tasks.save','payload':self.definition()})
|
||||
self.assertEqual(error.exception.status,401);save.assert_not_called()
|
||||
login=taskd.login({'username':'admin','password':'only-a-test-password'},'qa')
|
||||
with patch.object(core,'set_enabled') as toggle:
|
||||
with self.assertRaises(core.TaskError) as error:taskd.dispatch({'action':'tasks.enable','token':login['token'],'csrf':'wrong','payload':{'id':'a'*32,'enabled':True}})
|
||||
self.assertEqual(error.exception.status,403);toggle.assert_not_called()
|
||||
self.assertTrue(taskd.dispatch({'action':'auth.status','token':login['token']})['logged_in'])
|
||||
taskd.dispatch({'action':'auth.logout','token':login['token'],'csrf':login['csrf']})
|
||||
self.assertFalse(taskd.dispatch({'action':'auth.status','token':login['token']})['logged_in'])
|
||||
|
||||
def test_password_change_revokes_all_sessions(self):
|
||||
one=taskd.login({'username':'admin','password':'only-a-test-password'},'qa')
|
||||
two=taskd.login({'username':'admin','password':'only-a-test-password'},'qa-2')
|
||||
taskd.dispatch({'action':'auth.password','token':one['token'],'csrf':one['csrf'],
|
||||
'payload':{'old_password':'only-a-test-password','new_password':'new-test-password-123'}})
|
||||
self.assertFalse(taskd.verify('only-a-test-password'));self.assertTrue(taskd.verify('new-test-password-123'))
|
||||
self.assertFalse(taskd.dispatch({'action':'auth.status','token':two['token']})['logged_in'])
|
||||
|
||||
def test_wrong_old_password_keeps_current_login(self):
|
||||
login=taskd.login({'username':'admin','password':'only-a-test-password'},'qa')
|
||||
with self.assertRaises(core.TaskError) as error:
|
||||
taskd.dispatch({'action':'auth.password','token':login['token'],'csrf':login['csrf'],
|
||||
'payload':{'old_password':'wrong-password','new_password':'new-test-password-123'}})
|
||||
self.assertEqual(error.exception.status,400)
|
||||
self.assertTrue(taskd.dispatch({'action':'auth.status','token':login['token']})['logged_in'])
|
||||
|
||||
def test_log_tail_is_bounded_and_missing_id_rejected(self):
|
||||
task_id='b'*32;run_id='c'*32
|
||||
with core.db() as con:
|
||||
con.execute('INSERT INTO runs(id,task_id,task_name,started,state) VALUES(?,?,?,?,?)',(run_id,task_id,'test',time.time(),'success'))
|
||||
(core.LOGS/(run_id+'.log')).write_text('x'*200000)
|
||||
result=core.read_run(run_id)
|
||||
self.assertTrue(result['tail_only']);self.assertEqual(len(result['output']),131072)
|
||||
with self.assertRaises(core.TaskError):core.read_run('../password')
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Loading…
Reference in New Issue
Block a user