1461a8d113
按 2026-11-02 需求补齐计划交付后端能力： - 角色授权：权限目录化，接口统一准入校验，未登记默认拒绝； 管理员判断改为按数据库角色重新判定，防止权限码持有者绕过 - 系统监控：单进程采集资源、请求与异常指标，分钟级聚合， 阈值配置化；敏感配置增加 repr 保护，Redis 密码 URL 转义 - 试用期计提：按回款日判定并写入快照，年度结清与调整单联动； 人员模型补试用期、离职日期与工资版本历史字段 - 计提模板：公式骨架受控，页面维护名称与参数绑定 - T+ 采集：自动化配置表与进度查询 内置角色禁停用保护对齐模块注释；核算规则版本提为常量并 补充批量核算失败堆栈日志。 影响范围：后端全部模块、数据库迁移 V22-V28 验证：uv run ruff check 通过；922 个单元测试全部通过； 集成测试因本机 MySQL 数据卷被其他容器锁定暂未运行
478 lines
20 KiB
Python
478 lines
20 KiB
Python
"""单应用进程的独立采集线程；探测有超时，写入异常不会进入业务请求。"""
|
||
|
||
import logging
|
||
import threading
|
||
from collections import OrderedDict
|
||
from copy import deepcopy
|
||
from datetime import timedelta
|
||
from pathlib import Path
|
||
from time import monotonic, perf_counter
|
||
|
||
import psutil
|
||
from redis import Redis
|
||
from sqlalchemy import create_engine, delete, select, text
|
||
from sqlalchemy.orm import sessionmaker
|
||
|
||
from app.core.config import BACKEND_DIR
|
||
from app.core.monitoring import (
|
||
REQUEST_ID,
|
||
Recorder,
|
||
fingerprint,
|
||
merge_metrics,
|
||
minute_of,
|
||
now_local,
|
||
)
|
||
from app.models.monitor import MonitorEvent, MonitorRequestMinute, MonitorSnapshot
|
||
from app.repositories import monitor_repository as repo
|
||
from app.services.tplus import automation
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class RuntimeErrorHandler(logging.Handler):
|
||
"""后台错误只采集固定摘要；HTTP 错误由中间件记录，避免同一异常重复。"""
|
||
|
||
def __init__(self, recorder):
|
||
super().__init__(logging.ERROR)
|
||
self.recorder = recorder
|
||
|
||
def emit(self, record):
|
||
if (
|
||
record.name.startswith("app.")
|
||
and ".monitor" not in record.name
|
||
and REQUEST_ID.get() == "-"
|
||
and getattr(record, "request_id", "-") == "-"
|
||
):
|
||
self.recorder.runtime_error(record.name)
|
||
|
||
|
||
class MonitorCollector:
|
||
def __init__(self, settings, recorder: Recorder):
|
||
self.settings, self.recorder = settings, recorder
|
||
self.recorder.slow_ms = settings.monitor_slow_request_ms
|
||
self.lock = threading.Lock()
|
||
self.stopped = threading.Event()
|
||
self.thread = None
|
||
self.engine = None
|
||
self.redis = None
|
||
self.process = None
|
||
self.samples = OrderedDict()
|
||
self.conditions = {}
|
||
self.unknown_codes = set()
|
||
self.pending_incidents = {}
|
||
self.since = {}
|
||
self.persisted = set()
|
||
self.last_log = 0
|
||
self.handler = RuntimeErrorHandler(recorder)
|
||
self.state = {
|
||
"collected_at": None,
|
||
"last_persisted_at": None,
|
||
"resources": {},
|
||
"dependencies": [],
|
||
"tasks": {},
|
||
"alerts": [],
|
||
"collection_status": "unknown",
|
||
"collection_message": "正在等待首次采集。"
|
||
if settings.monitor_enabled
|
||
else "后台采集未启用。",
|
||
}
|
||
|
||
def current(self):
|
||
with self.lock:
|
||
return deepcopy(self.state)
|
||
|
||
def sample_history(self):
|
||
with self.lock:
|
||
return deepcopy(self.samples)
|
||
|
||
def start(self):
|
||
if not self.settings.monitor_enabled or self.thread:
|
||
return
|
||
self.stopped.clear()
|
||
logging.getLogger().addHandler(self.handler)
|
||
self.thread = threading.Thread(target=self._run, name="system-monitor", daemon=True)
|
||
self.thread.start()
|
||
|
||
def stop(self):
|
||
self.stopped.set()
|
||
if self.thread:
|
||
self.thread.join(timeout=15)
|
||
if self.thread.is_alive():
|
||
logger.warning("监控采集线程未在关闭窗口内结束")
|
||
return
|
||
logging.getLogger().removeHandler(self.handler)
|
||
if self.redis:
|
||
self.redis.close()
|
||
if self.engine:
|
||
self.engine.dispose()
|
||
self.thread = None
|
||
|
||
def _factory(self):
|
||
if self.engine is None:
|
||
self.engine = create_engine(
|
||
self.settings.database_url,
|
||
pool_size=1,
|
||
max_overflow=0,
|
||
pool_timeout=2,
|
||
pool_pre_ping=True,
|
||
connect_args={"connect_timeout": 2, "read_timeout": 2, "write_timeout": 2},
|
||
)
|
||
return sessionmaker(self.engine, expire_on_commit=False)
|
||
|
||
def _resources(self):
|
||
first = self.process is None
|
||
if first:
|
||
self.process = psutil.Process()
|
||
psutil.cpu_percent(None)
|
||
self.process.cpu_percent(None)
|
||
memory = psutil.virtual_memory()
|
||
disk = psutil.disk_usage(str(BACKEND_DIR.anchor))
|
||
return {
|
||
"cpu_percent": None if first else psutil.cpu_percent(None),
|
||
"memory_percent": memory.percent,
|
||
"disk_percent": disk.percent,
|
||
"memory_total_mb": round(memory.total / 1024**2),
|
||
"disk_free_gb": round(disk.free / 1024**3, 2),
|
||
"process_cpu_percent": None
|
||
if first
|
||
else round(self.process.cpu_percent(None) / (psutil.cpu_count() or 1), 2),
|
||
"process_memory_mb": round(self.process.memory_info().rss / 1024**2, 2),
|
||
"scope": "进程可见系统资源，未换算容器限额"
|
||
if Path("/.dockerenv").exists()
|
||
else "应用所在系统的可见资源",
|
||
"process_cpu_scope": "占可见 CPU 总量的比例",
|
||
}
|
||
|
||
def _dependencies(self, at):
|
||
dependencies = [
|
||
{
|
||
"name": "应用",
|
||
"status": "normal",
|
||
"message": "采集线程正在运行。",
|
||
"checked_at": at,
|
||
"duration_ms": None,
|
||
}
|
||
]
|
||
tasks = {"status": "unknown", "message": "尚无采集记录。", "alerts": []}
|
||
for name in ("MySQL", "Redis"):
|
||
started = perf_counter()
|
||
try:
|
||
if name == "MySQL":
|
||
with self._factory()() as session:
|
||
session.execute(text("SELECT 1"))
|
||
else:
|
||
if self.redis is None:
|
||
self.redis = Redis.from_url(
|
||
self.settings.redis_url, socket_connect_timeout=2, socket_timeout=2
|
||
)
|
||
self.redis.ping()
|
||
status, message = "normal", "连接正常。"
|
||
except Exception:
|
||
status, message = "error", "连接检查失败，请核对服务和网络。"
|
||
dependencies.append(
|
||
{
|
||
"name": name,
|
||
"status": status,
|
||
"message": message,
|
||
"checked_at": at,
|
||
"duration_ms": round((perf_counter() - started) * 1000, 2),
|
||
}
|
||
)
|
||
if dependencies[1]["status"] == "normal":
|
||
try:
|
||
with self._factory()() as session:
|
||
tasks = automation.overview(session, settings=self.settings, now=at)
|
||
source_configured = bool(
|
||
self.settings.t_plus_enabled or self.settings.t_plus_connection_file
|
||
)
|
||
if not source_configured:
|
||
status, message = "unknown", "T+ 未配置。"
|
||
elif tasks.get("stop_requested"):
|
||
status, message = "unknown", "已主动请求停止采集。"
|
||
elif not tasks["enabled"]:
|
||
status, message = "unknown", "自动采集未启用，可在原页面查看手动任务。"
|
||
else:
|
||
status = "warning" if tasks["alerts"] else "normal"
|
||
message = (
|
||
"存在采集异常，请查看任务与同步。"
|
||
if tasks["alerts"]
|
||
else "周期采集状态正常。"
|
||
)
|
||
tasks.update(status=status, message=message)
|
||
except Exception:
|
||
tasks = {
|
||
"status": "unknown",
|
||
"message": "任务状态读取失败，请检查迁移和服务日志。",
|
||
"alerts": [],
|
||
}
|
||
dependencies.append(
|
||
{
|
||
"name": "T+ 采集进程",
|
||
"status": tasks["status"],
|
||
"message": tasks["message"],
|
||
"checked_at": at,
|
||
"duration_ms": None,
|
||
}
|
||
)
|
||
return dependencies, tasks
|
||
|
||
def _evaluate(self, at, resources, dependencies, tasks):
|
||
active = {}
|
||
self.unknown_codes = set()
|
||
for dependency in dependencies:
|
||
if dependency["status"] == "error":
|
||
code = "MYSQL_UNAVAILABLE" if dependency["name"] == "MySQL" else "REDIS_UNAVAILABLE"
|
||
active[code] = ("error", dependency["name"], dependency["message"], None)
|
||
for field, threshold, code, title in (
|
||
("cpu_percent", self.settings.monitor_cpu_threshold, "CPU_HIGH", "CPU 占用持续偏高"),
|
||
(
|
||
"memory_percent",
|
||
self.settings.monitor_memory_threshold,
|
||
"MEMORY_HIGH",
|
||
"内存占用持续偏高",
|
||
),
|
||
("disk_percent", self.settings.monitor_disk_threshold, "DISK_HIGH", "磁盘空间不足"),
|
||
):
|
||
value = resources.get(field)
|
||
if value is None:
|
||
self.unknown_codes.add(code)
|
||
self.since.pop(code, None)
|
||
# 采集未知时不将仍存在的问题标为恢复。
|
||
if code in self.conditions:
|
||
previous = self.conditions[code]
|
||
active[code] = (
|
||
previous["level"],
|
||
previous["component"],
|
||
previous["message"],
|
||
None,
|
||
)
|
||
continue
|
||
if value >= threshold:
|
||
since = self.since.setdefault(code, at)
|
||
# 重启后的持续时间尚未建立时，不关闭历史同类告警。
|
||
self.unknown_codes.add(code)
|
||
if (
|
||
field == "disk_percent"
|
||
or (at - since).total_seconds()
|
||
>= self.settings.monitor_resource_duration_seconds
|
||
):
|
||
active[code] = ("warning", "resources", title + "，请检查资源趋势。", None)
|
||
else:
|
||
self.since.pop(code, None)
|
||
recent = self.recorder.metrics(minute_of(at) - timedelta(minutes=4))
|
||
metric = merge_metrics(m for endpoints in recent.values() for m in endpoints.values())
|
||
if (
|
||
metric["count"] >= self.settings.monitor_error_min_requests
|
||
and metric["error_rate"] >= self.settings.monitor_error_rate_threshold
|
||
):
|
||
active["HTTP_ERROR_RATE"] = (
|
||
"warning",
|
||
"http",
|
||
"近 5 分钟服务端错误率偏高，请查看失败接口。",
|
||
None,
|
||
)
|
||
if tasks.get("status") == "unknown" and "任务状态读取失败" in tasks.get("message", ""):
|
||
active["TASK_STATUS_UNKNOWN"] = ("warning", "tplus", tasks["message"], None)
|
||
for alert in tasks.get("alerts", []):
|
||
active[alert["code"]] = ("warning", "tplus", alert["message"], alert.get("batch_no"))
|
||
if "enabled" not in tasks:
|
||
self.unknown_codes.update(
|
||
{"WORKER_OFFLINE", "QUEUE_OVERDUE", "SYNC_STALE", "SYNC_REQUIRES_ATTENTION"}
|
||
)
|
||
for code, previous in self.conditions.items():
|
||
if code in self.unknown_codes:
|
||
active[code] = (
|
||
previous["level"],
|
||
previous["component"],
|
||
previous["message"],
|
||
previous.get("batch_no"),
|
||
)
|
||
for code, (level, component, message, batch) in active.items():
|
||
previous = self.conditions.get(code)
|
||
incident = previous or {
|
||
"id": 0,
|
||
"instance_id": self.recorder.instance_id,
|
||
"kind": "HEALTH",
|
||
"first_seen": at,
|
||
"occurrences": 0,
|
||
"code": code,
|
||
"fingerprint": fingerprint(code),
|
||
"resolved_at": None,
|
||
}
|
||
incident.update(
|
||
last_seen=at,
|
||
occurrences=incident["occurrences"] + 1,
|
||
level=level,
|
||
component=component,
|
||
message=message,
|
||
batch_no=batch,
|
||
)
|
||
self.conditions[code] = incident
|
||
self.pending_incidents[code] = deepcopy(incident)
|
||
for code in set(self.conditions) - set(active):
|
||
incident = self.conditions.pop(code)
|
||
incident.update(resolved_at=at)
|
||
self.pending_incidents[code] = deepcopy(incident)
|
||
|
||
def _persist(self, at, *, final=False):
|
||
pending = self.recorder.take_events()
|
||
try:
|
||
with self._factory()() as session:
|
||
# 首次采集复用历史未恢复事件，跨应用重启保留故障起点。
|
||
historical = {item.code: item for item in repo.active_events(session)}
|
||
for code, item in self.pending_incidents.items():
|
||
row = historical.get(code)
|
||
fields = {key: value for key, value in item.items() if key != "id"}
|
||
if row is None:
|
||
row = MonitorEvent(**fields)
|
||
session.add(row)
|
||
else:
|
||
for key in (
|
||
"last_seen",
|
||
"resolved_at",
|
||
"level",
|
||
"message",
|
||
"batch_no",
|
||
"instance_id",
|
||
):
|
||
setattr(row, key, fields.get(key))
|
||
row.occurrences = (
|
||
max(row.occurrences, fields["occurrences"])
|
||
if item.get("id") == row.id
|
||
else row.occurrences + fields["occurrences"]
|
||
)
|
||
session.flush()
|
||
if code in self.conditions:
|
||
self.conditions[code].update(
|
||
id=row.id, first_seen=row.first_seen, occurrences=row.occurrences
|
||
)
|
||
for code, row in historical.items():
|
||
if (
|
||
code not in self.conditions
|
||
and code not in self.pending_incidents
|
||
and code not in self.unknown_codes
|
||
):
|
||
# 只在本次完整采集后确认历史异常已恢复。
|
||
row.resolved_at = at
|
||
for item in pending:
|
||
session.add(MonitorEvent(instance_id=self.recorder.instance_id, **item))
|
||
completed = minute_of(at) + (timedelta(minutes=1) if final else timedelta())
|
||
metrics = self.recorder.metrics(end=completed)
|
||
written = []
|
||
for minute in sorted(set(self.samples) | set(metrics)):
|
||
if minute >= completed or minute in self.persisted:
|
||
continue
|
||
samples = self.samples.get(minute, [])
|
||
endpoints = metrics.get(minute, {})
|
||
resources = dict(samples[-1]) if samples else {}
|
||
for key in (
|
||
"cpu_percent",
|
||
"memory_percent",
|
||
"disk_percent",
|
||
"process_cpu_percent",
|
||
"process_memory_mb",
|
||
):
|
||
numbers = [sample[key] for sample in samples if sample.get(key) is not None]
|
||
resources[key] = round(sum(numbers) / len(numbers), 2) if numbers else None
|
||
session.add(
|
||
MonitorSnapshot(
|
||
instance_id=self.recorder.instance_id,
|
||
minute=minute,
|
||
sample_count=len(samples),
|
||
resources=resources,
|
||
requests=merge_metrics(endpoints.values()),
|
||
)
|
||
)
|
||
for (method, route), values in endpoints.items():
|
||
session.add(
|
||
MonitorRequestMinute(
|
||
instance_id=self.recorder.instance_id,
|
||
minute=minute,
|
||
method=method,
|
||
route=route,
|
||
metrics=values,
|
||
)
|
||
)
|
||
written.append(minute)
|
||
# 每次最多删除每表 1000 条，限定监控表及过期时间，避免长事务。
|
||
cutoff = at - timedelta(days=7)
|
||
for model, column in (
|
||
(MonitorSnapshot, MonitorSnapshot.minute),
|
||
(MonitorRequestMinute, MonitorRequestMinute.minute),
|
||
(MonitorEvent, MonitorEvent.last_seen),
|
||
):
|
||
expired = select(model.id).where(column < cutoff)
|
||
if model is MonitorEvent:
|
||
expired = expired.where(
|
||
(MonitorEvent.kind != "HEALTH") | MonitorEvent.resolved_at.is_not(None)
|
||
)
|
||
ids = session.scalars(expired.order_by(column).limit(1000)).all()
|
||
if ids:
|
||
session.execute(delete(model).where(model.id.in_(ids)))
|
||
session.commit()
|
||
self.persisted.update(written)
|
||
self.persisted = {
|
||
minute
|
||
for minute in self.persisted
|
||
if minute >= minute_of(at) - timedelta(minutes=10)
|
||
}
|
||
self.pending_incidents.clear()
|
||
with self.lock:
|
||
self.state.update(
|
||
last_persisted_at=at,
|
||
collection_status="normal",
|
||
collection_message="采集正常，分钟历史保留 7 天。",
|
||
)
|
||
except Exception:
|
||
self.recorder.restore_events(pending)
|
||
with self.lock:
|
||
self.state.update(
|
||
collection_status="warning",
|
||
collection_message="历史写入失败，部分数据可能缺失；请检查数据库迁移和服务日志。",
|
||
)
|
||
if monotonic() - self.last_log >= 60:
|
||
logger.warning("监控历史写入失败，请检查迁移及数据库", exc_info=True)
|
||
self.last_log = monotonic()
|
||
|
||
def tick(self):
|
||
at = now_local()
|
||
try:
|
||
resources = self._resources()
|
||
except Exception:
|
||
resources = {"scope": "资源采集失败"}
|
||
logger.warning("系统资源采集失败", exc_info=True)
|
||
dependencies, tasks = self._dependencies(at)
|
||
with self.lock:
|
||
self.samples.setdefault(minute_of(at), []).append(resources)
|
||
while len(self.samples) > 10:
|
||
self.samples.popitem(last=False)
|
||
self._evaluate(at, resources, dependencies, tasks)
|
||
with self.lock:
|
||
self.state.update(
|
||
collected_at=at, resources=resources, dependencies=dependencies, tasks=tasks
|
||
)
|
||
self._persist(at)
|
||
with self.lock:
|
||
self.state["alerts"] = deepcopy(list(self.conditions.values()))
|
||
if not resources.get("memory_percent"):
|
||
self.state.update(
|
||
collection_status="warning",
|
||
collection_message="资源采集失败，当前资源指标未知。",
|
||
)
|
||
|
||
def _run(self):
|
||
while not self.stopped.is_set():
|
||
started = monotonic()
|
||
try:
|
||
self.tick()
|
||
except Exception:
|
||
logger.warning("监控采集周期失败", exc_info=True)
|
||
with self.lock:
|
||
self.state.update(
|
||
collection_status="warning",
|
||
collection_message="采集周期失败，请检查服务日志。",
|
||
)
|
||
self.stopped.wait(max(0, 15 - (monotonic() - started)))
|
||
if self.engine is not None:
|
||
self._persist(now_local(), final=True)
|