"""单应用进程的独立采集线程；探测有超时，写入异常不会进入业务请求。""" import logging import threading from collections import OrderedDict from copy import deepcopy from datetime import timedelta from pathlib import Path from time import monotonic, perf_counter import psutil from redis import Redis from sqlalchemy import create_engine, delete, select, text from sqlalchemy.orm import sessionmaker from app.core.config import BACKEND_DIR from app.core.monitoring import ( REQUEST_ID, Recorder, fingerprint, merge_metrics, minute_of, now_local, ) from app.models.monitor import MonitorEvent, MonitorRequestMinute, MonitorSnapshot from app.repositories import monitor_repository as repo from app.services.tplus import automation logger = logging.getLogger(__name__) class RuntimeErrorHandler(logging.Handler): """后台错误只采集固定摘要；HTTP 错误由中间件记录，避免同一异常重复。""" def __init__(self, recorder): super().__init__(logging.ERROR) self.recorder = recorder def emit(self, record): if ( record.name.startswith("app.") and ".monitor" not in record.name and REQUEST_ID.get() == "-" and getattr(record, "request_id", "-") == "-" ): self.recorder.runtime_error(record.name) class MonitorCollector: def __init__(self, settings, recorder: Recorder): self.settings, self.recorder = settings, recorder self.recorder.slow_ms = settings.monitor_slow_request_ms self.lock = threading.Lock() self.stopped = threading.Event() self.thread = None self.engine = None self.redis = None self.process = None self.samples = OrderedDict() self.conditions = {} self.unknown_codes = set() self.pending_incidents = {} self.since = {} self.persisted = set() self.last_log = 0 self.handler = RuntimeErrorHandler(recorder) self.state = { "collected_at": None, "last_persisted_at": None, "resources": {}, "dependencies": [], "tasks": {}, "alerts": [], "collection_status": "unknown", "collection_message": "正在等待首次采集。" if settings.monitor_enabled else "后台采集未启用。", } def current(self): with self.lock: return deepcopy(self.state) def sample_history(self): with self.lock: return deepcopy(self.samples) def start(self): if not self.settings.monitor_enabled or self.thread: return self.stopped.clear() logging.getLogger().addHandler(self.handler) self.thread = threading.Thread(target=self._run, name="system-monitor", daemon=True) self.thread.start() def stop(self): self.stopped.set() if self.thread: self.thread.join(timeout=15) if self.thread.is_alive(): logger.warning("监控采集线程未在关闭窗口内结束") return logging.getLogger().removeHandler(self.handler) if self.redis: self.redis.close() if self.engine: self.engine.dispose() self.thread = None def _factory(self): if self.engine is None: self.engine = create_engine( self.settings.database_url, pool_size=1, max_overflow=0, pool_timeout=2, pool_pre_ping=True, connect_args={"connect_timeout": 2, "read_timeout": 2, "write_timeout": 2}, ) return sessionmaker(self.engine, expire_on_commit=False) def _resources(self): first = self.process is None if first: self.process = psutil.Process() psutil.cpu_percent(None) self.process.cpu_percent(None) memory = psutil.virtual_memory() disk = psutil.disk_usage(str(BACKEND_DIR.anchor)) return { "cpu_percent": None if first else psutil.cpu_percent(None), "memory_percent": memory.percent, "disk_percent": disk.percent, "memory_total_mb": round(memory.total / 1024**2), "disk_free_gb": round(disk.free / 1024**3, 2), "process_cpu_percent": None if first else round(self.process.cpu_percent(None) / (psutil.cpu_count() or 1), 2), "process_memory_mb": round(self.process.memory_info().rss / 1024**2, 2), "scope": "进程可见系统资源，未换算容器限额" if Path("/.dockerenv").exists() else "应用所在系统的可见资源", "process_cpu_scope": "占可见 CPU 总量的比例", } def _dependencies(self, at): dependencies = [ { "name": "应用", "status": "normal", "message": "采集线程正在运行。", "checked_at": at, "duration_ms": None, } ] tasks = {"status": "unknown", "message": "尚无采集记录。", "alerts": []} for name in ("MySQL", "Redis"): started = perf_counter() try: if name == "MySQL": with self._factory()() as session: session.execute(text("SELECT 1")) else: if self.redis is None: self.redis = Redis.from_url( self.settings.redis_url, socket_connect_timeout=2, socket_timeout=2 ) self.redis.ping() status, message = "normal", "连接正常。" except Exception: status, message = "error", "连接检查失败，请核对服务和网络。" dependencies.append( { "name": name, "status": status, "message": message, "checked_at": at, "duration_ms": round((perf_counter() - started) * 1000, 2), } ) if dependencies[1]["status"] == "normal": try: with self._factory()() as session: tasks = automation.overview(session, settings=self.settings, now=at) source_configured = bool( self.settings.t_plus_enabled or self.settings.t_plus_connection_file ) if not source_configured: status, message = "unknown", "T+ 未配置。" elif tasks.get("stop_requested"): status, message = "unknown", "已主动请求停止采集。" elif not tasks["enabled"]: status, message = "unknown", "自动采集未启用，可在原页面查看手动任务。" else: status = "warning" if tasks["alerts"] else "normal" message = ( "存在采集异常，请查看任务与同步。" if tasks["alerts"] else "周期采集状态正常。" ) tasks.update(status=status, message=message) except Exception: tasks = { "status": "unknown", "message": "任务状态读取失败，请检查迁移和服务日志。", "alerts": [], } dependencies.append( { "name": "T+ 采集进程", "status": tasks["status"], "message": tasks["message"], "checked_at": at, "duration_ms": None, } ) return dependencies, tasks def _evaluate(self, at, resources, dependencies, tasks): active = {} self.unknown_codes = set() for dependency in dependencies: if dependency["status"] == "error": code = "MYSQL_UNAVAILABLE" if dependency["name"] == "MySQL" else "REDIS_UNAVAILABLE" active[code] = ("error", dependency["name"], dependency["message"], None) for field, threshold, code, title in ( ("cpu_percent", self.settings.monitor_cpu_threshold, "CPU_HIGH", "CPU 占用持续偏高"), ( "memory_percent", self.settings.monitor_memory_threshold, "MEMORY_HIGH", "内存占用持续偏高", ), ("disk_percent", self.settings.monitor_disk_threshold, "DISK_HIGH", "磁盘空间不足"), ): value = resources.get(field) if value is None: self.unknown_codes.add(code) self.since.pop(code, None) # 采集未知时不将仍存在的问题标为恢复。 if code in self.conditions: previous = self.conditions[code] active[code] = ( previous["level"], previous["component"], previous["message"], None, ) continue if value >= threshold: since = self.since.setdefault(code, at) # 重启后的持续时间尚未建立时，不关闭历史同类告警。 self.unknown_codes.add(code) if ( field == "disk_percent" or (at - since).total_seconds() >= self.settings.monitor_resource_duration_seconds ): active[code] = ("warning", "resources", title + "，请检查资源趋势。", None) else: self.since.pop(code, None) recent = self.recorder.metrics(minute_of(at) - timedelta(minutes=4)) metric = merge_metrics(m for endpoints in recent.values() for m in endpoints.values()) if ( metric["count"] >= self.settings.monitor_error_min_requests and metric["error_rate"] >= self.settings.monitor_error_rate_threshold ): active["HTTP_ERROR_RATE"] = ( "warning", "http", "近 5 分钟服务端错误率偏高，请查看失败接口。", None, ) if tasks.get("status") == "unknown" and "任务状态读取失败" in tasks.get("message", ""): active["TASK_STATUS_UNKNOWN"] = ("warning", "tplus", tasks["message"], None) for alert in tasks.get("alerts", []): active[alert["code"]] = ("warning", "tplus", alert["message"], alert.get("batch_no")) if "enabled" not in tasks: self.unknown_codes.update( {"WORKER_OFFLINE", "QUEUE_OVERDUE", "SYNC_STALE", "SYNC_REQUIRES_ATTENTION"} ) for code, previous in self.conditions.items(): if code in self.unknown_codes: active[code] = ( previous["level"], previous["component"], previous["message"], previous.get("batch_no"), ) for code, (level, component, message, batch) in active.items(): previous = self.conditions.get(code) incident = previous or { "id": 0, "instance_id": self.recorder.instance_id, "kind": "HEALTH", "first_seen": at, "occurrences": 0, "code": code, "fingerprint": fingerprint(code), "resolved_at": None, } incident.update( last_seen=at, occurrences=incident["occurrences"] + 1, level=level, component=component, message=message, batch_no=batch, ) self.conditions[code] = incident self.pending_incidents[code] = deepcopy(incident) for code in set(self.conditions) - set(active): incident = self.conditions.pop(code) incident.update(resolved_at=at) self.pending_incidents[code] = deepcopy(incident) def _persist(self, at, *, final=False): pending = self.recorder.take_events() try: with self._factory()() as session: # 首次采集复用历史未恢复事件，跨应用重启保留故障起点。 historical = {item.code: item for item in repo.active_events(session)} for code, item in self.pending_incidents.items(): row = historical.get(code) fields = {key: value for key, value in item.items() if key != "id"} if row is None: row = MonitorEvent(**fields) session.add(row) else: for key in ( "last_seen", "resolved_at", "level", "message", "batch_no", "instance_id", ): setattr(row, key, fields.get(key)) row.occurrences = ( max(row.occurrences, fields["occurrences"]) if item.get("id") == row.id else row.occurrences + fields["occurrences"] ) session.flush() if code in self.conditions: self.conditions[code].update( id=row.id, first_seen=row.first_seen, occurrences=row.occurrences ) for code, row in historical.items(): if ( code not in self.conditions and code not in self.pending_incidents and code not in self.unknown_codes ): # 只在本次完整采集后确认历史异常已恢复。 row.resolved_at = at for item in pending: session.add(MonitorEvent(instance_id=self.recorder.instance_id, **item)) completed = minute_of(at) + (timedelta(minutes=1) if final else timedelta()) metrics = self.recorder.metrics(end=completed) written = [] for minute in sorted(set(self.samples) | set(metrics)): if minute >= completed or minute in self.persisted: continue samples = self.samples.get(minute, []) endpoints = metrics.get(minute, {}) resources = dict(samples[-1]) if samples else {} for key in ( "cpu_percent", "memory_percent", "disk_percent", "process_cpu_percent", "process_memory_mb", ): numbers = [sample[key] for sample in samples if sample.get(key) is not None] resources[key] = round(sum(numbers) / len(numbers), 2) if numbers else None session.add( MonitorSnapshot( instance_id=self.recorder.instance_id, minute=minute, sample_count=len(samples), resources=resources, requests=merge_metrics(endpoints.values()), ) ) for (method, route), values in endpoints.items(): session.add( MonitorRequestMinute( instance_id=self.recorder.instance_id, minute=minute, method=method, route=route, metrics=values, ) ) written.append(minute) # 每次最多删除每表 1000 条，限定监控表及过期时间，避免长事务。 cutoff = at - timedelta(days=7) for model, column in ( (MonitorSnapshot, MonitorSnapshot.minute), (MonitorRequestMinute, MonitorRequestMinute.minute), (MonitorEvent, MonitorEvent.last_seen), ): expired = select(model.id).where(column < cutoff) if model is MonitorEvent: expired = expired.where( (MonitorEvent.kind != "HEALTH") | MonitorEvent.resolved_at.is_not(None) ) ids = session.scalars(expired.order_by(column).limit(1000)).all() if ids: session.execute(delete(model).where(model.id.in_(ids))) session.commit() self.persisted.update(written) self.persisted = { minute for minute in self.persisted if minute >= minute_of(at) - timedelta(minutes=10) } self.pending_incidents.clear() with self.lock: self.state.update( last_persisted_at=at, collection_status="normal", collection_message="采集正常，分钟历史保留 7 天。", ) except Exception: self.recorder.restore_events(pending) with self.lock: self.state.update( collection_status="warning", collection_message="历史写入失败，部分数据可能缺失；请检查数据库迁移和服务日志。", ) if monotonic() - self.last_log >= 60: logger.warning("监控历史写入失败，请检查迁移及数据库", exc_info=True) self.last_log = monotonic() def tick(self): at = now_local() try: resources = self._resources() except Exception: resources = {"scope": "资源采集失败"} logger.warning("系统资源采集失败", exc_info=True) dependencies, tasks = self._dependencies(at) with self.lock: self.samples.setdefault(minute_of(at), []).append(resources) while len(self.samples) > 10: self.samples.popitem(last=False) self._evaluate(at, resources, dependencies, tasks) with self.lock: self.state.update( collected_at=at, resources=resources, dependencies=dependencies, tasks=tasks ) self._persist(at) with self.lock: self.state["alerts"] = deepcopy(list(self.conditions.values())) if not resources.get("memory_percent"): self.state.update( collection_status="warning", collection_message="资源采集失败，当前资源指标未知。", ) def _run(self): while not self.stopped.is_set(): started = monotonic() try: self.tick() except Exception: logger.warning("监控采集周期失败", exc_info=True) with self.lock: self.state.update( collection_status="warning", collection_message="采集周期失败，请检查服务日志。", ) self.stopped.wait(max(0, 15 - (monotonic() - started))) if self.engine is not None: self._persist(now_local(), final=True)