"""管理员监控视图：已落库历史与当前分钟合并，保持无数据与故障语义。""" from collections import defaultdict from datetime import timedelta from time import monotonic from app.core.exceptions import BadRequestError, NotFoundError from app.core.monitoring import SHANGHAI, merge_metrics, minute_of, now_local from app.core.response import PageResult from app.repositories import monitor_repository as repo from app.schemas.monitor import EventOut, OverviewOut def window(query): now = now_local() end = query.end or now if end.tzinfo: end = end.astimezone(SHANGHAI).replace(tzinfo=None) if end > now + timedelta(seconds=30) or end < now - timedelta(days=7): raise BadRequestError("查询时间须在最近 7 天内") return max(end - timedelta(hours=query.hours), now - timedelta(days=7)), end def overview(session, collector): state = collector.current() now = now_local() current_minute = minute_of(now) stored = repo.snapshots(session, current_minute - timedelta(minutes=4), current_minute) # 未落库分钟从有界缓冲补齐；仅跳过同实例已持久化的分钟。 known = {(row.instance_id, row.minute) for row in stored} recent = collector.recorder.metrics(current_minute - timedelta(minutes=4)) values = [row.requests for row in stored] values.extend( metric for minute, endpoints in recent.items() if (collector.recorder.instance_id, minute) not in known for metric in endpoints.values() ) state["requests"] = merge_metrics(values) stale = not state["collected_at"] or (now - state["collected_at"]).total_seconds() > 45 if stale: state.update( collection_status="unknown", collection_message="采集未启用或已超过 45 秒未更新，当前状态未知。", ) for dependency in state["dependencies"]: dependency.update(status="unknown", message="采集已过期，请检查服务日志。") persisted_alerts = { row.code: EventOut.model_validate(row) for row in repo.active_events(session) } for item in state["alerts"]: persisted_alerts[item["code"]] = EventOut.model_validate(item) state["alerts"] = list(persisted_alerts.values()) status = ( "unknown" if stale else "error" if any(dep["status"] == "error" for dep in state["dependencies"]) else "warning" if state["alerts"] or state["collection_status"] == "warning" else "normal" ) return OverviewOut( status=status, environment=collector.settings.app_env, version="0.1.0", instance_id=collector.recorder.instance_id, uptime_seconds=int(monotonic() - collector.recorder.started), dropped_events=collector.recorder.dropped_events, **state, ) def trends(session, collector, query): start, end = window(query) stored = repo.snapshots(session, minute_of(start), end) # 24 小时按 5 分钟、7 天按 30 分钟聚合，间断采集保留 null 点。 step = 1 if query.hours == 1 else 5 if query.hours == 24 else 30 groups = defaultdict(list) for row in stored: minute = row.minute.replace(minute=(row.minute.minute // step) * step) groups[minute].append( {"sample_count": row.sample_count, "resources": row.resources, "requests": row.requests} ) at = minute_of(end) recent = collector.recorder.metrics(start, end) # 只补尚未持久化的分钟，历史查看不引入窗口外最新资源。 known = {(row.instance_id, row.minute) for row in stored} sample_history = collector.sample_history() for minute in set(recent) | set(sample_history): if not start <= minute < end or (collector.recorder.instance_id, minute) in known: continue samples = sample_history.get(minute, []) groups[minute.replace(minute=(minute.minute // step) * step)].append( { "sample_count": len(samples), "resources": samples[-1] if samples else {}, "requests": merge_metrics(recent.get(minute, {}).values()), } ) cursor = minute_of(start).replace(minute=(start.minute // step) * step) points = [] while cursor <= at: rows = groups.get(cursor, []) point = {"at": cursor, "sample_count": sum(row["sample_count"] for row in rows)} for key in ("cpu_percent", "memory_percent", "disk_percent"): values = [ (row["resources"].get(key), row["sample_count"]) for row in rows if row["resources"].get(key) is not None and row["sample_count"] ] point[key] = ( round( sum(value * count for value, count in values) / sum(count for _, count in values), 2, ) if values else None ) metric = merge_metrics(row["requests"] for row in rows) point.update( request_count=metric["count"] if rows else None, error_rate=metric["error_rate"] if rows else None, p95_ms=metric["p95_ms"], ) points.append(point) cursor += timedelta(minutes=step) return { "start": start, "end": end, "step_minutes": step, "points": points, "p95_approximate": True, } def interfaces(session, collector, query): start, end = window(query) rows = repo.request_minutes(session, minute_of(start), end, query.route) truncated = len(rows) > 20000 rows = rows[:20000] grouped, known = defaultdict(list), set() for row in rows: grouped[(row.method, row.route)].append(row.metrics) known.add((row.instance_id, row.minute, row.method, row.route)) for minute, endpoints in collector.recorder.metrics(minute_of(start), end).items(): for (method, route), metric in endpoints.items(): if (not query.route or route == query.route) and ( collector.recorder.instance_id, minute, method, route, ) not in known: grouped[(method, route)].append(metric) records = [ {"method": method, "route": route, **merge_metrics(items)} for (method, route), items in grouped.items() ] records.sort(key=lambda item: (item["p95_ms"] or 0, item["count"]), reverse=True) return { "start": start, "end": end, "list": records[:200], "truncated": truncated or len(records) > 200, "p95_approximate": True, } def events(session, query): start, end = window(query) rows, total = repo.events(session, query, start, end) return PageResult( records=[EventOut.model_validate(row) for row in rows], total=total, page_num=query.page_num, page_size=query.page_size, ) def event(session, event_id): row = repo.event(session, event_id) if row is None or ( row.last_seen < now_local() - timedelta(days=7) and not (row.kind == "HEALTH" and row.resolved_at is None) ): raise NotFoundError("诊断事件不存在或已过保留期") return EventOut.model_validate(row) def diagnostic(session, collector, query): # 仅导出与页面相同的白名单模型；不读取日志文件、配置、请求正文或工资明细。 return { "generated_at": now_local(), "overview": overview(session, collector), "events": events(session, query), "note": "仅含脱敏运行摘要；完整异常请按请求编号检索服务端日志。", }