PayrollManagementSystem/backend/app/services/tplus/supervisor.py

"""跨平台托管循环；单次任务在独立线程执行，停止时等待其完成。"""

import json
import logging
import platform
import signal
import socket
import threading
import time
from pathlib import Path
from urllib.error import URLError
from urllib.parse import urlsplit
from urllib.request import urlopen
from uuid import uuid4

from sqlalchemy import create_engine, select, text, update
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.pool import NullPool

from app.core.tplus_db import resolve_tplus_settings
from app.models import TPlusRuntimeControl
from app.services.tplus.automation import local_now, tick
from app.services.tplus.worker import run_once

logger = logging.getLogger(__name__)
LOCK_NAME = "ys:tplus:supervisor"


def health_error(settings):
    """只访问本机健康端点，禁止通过配置探测外部网络。"""
    try:
        url = settings.t_plus_supervisor_health_url
        parsed = urlsplit(url)
        if parsed.scheme != "http" or parsed.hostname not in ("localhost", "127.0.0.1", "::1"):
            return "后端健康检查地址必须是本机 HTTP 地址"
        with urlopen(url, timeout=3) as response:
            body = json.load(response)
        data = body.get("data") if isinstance(body, dict) else None
        if not isinstance(data, dict) or not all(data.get(key) is True for key in ("app", "db", "redis")):
            return "系统或依赖尚未就绪"
        source = resolve_tplus_settings(settings)
        try:
            import pyodbc
        except ImportError:
            return "未安装 T+ 数据库连接组件"
        if source.t_plus_odbc_driver not in pyodbc.drivers():
            return "T+ 数据库驱动不可用，请联系维护人员"
        if settings.t_plus_stop_file and Path(settings.t_plus_stop_file).exists():
            return "旧版停止标记存在，请联系维护人员处理"
    except (OSError, URLError, ValueError, RuntimeError):
        return "系统健康检查或 T+ 连接配置未就绪"
    engine = None
    try:
        engine = create_engine(
            source.t_plus_database_url, poolclass=NullPool, hide_parameters=True,
            connect_args={"timeout": source.t_plus_connect_timeout},
        )
        with engine.connect() as connection:
            connection.connection.driver_connection.timeout = source.t_plus_query_timeout
            connection.exec_driver_sql("SELECT 1")
    except (SQLAlchemyError, OSError, ValueError, RuntimeError):
        return "T+ 数据库连接不可用，请检查连接配置"
    finally:
        if engine is not None:
            engine.dispose()
    return None


def may_claim(session):
    """与页面更新同一行锁，确认停止后不再领取新任务。"""
    row = session.scalar(select(TPlusRuntimeControl).where(TPlusRuntimeControl.id == 1)
                         .with_for_update().execution_options(populate_existing=True))
    return row is not None and row.desired_state == "RUNNING"


def run(factory, settings, *, interval=5, stop_event=None):
    """持有数据库命名锁期间，轮询页面设置并完成已领取的工作。"""
    stopped = stop_event or threading.Event()
    owner = str(uuid4())
    worker = None
    failures = []
    retry_after = 0.0
    last_maintenance = 0.0
    handlers = {}
    if threading.current_thread() is threading.main_thread():
        for sig in (signal.SIGINT, signal.SIGTERM):
            handlers[sig] = signal.signal(sig, lambda *_: stopped.set())
    connection = None
    acquired = False

    def execute():
        try:
            with factory() as session:
                run_once(session, worker_owner=owner, may_claim=may_claim)
        except Exception as exc:
            logger.error("T+ 读取任务执行异常，异常类型=%s", type(exc).__name__)
            failures.append("读取服务执行失败，请检查服务日志")

    try:
        connection = factory.kw["bind"].connect()
        if connection.dialect.name != "mysql":
            raise RuntimeError("T+ 托管服务需要 MySQL 命名锁")
        acquired = connection.execute(text("SELECT GET_LOCK(:name, 0)"), {"name": LOCK_NAME}).scalar() == 1
        if not acquired:
            logger.error("T+ 托管服务已在运行")
            return 1
        while not stopped.is_set():
            try:
                if connection.execute(text("SELECT IS_USED_LOCK(:name) = CONNECTION_ID()"),
                                      {"name": LOCK_NAME}).scalar() != 1:
                    raise RuntimeError("托管服务锁已失效")
                with factory() as session:
                    row = session.scalar(select(TPlusRuntimeControl).where(TPlusRuntimeControl.id == 1)
                                         .execution_options(populate_existing=True))
                    if row is None:
                        raise RuntimeError("读取服务控制尚未初始化")
                    desired = row.desired_state
                    error = health_error(settings) if desired == "RUNNING" else None
                    active = worker is not None and worker.is_alive()
                    if failures:
                        error = failures.pop(0)
                        retry_after = time.monotonic() + 30
                    elif time.monotonic() < retry_after:
                        error = "读取服务异常，等待重试"
                    if desired == "RUNNING" and not error and not active:
                        if time.monotonic() - last_maintenance >= 60:
                            tick(session, settings=settings)
                            last_maintenance = time.monotonic()
                        worker = threading.Thread(target=execute, name="tplus-worker")
                        worker.start()
                        active = True
                    state = ("STOPPING" if active else "STOPPED") if desired == "STOPPED" else (
                        "DEGRADED" if error else "RUNNING")
                    session.execute(update(TPlusRuntimeControl).where(TPlusRuntimeControl.id == 1).values(
                        actual_state=state, worker_phase="RUNNING" if active else "IDLE" if state == "RUNNING" else "STOPPED",
                        heartbeat_at=local_now(), last_error=error, host=socket.gethostname()[:128],
                        platform=platform.system()[:32], owner=owner))
                    session.commit()
            except Exception as exc:
                logger.error("T+ 托管服务检查失败，异常类型=%s", type(exc).__name__)
                # 锁或数据库状态不明时退出，绝不启动新的读取任务。
                break
            stopped.wait(interval)
    finally:
        if worker is not None:
            worker.join()
        if acquired:
            try:
                with factory() as session:
                    session.execute(update(TPlusRuntimeControl).where(
                        TPlusRuntimeControl.id == 1, TPlusRuntimeControl.owner == owner
                    ).values(actual_state="STOPPED", worker_phase="STOPPED", heartbeat_at=local_now()))
                    session.commit()
            except Exception:
                logger.warning("无法记录 T+ 托管服务停止状态")
            try:
                released = connection.execute(
                    text("SELECT RELEASE_LOCK(:name)"), {"name": LOCK_NAME}
                ).scalar()
                if released != 1:
                    connection.invalidate()
            except Exception:
                connection.invalidate()
        if connection is not None:
            connection.close()
        for sig, handler in handlers.items():
            signal.signal(sig, handler)
    return 0 if stopped.is_set() else 1