1461a8d113
按 2026-11-02 需求补齐计划交付后端能力： - 角色授权：权限目录化，接口统一准入校验，未登记默认拒绝； 管理员判断改为按数据库角色重新判定，防止权限码持有者绕过 - 系统监控：单进程采集资源、请求与异常指标，分钟级聚合， 阈值配置化；敏感配置增加 repr 保护，Redis 密码 URL 转义 - 试用期计提：按回款日判定并写入快照，年度结清与调整单联动； 人员模型补试用期、离职日期与工资版本历史字段 - 计提模板：公式骨架受控，页面维护名称与参数绑定 - T+ 采集：自动化配置表与进度查询 内置角色禁停用保护对齐模块注释；核算规则版本提为常量并 补充批量核算失败堆栈日志。 影响范围：后端全部模块、数据库迁移 V22-V28 验证：uv run ruff check 通过；922 个单元测试全部通过； 集成测试因本机 MySQL 数据卷被其他容器锁定暂未运行
171 lines
7.8 KiB
Python
171 lines
7.8 KiB
Python
"""T+ 只读快照同步：批量 staging、幂等去重和失败隔离。"""
|
||
|
||
import logging
|
||
from collections.abc import Iterable
|
||
from datetime import datetime
|
||
from itertools import batched
|
||
from pathlib import Path
|
||
|
||
from sqlalchemy import func, select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.config import get_settings
|
||
from app.core.tplus_lock import sync_lock
|
||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||
from app.services.tplus import progress
|
||
from app.services.tplus.errors import TRANSIENT_CODES, classify_source_error
|
||
from app.services.tplus.mapping import TPlusMapping, get_mapping
|
||
from app.services.tplus.reader import TPlusReader
|
||
from app.services.tplus.source_heads import observe_rows
|
||
from app.services.tplus.staging import stage_rows
|
||
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class TPlusSyncService:
|
||
"""按分页提交原始快照，业务表始终不参与本服务。"""
|
||
|
||
def __init__(self, source_session: Session, target_session: Session) -> None:
|
||
self.source_session = source_session
|
||
self.target_session = target_session
|
||
configured = get_settings().t_plus_query_dir
|
||
self.reader = TPlusReader(source_session, Path(configured) if configured else None)
|
||
|
||
def run(
|
||
self,
|
||
*,
|
||
entity_types: Iterable[str],
|
||
watermark: datetime | None = None,
|
||
batch_no: str | None = None,
|
||
batch_size: int = 500,
|
||
) -> TPlusSyncBatch:
|
||
"""执行可重复的全量读取，提交每页进度；字段语义未定时拒绝时间增量。"""
|
||
entities = tuple(dict.fromkeys(entity_types))
|
||
if not entities or not 1 <= batch_size <= 5000:
|
||
raise ValueError("至少指定一个实体，batch_size 必须在 1 到 5000 之间")
|
||
if watermark is not None:
|
||
raise ValueError("增量水位语义尚未确认，当前只支持全量读取和哈希去重")
|
||
for entity in entities:
|
||
get_mapping(entity)
|
||
with sync_lock(self.target_session):
|
||
recover_running(self.target_session)
|
||
batch = self._start_direct(entities, batch_no)
|
||
try:
|
||
return self.execute_claimed(batch, batch_size)
|
||
except BaseException:
|
||
self.target_session.rollback()
|
||
fail_batch(self.target_session, batch, "INTERRUPTED")
|
||
raise
|
||
|
||
def _start_direct(self, entities, batch_no):
|
||
batch = TPlusSyncBatch(
|
||
batch_no=batch_no or new_batch_no(),
|
||
sync_mode="FULL",
|
||
status="RUNNING",
|
||
source_database=get_settings().t_plus_database or None,
|
||
started_at=datetime.now(),
|
||
source_record_count=0,
|
||
staged_record_count=0,
|
||
failed_record_count=0,
|
||
skipped_record_count=0,
|
||
entity_types_json=list(entities),
|
||
)
|
||
self.target_session.add(batch)
|
||
self.target_session.commit()
|
||
return batch
|
||
|
||
def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
|
||
"""处理已持锁且标记 RUNNING 的任务；仅由直接入口或 worker 调用。"""
|
||
if batch.status != "RUNNING" or not batch.entity_types_json:
|
||
raise ValueError("任务尚未领取或读取范围不完整")
|
||
mappings = [get_mapping(entity) for entity in batch.entity_types_json]
|
||
progress.write(batch, phase="COUNTING")
|
||
self.target_session.commit()
|
||
try:
|
||
totals = self.reader.count_entities(batch.entity_types_json)
|
||
if isinstance(totals, dict) and all(isinstance(v, int) and v >= 0 for v in totals.values()) and set(totals) == set(batch.entity_types_json):
|
||
progress.write(batch, phase="READING", totals=totals)
|
||
except Exception as exc:
|
||
# 总量统计不可用时仍继续读源；页面显示未知总量，不伪造百分比。
|
||
self.source_session.rollback()
|
||
logger.warning("T+总量统计暂不可用，继续读取（%s）", type(exc).__name__)
|
||
self.target_session.commit()
|
||
interrupted = False
|
||
for mapping in mappings:
|
||
try:
|
||
self._read_entity(batch, mapping, batch_size)
|
||
except Exception as exc:
|
||
self.target_session.rollback()
|
||
code = classify_source_error(exc)
|
||
self._record_error(batch, mapping, {}, code, type(exc).__name__)
|
||
progress.write(batch, failed=mapping.entity_type)
|
||
self.target_session.commit()
|
||
if code in TRANSIENT_CODES:
|
||
interrupted = True
|
||
break
|
||
batch.status = "SUCCEEDED"
|
||
if batch.failed_record_count:
|
||
failed_rows = (
|
||
self.target_session.scalar(
|
||
select(func.count())
|
||
.select_from(TPlusSyncError)
|
||
.where(
|
||
TPlusSyncError.batch_no == batch.batch_no,
|
||
TPlusSyncError.error_code == "STAGE_ERROR",
|
||
)
|
||
)
|
||
or 0
|
||
)
|
||
# 中途断连仍是未完成的全量采集；保留已暂存页，重跑由哈希防重。
|
||
batch.status = "PARTIAL" if not interrupted and batch.source_record_count > failed_rows else "FAILED"
|
||
batch.error_message = "部分数据未完成，请查询同步错误记录并重跑全量快照"
|
||
batch.finished_at = datetime.now()
|
||
progress.write(batch, phase="COMPLETED" if batch.status == "SUCCEEDED" else "FAILED")
|
||
batch.watermark_to = None
|
||
self.target_session.commit()
|
||
return batch
|
||
|
||
def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
|
||
"""批量写入一类数据；每页提交后即使进程中断也可幂等重跑。"""
|
||
progress.write(batch, phase="READING", active=mapping.entity_type)
|
||
self.target_session.commit()
|
||
read = 0
|
||
source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
|
||
for chunk in batched(source, size):
|
||
inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
|
||
observe_rows(self.target_session, batch, mapping, list(chunk))
|
||
batch.source_record_count += len(chunk)
|
||
read += len(chunk)
|
||
progress.write(batch, read=read)
|
||
batch.staged_record_count += inserted
|
||
batch.skipped_record_count += len(chunk) - inserted - len(errors)
|
||
for row, reason in errors:
|
||
self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
|
||
self.target_session.commit()
|
||
report = dict(batch.audit_result_json or {})
|
||
report["source_head_version"] = 1
|
||
report["observed_entities"] = list(
|
||
dict.fromkeys([*report.get("observed_entities", []), mapping.entity_type])
|
||
)
|
||
batch.audit_result_json = report
|
||
progress.write(batch, done=mapping.entity_type)
|
||
self.target_session.commit()
|
||
|
||
def _record_error(
|
||
self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
|
||
) -> None:
|
||
"""只记录错误类别和源 ID，避免数据库异常携带个人信息或凭据。"""
|
||
self.target_session.add(
|
||
TPlusSyncError(
|
||
batch_no=batch.batch_no,
|
||
entity_type=mapping.entity_type,
|
||
source_table=mapping.source_table,
|
||
source_id=str(row["id"]) if row.get("id") is not None else None,
|
||
source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
|
||
error_code=code,
|
||
error_message=f"源数据处理失败（{reason}），请核对连接和字段",
|
||
)
|
||
)
|
||
batch.failed_record_count += 1
|