108bf87225
补齐源快照、资料预演、正式导入及工资核算的依赖链路。 新增版本化输入、逐段 FIFO、生成锁和结构化当月调整， 并提供默认关闭的周期采集、有限重试及 Windows 托管。 原始客户资料、私有配置及首页改动不纳入本次提交。 BREAKING CHANGE: 升级前需应用新增迁移，并通过 PAYROLL_MAKER_USER_ID 绑定实际工资制作人，否则业务只读。 旧单批次核算默认关闭；历史兼容需显式启用 PAYROLL_ALLOW_LEGACY_CALCULATION。生产启用和真实金额待验收。
148 lines
6.6 KiB
Python
148 lines
6.6 KiB
Python
"""T+ 只读快照同步：批量 staging、幂等去重和失败隔离。"""
|
||
|
||
from collections.abc import Iterable
|
||
from datetime import datetime
|
||
from itertools import batched
|
||
from pathlib import Path
|
||
|
||
from sqlalchemy import func, select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.config import get_settings
|
||
from app.core.tplus_lock import sync_lock
|
||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||
from app.services.tplus.errors import TRANSIENT_CODES, classify_source_error
|
||
from app.services.tplus.mapping import TPlusMapping, get_mapping
|
||
from app.services.tplus.reader import TPlusReader
|
||
from app.services.tplus.source_heads import observe_rows
|
||
from app.services.tplus.staging import stage_rows
|
||
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running
|
||
|
||
|
||
class TPlusSyncService:
|
||
"""按分页提交原始快照，业务表始终不参与本服务。"""
|
||
|
||
def __init__(self, source_session: Session, target_session: Session) -> None:
|
||
self.source_session = source_session
|
||
self.target_session = target_session
|
||
configured = get_settings().t_plus_query_dir
|
||
self.reader = TPlusReader(source_session, Path(configured) if configured else None)
|
||
|
||
def run(
|
||
self,
|
||
*,
|
||
entity_types: Iterable[str],
|
||
watermark: datetime | None = None,
|
||
batch_no: str | None = None,
|
||
batch_size: int = 500,
|
||
) -> TPlusSyncBatch:
|
||
"""执行可重复的全量读取，提交每页进度；字段语义未定时拒绝时间增量。"""
|
||
entities = tuple(dict.fromkeys(entity_types))
|
||
if not entities or not 1 <= batch_size <= 5000:
|
||
raise ValueError("至少指定一个实体，batch_size 必须在 1 到 5000 之间")
|
||
if watermark is not None:
|
||
raise ValueError("增量水位语义尚未确认，当前只支持全量读取和哈希去重")
|
||
for entity in entities:
|
||
get_mapping(entity)
|
||
with sync_lock(self.target_session):
|
||
recover_running(self.target_session)
|
||
batch = self._start_direct(entities, batch_no)
|
||
try:
|
||
return self.execute_claimed(batch, batch_size)
|
||
except BaseException:
|
||
self.target_session.rollback()
|
||
fail_batch(self.target_session, batch, "INTERRUPTED")
|
||
raise
|
||
|
||
def _start_direct(self, entities, batch_no):
|
||
batch = TPlusSyncBatch(
|
||
batch_no=batch_no or new_batch_no(),
|
||
sync_mode="FULL",
|
||
status="RUNNING",
|
||
source_database=get_settings().t_plus_database or None,
|
||
started_at=datetime.now(),
|
||
source_record_count=0,
|
||
staged_record_count=0,
|
||
failed_record_count=0,
|
||
skipped_record_count=0,
|
||
entity_types_json=list(entities),
|
||
)
|
||
self.target_session.add(batch)
|
||
self.target_session.commit()
|
||
return batch
|
||
|
||
def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
|
||
"""处理已持锁且标记 RUNNING 的任务；仅由直接入口或 worker 调用。"""
|
||
if batch.status != "RUNNING" or not batch.entity_types_json:
|
||
raise ValueError("任务尚未领取或读取范围不完整")
|
||
mappings = [get_mapping(entity) for entity in batch.entity_types_json]
|
||
interrupted = False
|
||
for mapping in mappings:
|
||
try:
|
||
self._read_entity(batch, mapping, batch_size)
|
||
except Exception as exc:
|
||
self.target_session.rollback()
|
||
code = classify_source_error(exc)
|
||
self._record_error(batch, mapping, {}, code, type(exc).__name__)
|
||
self.target_session.commit()
|
||
if code in TRANSIENT_CODES:
|
||
interrupted = True
|
||
break
|
||
batch.status = "SUCCEEDED"
|
||
if batch.failed_record_count:
|
||
failed_rows = (
|
||
self.target_session.scalar(
|
||
select(func.count())
|
||
.select_from(TPlusSyncError)
|
||
.where(
|
||
TPlusSyncError.batch_no == batch.batch_no,
|
||
TPlusSyncError.error_code == "STAGE_ERROR",
|
||
)
|
||
)
|
||
or 0
|
||
)
|
||
# 中途断连仍是未完成的全量采集；保留已暂存页，重跑由哈希防重。
|
||
batch.status = "PARTIAL" if not interrupted and batch.source_record_count > failed_rows else "FAILED"
|
||
batch.error_message = "部分数据未完成，请查询同步错误记录并重跑全量快照"
|
||
batch.finished_at = datetime.now()
|
||
batch.watermark_to = None
|
||
self.target_session.commit()
|
||
return batch
|
||
|
||
def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
|
||
"""批量写入一类数据；每页提交后即使进程中断也可幂等重跑。"""
|
||
source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
|
||
for chunk in batched(source, size):
|
||
inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
|
||
observe_rows(self.target_session, batch, mapping, list(chunk))
|
||
batch.source_record_count += len(chunk)
|
||
batch.staged_record_count += inserted
|
||
batch.skipped_record_count += len(chunk) - inserted - len(errors)
|
||
for row, reason in errors:
|
||
self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
|
||
self.target_session.commit()
|
||
report = dict(batch.audit_result_json or {})
|
||
report["source_head_version"] = 1
|
||
report["observed_entities"] = list(
|
||
dict.fromkeys([*report.get("observed_entities", []), mapping.entity_type])
|
||
)
|
||
batch.audit_result_json = report
|
||
self.target_session.commit()
|
||
|
||
def _record_error(
|
||
self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
|
||
) -> None:
|
||
"""只记录错误类别和源 ID，避免数据库异常携带个人信息或凭据。"""
|
||
self.target_session.add(
|
||
TPlusSyncError(
|
||
batch_no=batch.batch_no,
|
||
entity_type=mapping.entity_type,
|
||
source_table=mapping.source_table,
|
||
source_id=str(row["id"]) if row.get("id") is not None else None,
|
||
source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
|
||
error_code=code,
|
||
error_message=f"源数据处理失败（{reason}），请核对连接和字段",
|
||
)
|
||
)
|
||
batch.failed_record_count += 1
|