"""T+ 只读快照同步:批量 staging、幂等去重和失败隔离。""" from collections.abc import Iterable from datetime import datetime from itertools import batched from pathlib import Path from sqlalchemy import func, select from sqlalchemy.orm import Session from app.core.config import get_settings from app.core.tplus_lock import sync_lock from app.models import TPlusSyncBatch, TPlusSyncError from app.services.tplus.mapping import TPlusMapping, get_mapping from app.services.tplus.reader import TPlusReader from app.services.tplus.staging import stage_rows from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running class TPlusSyncService: """按分页提交原始快照,业务表始终不参与本服务。""" def __init__(self, source_session: Session, target_session: Session) -> None: self.source_session = source_session self.target_session = target_session configured = get_settings().t_plus_query_dir self.reader = TPlusReader(source_session, Path(configured) if configured else None) def run( self, *, entity_types: Iterable[str], watermark: datetime | None = None, batch_no: str | None = None, batch_size: int = 500, ) -> TPlusSyncBatch: """执行可重复的全量读取,提交每页进度;字段语义未定时拒绝时间增量。""" entities = tuple(dict.fromkeys(entity_types)) if not entities or not 1 <= batch_size <= 5000: raise ValueError("至少指定一个实体,batch_size 必须在 1 到 5000 之间") if watermark is not None: raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重") for entity in entities: get_mapping(entity) with sync_lock(self.target_session): recover_running(self.target_session) batch = self._start_direct(entities, batch_no) try: return self.execute_claimed(batch, batch_size) except BaseException: self.target_session.rollback() fail_batch(self.target_session, batch, "INTERRUPTED") raise def _start_direct(self, entities, batch_no): batch = TPlusSyncBatch( batch_no=batch_no or new_batch_no(), sync_mode="FULL", status="RUNNING", source_database=get_settings().t_plus_database or None, started_at=datetime.now(), source_record_count=0, staged_record_count=0, failed_record_count=0, skipped_record_count=0, entity_types_json=list(entities), ) self.target_session.add(batch) self.target_session.commit() return batch def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500): """处理已持锁且标记 RUNNING 的任务;仅由直接入口或 worker 调用。""" if batch.status != "RUNNING" or not batch.entity_types_json: raise ValueError("任务尚未领取或读取范围不完整") mappings = [get_mapping(entity) for entity in batch.entity_types_json] for mapping in mappings: try: self._read_entity(batch, mapping, batch_size) except Exception as exc: self.target_session.rollback() self._record_error(batch, mapping, {}, "READ_ERROR", type(exc).__name__) self.target_session.commit() batch.status = "SUCCEEDED" if batch.failed_record_count: failed_rows = ( self.target_session.scalar( select(func.count()) .select_from(TPlusSyncError) .where( TPlusSyncError.batch_no == batch.batch_no, TPlusSyncError.error_code == "STAGE_ERROR", ) ) or 0 ) batch.status = "PARTIAL" if batch.source_record_count > failed_rows else "FAILED" batch.error_message = "部分数据未完成,请查询同步错误记录并重跑全量快照" batch.finished_at = datetime.now() batch.watermark_to = None self.target_session.commit() return batch def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None: """批量写入一类数据;每页提交后即使进程中断也可幂等重跑。""" source = self.reader.iter_entity(mapping.entity_type, batch_size=size) for chunk in batched(source, size): inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk)) batch.source_record_count += len(chunk) batch.staged_record_count += inserted batch.skipped_record_count += len(chunk) - inserted - len(errors) for row, reason in errors: self._record_error(batch, mapping, row, "STAGE_ERROR", reason) self.target_session.commit() def _record_error( self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str ) -> None: """只记录错误类别和源 ID,避免数据库异常携带个人信息或凭据。""" self.target_session.add( TPlusSyncError( batch_no=batch.batch_no, entity_type=mapping.entity_type, source_table=mapping.source_table, source_id=str(row["id"]) if row.get("id") is not None else None, source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None, error_code=code, error_message=f"源数据处理失败({reason}),请核对连接和字段", ) ) batch.failed_record_count += 1