feat(tplus): 新增 T+ 只读同步服务与同步管理

This commit is contained in:
liuwenxin committed 2026-10-28 11:42:15 +08:00
1 parent 78bef43c06
commit 4fe5916c2c
43 files changed
+3309 -388

No files matched your search

+2
View File
@@ -26,6 +26,7 @@ from app.api.v1.system import logs as system_logs
from app.api.v1.system import params as system_params
from app.api.v1.system import roles as system_roles
from app.api.v1.system import settings as system_settings
from app.api.v1.system import tplus as system_tplus
from app.api.v1.system import users as system_users
api_router = APIRouter()
@@ -47,6 +48,7 @@ system_router.include_router(system_settings.router)
system_router.include_router(system_approvals.router)
system_router.include_router(system_logs.operation_router)
system_router.include_router(system_logs.login_router)
system_router.include_router(system_tplus.router)
v1_router.include_router(system_router)
# 基础数据(需求 2.1):/api/v1/base/*,业务操作登录即可访问
+71
View File
@@ -0,0 +1,71 @@
"""管理员管理 T+ 只读任务;长任务交给独立 worker 执行。"""
from typing import Annotated
from fastapi import APIRouter, Path, Query
from app.api.deps import AdminUser, DbSession, RequestMetaDep
from app.core.response import ApiResponse, PageResult, ok
from app.schemas.tplus import (
TPlusBatchDetailOut,
TPlusBatchOut,
TPlusBatchQuery,
TPlusCreateIn,
TPlusErrorOut,
TPlusErrorQuery,
TPlusOverviewOut,
TPlusRetryIn,
)
from app.services.tplus import query_service, task_service
router = APIRouter(prefix="/tplus", tags=["系统管理 · T+ 数据同步"])
BatchNumber = Annotated[str, Path(min_length=1, max_length=32, pattern=r"^[A-Za-z0-9_-]+$")]
@router.get("/overview", response_model=ApiResponse[TPlusOverviewOut])
def overview(session: DbSession, _: AdminUser):
"""查询配置状态、任务统计和最近核验。"""
return ok(query_service.overview(session))
@router.get("/batches", response_model=ApiResponse[PageResult[TPlusBatchOut]])
def batches(session: DbSession, _: AdminUser, query: Annotated[TPlusBatchQuery, Query()]):
"""分页查询批次元数据。"""
return ok(query_service.list_batches(session, query))
@router.get("/batches/{batch_no}", response_model=ApiResponse[TPlusBatchDetailOut])
def detail(session: DbSession, _: AdminUser, batch_no: BatchNumber):
"""查询批次详情与受控的核验报告。"""
return ok(query_service.get_batch(session, batch_no))
@router.get("/batches/{batch_no}/errors", response_model=ApiResponse[PageResult[TPlusErrorOut]])
def errors(
session: DbSession,
_: AdminUser,
batch_no: BatchNumber,
query: Annotated[TPlusErrorQuery, Query()],
):
"""分页查询异常定位和固定错误说明。"""
return ok(query_service.list_errors(session, batch_no, query))
@router.post("/batches", response_model=ApiResponse[TPlusBatchDetailOut])
def create(session: DbSession, admin: AdminUser, meta: RequestMetaDep, payload: TPlusCreateIn):
"""将读取或核验任务持久化排队。"""
batch = task_service.create_task(session, payload, task_service.TaskActor(admin, meta))
return ok(query_service.get_batch(session, batch.batch_no))
@router.post("/batches/{batch_no}/retry", response_model=ApiResponse[TPlusBatchDetailOut])
def retry(
session: DbSession,
admin: AdminUser,
meta: RequestMetaDep,
batch_no: BatchNumber,
payload: TPlusRetryIn,
):
"""新建重试批次,保留原批次与异常历史。"""
batch = task_service.retry_task(session, batch_no, payload, task_service.TaskActor(admin, meta))
return ok(query_service.get_batch(session, batch.batch_no))
+93 -1
View File
@@ -11,13 +11,20 @@
import argparse
import getpass
import sys
import time
from datetime import datetime
from sqlalchemy import select
from app.core.config import get_settings
from app.core.constants import PASSWORD_MAX_BYTES, PASSWORD_MIN_BYTES, ROLE_ADMIN
from app.core.db import dispose_engine, get_session_factory
from app.core.security import hash_password
from app.core.tplus_db import dispose_tplus_engine, get_tplus_session_factory
from app.models import Role, User, UserRole
from app.repositories.tplus_repository import get_batch
from app.services.tplus.sync_service import TPlusSyncService
from app.services.tplus.worker import run_once
def _prompt_password() -> str:
@@ -80,6 +87,54 @@ def _create_admin(args: argparse.Namespace) -> int:
return 0
def _stage_tplus(args: argparse.Namespace) -> int:
"""读取 T+ 源库并写入 MySQL staging。"""
target_factory = get_session_factory()
source_factory = get_tplus_session_factory()
watermark = datetime.fromisoformat(args.watermark) if args.watermark else None
with source_factory() as source_session, target_factory() as target_session:
batch = TPlusSyncService(source_session, target_session).run(
entity_types=args.entity,
watermark=watermark,
batch_size=args.batch_size,
)
print(
f"T+ staging 完成:批次={batch.batch_no} 状态={batch.status} "
f"源记录={batch.source_record_count} 新增={batch.staged_record_count} "
f"失败={batch.failed_record_count}"
)
return 0 if batch.status == "SUCCEEDED" else 1
def _tplus_worker(args: argparse.Namespace) -> int:
"""独立消费持久化任务;HTTP 进程不承担长时间读取。"""
if args.connection_file:
settings = get_settings()
settings.t_plus_enabled, settings.t_plus_connection_file = True, args.connection_file
if not 2 <= args.poll_seconds <= 60:
print("轮询间隔必须在 2 到 60 秒之间", file=sys.stderr)
return 1
factory = get_session_factory()
try:
while True:
with factory() as session:
batch_no = run_once(session)
batch = get_batch(session, batch_no) if batch_no else None
status = batch.status if batch else None
if batch_no:
print(f"任务完成:{batch_no} 状态={status}", flush=True)
if not args.watch:
return 0 if status in (None, "SUCCEEDED") else 1
if batch_no is None or status != "SUCCEEDED":
time.sleep(args.poll_seconds)
except KeyboardInterrupt:
print("T+ 读取服务已停止")
return 130
except Exception as exc:
print(f"T+ 读取服务停止({type(exc).__name__}),请检查运行环境", file=sys.stderr)
return 1
def main() -> int:
"""命令行入口。"""
parser = argparse.ArgumentParser(
@@ -96,12 +151,49 @@ def main() -> int:
help="密码。不传则交互式输入(推荐)——命令行传参会留在 shell 历史中",
)
p_tplus = sub.add_parser("tplus-stage", help="读取 T+ 源库并写入原始 staging")
p_tplus.add_argument(
"--entity",
action="append",
choices=[
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
"business_items",
"shipment_items",
"receipt_items",
],
required=True,
help="实体类型,可重复指定",
)
p_tplus.add_argument(
"--watermark",
default=None,
help="增量起始时间,ISO 8601 格式;不传表示全量读取",
)
p_tplus.add_argument("--batch-size", type=int, default=500, help="每次从源库读取的行数")
p_worker = sub.add_parser("tplus-worker", help="执行已排队的 T+ 读取或关联核验")
modes = p_worker.add_mutually_exclusive_group()
modes.add_argument("--once", action="store_true", help="执行一个任务后退出(默认)")
modes.add_argument("--watch", action="store_true", help="持续消费手动排队任务")
p_worker.add_argument("--connection-file", default=None, help="显式指定项目外只读连接 JSON")
p_worker.add_argument("--poll-seconds", type=int, default=3, help="空闲轮询间隔,2–60 秒")
args = parser.parse_args()
try:
return _create_admin(args)
if args.command == "create-admin":
return _create_admin(args)
if args.command == "tplus-worker":
return _tplus_worker(args)
return _stage_tplus(args)
finally:
dispose_engine()
dispose_tplus_engine()
if __name__ == "__main__":
+42
View File
@@ -58,6 +58,23 @@ class Settings(BaseSettings):
redis_password: str = ""
redis_db: int = 0
# ===== T+ 只读 SQL Server =====
# T+ 数据库默认关闭,避免开发者未配置驱动时影响主业务启动。
t_plus_enabled: bool = False
t_plus_host: str = "127.0.0.1"
t_plus_port: int = 1433
t_plus_database: str = ""
t_plus_user: str = ""
t_plus_password: str = Field(default="", repr=False)
t_plus_connection_file: str = ""
# 部署前必须指定已在 SQL Server 2008 R2 实测的驱动,不自动选择最新版。
t_plus_odbc_driver: str = ""
t_plus_encrypt: bool = False
t_plus_trust_server_certificate: bool = True
t_plus_connect_timeout: int = 10
t_plus_query_timeout: int = Field(default=120, ge=1, le=600)
t_plus_query_dir: str = ""
@field_validator("app_secret_key")
@classmethod
def _check_secret_key(cls, v: str) -> str:
@@ -92,6 +109,31 @@ class Settings(BaseSettings):
auth = f":{self.redis_password}@" if self.redis_password else ""
return f"redis://{auth}{self.redis_host}:{self.redis_port}/{self.redis_db}"
@property
def t_plus_database_url(self) -> URL:
"""构造 T+ 只读 SQL Server 连接 URL。
连接 URL 只在启用 T+ 适配器时使用;密码由 SQLAlchemy URL 对象保存,
日志中调用 ``render_as_string()`` 时仍会默认隐藏密码。
"""
query = {
"driver": self.t_plus_odbc_driver,
"Encrypt": "yes" if self.t_plus_encrypt else "no",
"TrustServerCertificate": "yes"
if self.t_plus_trust_server_certificate
else "no",
"Connection Timeout": str(self.t_plus_connect_timeout),
}
return URL.create(
drivername="mssql+pyodbc",
username=self.t_plus_user,
password=self.t_plus_password,
host=self.t_plus_host,
port=self.t_plus_port,
database=self.t_plus_database,
query=query,
)
@property
def upload_dir(self) -> Path:
"""上传文件根目录(公司 logo 等):backend/uploads/。
+112
View File
@@ -0,0 +1,112 @@
"""T+ 只读 SQL Server 连接。
T+ 是外部源库,业务接口不直接依赖它。连接采用延迟初始化,未配置或未启用
时不会影响主业务服务启动;适配器执行查询时才建立连接池。
"""
import json
from collections.abc import Generator
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
from sqlalchemy import Engine, create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from app.core.config import Settings, get_settings
_engine: Engine | None = None
_session_factory: sessionmaker[Session] | None = None
class _ConnectionFile(BaseModel):
"""连接文件仅允许源库字段,密码不会出现在模型表示中。"""
model_config = ConfigDict(extra="forbid", strict=True)
host: str = Field(min_length=1, max_length=255)
port: int = Field(ge=1, le=65535)
database: str = Field(min_length=1, max_length=128)
user: str = Field(min_length=1, max_length=128)
password: SecretStr
driver: str = Field(min_length=1, max_length=128)
def resolve_tplus_settings(settings: Settings) -> Settings:
"""解析显式配置的只读连接,返回副本,不修改应用全局配置。"""
if not settings.t_plus_enabled:
raise RuntimeError("T+ 数据源未启用,请设置 T_PLUS_ENABLED=true")
resolved = settings
if settings.t_plus_connection_file:
try:
path = Path(settings.t_plus_connection_file)
if path.stat().st_size > 16384:
raise ValueError("连接文件过大")
config = _ConnectionFile.model_validate(
json.loads(path.read_text(encoding="utf-8-sig"))
)
except (OSError, ValueError, ValidationError):
raise RuntimeError("T+ 只读连接文件不可用或字段不正确") from None
values = {
"t_plus_" + key: getattr(config, key) for key in ("host", "port", "database", "user")
}
values.update(
t_plus_password=config.password.get_secret_value(), t_plus_odbc_driver=config.driver
)
resolved = settings.model_copy(update=values)
if resolved.t_plus_user.strip().casefold() == "sa":
raise RuntimeError("应用必须使用 T+ 专用只读账号,不能使用 sa")
if not resolved.t_plus_database or not resolved.t_plus_user or not resolved.t_plus_odbc_driver:
raise RuntimeError("T+ 只读连接缺少数据库、账号或已验证的 ODBC 驱动")
return resolved
def get_tplus_engine() -> Engine:
"""获取 T+ 只读引擎;未启用或缺少 ODBC 驱动时给出可读错误。"""
global _engine, _session_factory
if _engine is not None:
return _engine
settings = resolve_tplus_settings(get_settings())
try:
_engine = create_engine(
settings.t_plus_database_url,
pool_pre_ping=True,
pool_recycle=1800,
pool_timeout=settings.t_plus_connect_timeout,
echo=False,
hide_parameters=True,
)
except ModuleNotFoundError as exc:
raise RuntimeError(
"T+ 连接需要 pyodbc(uv sync --extra tplus)和已验证的 SQL Server ODBC 驱动"
) from exc
_session_factory = sessionmaker(_engine, expire_on_commit=False)
def configure_timeout(connection, _record):
connection.timeout = settings.t_plus_query_timeout
event.listen(_engine, "connect", configure_timeout)
return _engine
def get_tplus_session_factory() -> sessionmaker[Session]:
"""获取 T+ SQLAlchemy 会话工厂。"""
get_tplus_engine()
assert _session_factory is not None
return _session_factory
def get_tplus_db() -> Generator[Session, None, None]:
"""提供只读 T+ 会话;调用方不得 commit 或执行写操作。"""
factory = get_tplus_session_factory()
with factory() as session:
yield session
def dispose_tplus_engine() -> None:
"""释放 T+ 连接池。"""
global _engine, _session_factory
if _engine is not None:
_engine.dispose()
_engine = None
_session_factory = None
+48
View File
@@ -0,0 +1,48 @@
"""同账套任务串行锁;独立持有 MySQL 连接,跨分页 commit 仍保持有效。"""
import hashlib
from contextlib import contextmanager
from sqlalchemy import text
from sqlalchemy.orm import Session
class TPlusBusyError(RuntimeError):
"""已有任务持有执行锁。"""
def lock_name(session: Session) -> str:
"""生成不含地址、账号或密码的固定长度锁名。"""
database = session.get_bind().url.database or "default"
return "ys:tplus:read:" + hashlib.sha256(database.encode()).hexdigest()[:24]
@contextmanager
def sync_lock(session: Session):
"""只在取得锁后执行;释放失败时使连接失效,防止带锁回池。"""
engine = session.get_bind()
if engine.dialect.name != "mysql":
raise RuntimeError("T+ 执行协调需要 MySQL 命名锁")
connection = engine.connect()
acquired = False
try:
acquired = (
connection.execute(
text("SELECT GET_LOCK(:name, 0)"), {"name": lock_name(session)}
).scalar_one()
== 1
)
if not acquired:
raise TPlusBusyError("已有 T+ 任务正在执行,请稍后重试")
yield
finally:
if acquired:
try:
released = connection.execute(
text("SELECT RELEASE_LOCK(:name)"), {"name": lock_name(session)}
).scalar_one()
if released != 1:
connection.invalidate()
except Exception:
connection.invalidate()
connection.close()
+2
View File
@@ -15,6 +15,7 @@ from app.core.db import dispose_engine
from app.core.exception_handlers import register_exception_handlers
from app.core.logging import setup_logging
from app.core.redis_client import close_redis
from app.core.tplus_db import dispose_tplus_engine
logger = logging.getLogger(__name__)
@@ -35,6 +36,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
)
yield
dispose_engine()
dispose_tplus_engine()
close_redis()
logger.info("应用已停止,连接池已释放")
+10
View File
@@ -26,6 +26,12 @@ from app.models.role import Permission, Role, RolePermission
from app.models.settlement import AdjustmentOrder, MonthlySettlement, SettlementItem
from app.models.shipment import Shipment, ShipmentItem
from app.models.system_setting import SystemSetting
from app.models.tplus_sync import (
TPlusSyncBatch,
TPlusSyncError,
TPlusSyncRecord,
TPlusSyncWatermark,
)
from app.models.user import User, UserRole
__all__ = [
@@ -60,6 +66,10 @@ __all__ = [
"Shipment",
"ShipmentItem",
"SystemSetting",
"TPlusSyncBatch",
"TPlusSyncError",
"TPlusSyncRecord",
"TPlusSyncWatermark",
"TimestampMixin",
"User",
"UserRole",
+3
View File
@@ -91,6 +91,9 @@ class TPlusSyncMixin:
String(32), comment="同步批次号(同步模块写入)"
)
synced_at: Mapped[datetime | None] = mapped_column(DateTime, comment="最近同步时间")
source_updated_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="T+ 源数据更新时间(未确认时可为空)"
)
def mysql_table_args(comment: str, *extra: object) -> tuple[object, ...]:
+149
View File
@@ -0,0 +1,149 @@
"""T+ 同步批次、错误和原始 staging 记录。"""
from datetime import datetime
from sqlalchemy import JSON, DateTime, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import BIGINT_UNSIGNED, Base, TimestampMixin, mysql_table_args
class TPlusSyncBatch(Base, TimestampMixin):
"""一次 T+ 全量或增量同步的审计记录。"""
__tablename__ = "tplus_sync_batches"
__table_args__ = mysql_table_args(
"T+ 同步批次表(记录读取范围、结果和对账统计)",
UniqueConstraint("batch_no", name="uk_tplus_sync_batches_batch_no"),
UniqueConstraint("request_key", name="uk_tplus_sync_batches_request_key"),
Index("idx_tplus_sync_batches_status", "status"),
Index("idx_tplus_sync_batches_started_at", "started_at"),
Index("idx_tplus_sync_batches_parent_status", "parent_batch_no", "status"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
sync_mode: Mapped[str] = mapped_column(
String(16), nullable=False, comment="任务方式:FULL全量 AUDIT关联核验 INCREMENTAL预留"
)
status: Mapped[str] = mapped_column(
String(16), nullable=False, comment="状态:QUEUED/RUNNING/SUCCEEDED/PARTIAL/FAILED"
)
source_database: Mapped[str | None] = mapped_column(String(128), comment="T+ 源数据库名")
started_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="开始执行时间(排队时为空)"
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime, comment="结束时间")
watermark_from: Mapped[datetime | None] = mapped_column(DateTime, comment="增量起始水位")
watermark_to: Mapped[datetime | None] = mapped_column(DateTime, comment="本批次结束水位")
source_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="源记录数"
)
staged_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="写入 staging 的记录数"
)
failed_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="失败记录数"
)
error_message: Mapped[str | None] = mapped_column(Text, comment="批次错误摘要")
entity_types_json: Mapped[list[str] | None] = mapped_column(
JSON, comment="请求读取的实体类型;历史批次可为空"
)
skipped_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="重复快照跳过条数"
)
parent_batch_no: Mapped[str | None] = mapped_column(
String(32), comment="重试来源批次号,保留原批次记录"
)
request_key: Mapped[str | None] = mapped_column(
String(36), comment="请求幂等键,空值用于历史和直接CLI任务"
)
requested_by: Mapped[int | None] = mapped_column(
BIGINT_UNSIGNED, comment="请求用户ID,CLI任务为空,无物理外键"
)
audit_result_json: Mapped[dict | None] = mapped_column(
JSON, comment="关联核验结果,只含系统字典和汇总"
)
class TPlusSyncError(Base, TimestampMixin):
"""单条 T+ 数据读取、清洗或 staging 失败记录。"""
__tablename__ = "tplus_sync_errors"
__table_args__ = mysql_table_args(
"T+ 同步错误表(记录失败行和重试信息)",
Index("idx_tplus_sync_errors_batch_no", "batch_no"),
Index("idx_tplus_sync_errors_entity_type", "entity_type"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
source_id: Mapped[str | None] = mapped_column(String(64), comment="T+ 源主键")
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
error_code: Mapped[str] = mapped_column(String(32), nullable=False, comment="错误编码")
error_message: Mapped[str] = mapped_column(Text, nullable=False, comment="错误说明")
retry_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="已重试次数"
)
class TPlusSyncRecord(Base, TimestampMixin):
"""T+ 原始行 staging,保存映射未定前的可追溯数据。"""
__tablename__ = "tplus_sync_records"
__table_args__ = mysql_table_args(
"T+ 原始同步 staging 表(映射确认前只保存原始行,不直接改业务表)",
UniqueConstraint(
"entity_type", "source_id", "row_hash", name="uk_tplus_sync_records_entity_source_hash"
),
Index("idx_tplus_sync_records_batch_no", "batch_no"),
Index("idx_tplus_sync_records_source_no", "source_no"),
Index("idx_tplus_sync_records_state", "state"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
source_id: Mapped[str] = mapped_column(String(64), nullable=False, comment="T+ 源主键")
source_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 原始编号")
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime, comment="T+ 源更新时间")
row_hash: Mapped[str] = mapped_column(String(64), nullable=False, comment="原始行 SHA-256")
state: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default="PENDING",
comment="状态:PENDING/APPLIED/SKIPPED/ERROR",
)
payload_json: Mapped[dict] = mapped_column(JSON, nullable=False, comment="原始行 JSON")
class TPlusSyncWatermark(Base, TimestampMixin):
"""按实体保存增量同步水位。"""
__tablename__ = "tplus_sync_watermarks"
__table_args__ = mysql_table_args(
"T+ 增量同步水位表",
UniqueConstraint("entity_type", name="uk_tplus_sync_watermarks_entity_type"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
last_source_updated_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="最近源更新时间"
)
last_source_id: Mapped[str | None] = mapped_column(String(64), comment="同一时间点的最后源主键")
status: Mapped[str] = mapped_column(
String(16), nullable=False, server_default="READY", comment="状态:READY/BLOCKED"
)
@@ -0,0 +1,149 @@
"""T+ 管理库访问;只 flush,事务由 service 管理。"""
from sqlalchemy import func, select, update
from sqlalchemy.orm import Session, defer
from app.models import TPlusSyncBatch, TPlusSyncError
def get_batch(
session: Session, batch_no: str, *, for_update: bool = False, with_result: bool = False
):
"""按批次号查询;普通查询不加载核验大字段。"""
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == batch_no)
if not with_result:
statement = statement.options(defer(TPlusSyncBatch.audit_result_json))
if for_update:
statement = statement.with_for_update().execution_options(populate_existing=True)
return session.scalar(statement)
def get_request(session: Session, key: str, *, for_update: bool = False):
"""锁读用于唯一约束竞争后的当前读,避免旧事务快照看不到新任务。"""
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.request_key == key)
if for_update:
statement = statement.with_for_update().execution_options(populate_existing=True)
return session.scalar(statement)
def active_retry(session: Session, parent_no: str):
"""调用方先锁父批次,再用当前读判断是否已有未完成重试。"""
return session.scalar(
select(TPlusSyncBatch)
.where(
TPlusSyncBatch.parent_batch_no == parent_no,
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
)
.with_for_update()
)
def increment_retry_errors(session: Session, parent_no: str) -> None:
"""创建新重试时对原异常累计重试次数。"""
session.execute(
update(TPlusSyncError)
.where(TPlusSyncError.batch_no == parent_no)
.values(retry_count=TPlusSyncError.retry_count + 1)
)
session.flush()
def next_queued(session: Session):
"""调用方持有账套执行锁后领取最早排队任务。"""
return session.scalar(
select(TPlusSyncBatch)
.where(TPlusSyncBatch.status == "QUEUED")
.order_by(TPlusSyncBatch.id)
.limit(1)
.with_for_update()
)
def running_batches(session: Session) -> list[TPlusSyncBatch]:
"""只在持有执行锁时使用,不能依据时间戳判断任务已经停止。"""
return list(
session.scalars(
select(TPlusSyncBatch).where(TPlusSyncBatch.status == "RUNNING").with_for_update()
).all()
)
def list_batches(session: Session, query):
"""分页只读元数据,核验报告和原异常不随列表加载。"""
filters = []
if query.status:
filters.append(TPlusSyncBatch.status == query.status)
if query.task_type:
filters.append(
TPlusSyncBatch.sync_mode == ("AUDIT" if query.task_type == "AUDIT" else "FULL")
)
total = session.scalar(select(func.count()).select_from(TPlusSyncBatch).where(*filters))
rows = session.scalars(
select(TPlusSyncBatch)
.where(*filters)
.options(defer(TPlusSyncBatch.audit_result_json), defer(TPlusSyncBatch.error_message))
.order_by(TPlusSyncBatch.id.desc())
.offset(query.offset)
.limit(query.page_size)
).all()
return rows, total or 0
def active_retry_parents(session: Session, numbers: list[str]) -> set[str]:
"""整页一次获取未完成的重试,避免逐行查询。"""
if not numbers:
return set()
return set(
session.scalars(
select(TPlusSyncBatch.parent_batch_no).where(
TPlusSyncBatch.parent_batch_no.in_(numbers),
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
)
).all()
)
def list_errors(session: Session, number: str, query):
"""只读取当前批次的异常定位元数据。"""
filters = [TPlusSyncError.batch_no == number]
if query.entity_type:
filters.append(TPlusSyncError.entity_type == query.entity_type)
total = session.scalar(select(func.count()).select_from(TPlusSyncError).where(*filters))
rows = session.scalars(
select(TPlusSyncError)
.where(*filters)
.options(defer(TPlusSyncError.error_message))
.order_by(TPlusSyncError.id.desc())
.offset(query.offset)
.limit(query.page_size)
).all()
return rows, total or 0
def status_counts(session: Session) -> dict[str, int]:
"""汇总任务表,不扫描原始快照。"""
return dict(
session.execute(
select(TPlusSyncBatch.status, func.count()).group_by(TPlusSyncBatch.status)
).all()
)
def latest_audit(session: Session):
"""最近完成的核验;返回批次供查询层过滤报告。"""
return session.scalar(
select(TPlusSyncBatch)
.where(
TPlusSyncBatch.sync_mode == "AUDIT",
TPlusSyncBatch.finished_at.is_not(None),
)
.order_by(TPlusSyncBatch.id.desc())
.limit(1)
)
def total_staged_records(session: Session) -> int:
"""从批次元数据汇总累计新增数,避免扫描原始快照。"""
return (
session.scalar(select(func.coalesce(func.sum(TPlusSyncBatch.staged_record_count), 0))) or 0
)
+109
View File
@@ -0,0 +1,109 @@
"""T+ 管理接口合同;只暴露范围、数量和受控核验结果。"""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field, model_validator
from app.core.response import PageQuery
EntityType = Literal[
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
"business_items",
"shipment_items",
"receipt_items",
]
TaskType = Literal["READ", "AUDIT"]
TaskStatus = Literal["QUEUED", "RUNNING", "SUCCEEDED", "PARTIAL", "FAILED"]
class TPlusCreateIn(BaseModel):
"""创建长任务;幂等键是调用意图的一部分,必须由调用方保留。"""
task_type: TaskType = "READ"
entity_types: list[EntityType] = Field(default_factory=list, max_length=9)
request_key: UUID
@model_validator(mode="after")
def check_scope(self):
if self.task_type == "READ" and not self.entity_types:
raise ValueError("请选择至少一种读取数据")
if self.task_type == "AUDIT" and self.entity_types:
raise ValueError("关联核验不接受自定义读取范围")
if len(set(self.entity_types)) != len(self.entity_types):
raise ValueError("读取类型不能重复")
return self
class TPlusRetryIn(BaseModel):
"""失败批次重试不接受任意新范围。"""
request_key: UUID
class TPlusBatchQuery(PageQuery):
"""批次筛选,与读取状态和任务类型保持一致。"""
status: TaskStatus | None = None
task_type: TaskType | None = None
class TPlusErrorQuery(PageQuery):
"""异常只能在指定批次内查询。"""
entity_type: EntityType | None = None
class TPlusBatchOut(BaseModel):
"""批次列表不包含原始数据、源配置或数据库异常。"""
batch_no: str
task_type: TaskType
status: str
entity_types: list[str] | None
source_record_count: int
staged_record_count: int
skipped_record_count: int
failed_record_count: int
parent_batch_no: str | None
create_time: datetime
started_at: datetime | None
finished_at: datetime | None
can_retry: bool
error_summary: str | None = None
class TPlusBatchDetailOut(TPlusBatchOut):
"""详情按需携带受控的关联核验结果。"""
audit_result: dict | None = None
class TPlusErrorOut(BaseModel):
"""异常使用固定说明,源定位仅开放整数 ID。"""
id: int
entity_type: str
source_id: str | None
error_code: str
error_message: str
retry_count: int
create_time: datetime
class TPlusOverviewOut(BaseModel):
"""配置就绪不等同于源库连通或兼容验收通过。"""
is_configured: bool
configuration_message: str
status_counts: dict[str, int]
total_staged_records: int = 0
compatibility_status: str = "PENDING"
business_import_enabled: bool = False
latest_audit: dict | None = None
+21
View File
@@ -0,0 +1,21 @@
# T+ 只读同步
> AI 摘要:读取外部 SQL Server 的原始快照,并管理持久化任务。关键文件为 task_service、worker、sync_service、relationship_audit 和 query_service;管理员 API 为 `/api/v1/system/tplus`。
## 职责与边界
- `task_service` 创建/重试任务,控制请求幂等和事务,保留原批次与异常;HTTP 不执行源库长查询。
- `worker.run_once` 取得 MySQL 执行锁后恢复中断任务、领取一个任务;读取和关联核验共用锁。
- `sync_service` 主键分页读取白名单实体,每页保存原始快照,相同源 ID 和行哈希去重。
- `relationship_audit` 执行 `db/queries/tplus/audit/queries.sql`,只返回系统字典和数量;按收款方向与单据类型核对关联。
- `query_service` 分页元数据,按需加载白名单报告;不返回原始 payload、驱动异常或源配置。
## 主要入口
管理员 API 提供概况、批次/异常分页、详情、创建与失败重试。CLI 为 `python -m app.cli tplus-worker --once/--watch`;原 `tplus-stage --entity personnel` 可继续直接读取。
连接配置、启动命令和异常处理参见项目 `docs/T+本地恢复与同步开发手册.md`。后端与 worker 必须指向同一 MySQL 业务库和 T+ 账套,查询文件随部署交付。
## 验收限制
当前只写 MySQL 暂存与管理表,不写工资业务表、不推进时间水位。关联、成本、日期和负向事件的业务规则待客户确认;SQL Server 2022 的实测不能替代客户 2008 R2(10.50.x)的兼容验收。
+6
View File
@@ -0,0 +1,6 @@
"""T+ 只读读取、字段映射和同步 staging。"""
from app.services.tplus.reader import TPlusReader
from app.services.tplus.sync_service import TPlusSyncService
__all__ = ["TPlusReader", "TPlusSyncService"]
+136
View File
@@ -0,0 +1,136 @@
"""T+ 字段映射元数据。
这里记录候选字段和确认状态,不把字典中的候选直接当成业务契约。
高风险口径在客户确认前保持 ``PENDING``,同步器只允许写入原始 staging。
"""
from dataclasses import dataclass
from enum import StrEnum
class MappingStatus(StrEnum):
"""字段映射的证据状态。"""
CONFIRMED = "CONFIRMED"
CANDIDATE = "CANDIDATE"
MISSING = "MISSING"
PENDING = "PENDING"
@dataclass(frozen=True, slots=True)
class TPlusMapping:
"""一个实体的源表、候选字段和目标表说明。"""
entity_type: str
source_table: str
source_fields: tuple[str, ...]
target_table: str
status: MappingStatus
unresolved: tuple[str, ...] = ()
version: str = "0.2"
schema_evidence: str = "ceshi20261026.bak,2026-10-27 恢复副本结构核验"
TPLUS_MAPPINGS: tuple[TPlusMapping, ...] = (
TPlusMapping(
"personnel",
"AA_Person",
("id", "code", "name", "disabled", "mobilePhoneNo", "officePhoneNo", "iddepartment"),
"personnel",
MappingStatus.CANDIDATE,
("工号是否等于 code", "经理隶属关系", "入职日期", "试用期"),
),
TPlusMapping(
"products",
"AA_Inventory",
("id", "code", "name", "specification", "idunit", "idinventoryclass"),
"products",
MappingStatus.CANDIDATE,
("库存数量语义", "参考价来源", "系统产品分类"),
),
TPlusMapping(
"customers",
"AA_Partner",
("id", "code", "name", "Contact", "representative", "disabled", "updated"),
"customers",
MappingStatus.CANDIDATE,
("客户唯一键", "改名/合并规则", "系统客户分类"),
),
TPlusMapping(
"businesses",
"SA_SaleOrder",
("id", "code", "voucherdate", "idcustomer", "idclerk", "idbusinesstype", "updated"),
"businesses",
MappingStatus.PENDING,
("成本口径", "业务负责人覆盖规则", "含税口径"),
),
TPlusMapping(
"shipments",
"SA_SaleDelivery",
(
"id",
"code",
"voucherdate",
"SendDate",
"deliveryDate",
"auditeddate",
"SourceVoucherCode",
"updated",
),
"shipments",
MappingStatus.PENDING,
("发货日期", "订单关联路径", "退货/红冲规则", "运费来源"),
),
TPlusMapping(
"receipts",
"ARAP_ReceivePayment",
(
"id",
"code",
"voucherDate",
"auditedDate",
"createdTime",
"SaleOrderCode",
"SaleOrderID",
"ts",
),
"receipts",
MappingStatus.PENDING,
("到账日期", "收款方式", "核销状态", "回款与发货关联"),
),
)
DETAIL_MAPPINGS: tuple[TPlusMapping, ...] = (
TPlusMapping(
"business_items",
"SA_SaleOrder_b",
("id", "idSaleOrderDTO", "LatestCost"),
"business_items",
MappingStatus.PENDING,
("成本口径", "含税口径"),
),
TPlusMapping(
"shipment_items",
"SA_SaleDelivery_b",
("ID", "idSaleDeliveryDTO", "saleOrderId"),
"shipment_items",
MappingStatus.PENDING,
("发货归属", "红冲和退货"),
),
TPlusMapping(
"receipt_items",
"ARAP_ReceivePayment_b",
("id", "idArapReceivePaymentDTO", "saleorderid"),
"receipts",
MappingStatus.PENDING,
("回款拆分", "核销语义", "增量水位"),
),
)
def get_mapping(entity_type: str) -> TPlusMapping:
"""按实体类型取得映射;未知实体直接拒绝。"""
for mapping in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS):
if mapping.entity_type == entity_type:
return mapping
raise KeyError(f"未知 T+ 实体类型:{entity_type}")
+122
View File
@@ -0,0 +1,122 @@
"""同步管理只读查询;只公开受控元数据,不公开源数据和原异常。"""
from app.core.exceptions import NotFoundError
from app.core.response import PageResult
from app.core.tplus_db import resolve_tplus_settings
from app.repositories import tplus_repository as repository
from app.schemas.tplus import (
TPlusBatchDetailOut,
TPlusBatchOut,
TPlusErrorOut,
TPlusOverviewOut,
)
from app.services.tplus import task_service
from app.services.tplus.mapping import DETAIL_MAPPINGS, TPLUS_MAPPINGS
from app.services.tplus.relationship_audit import public_report
ERROR_MESSAGES = {
"READ_ERROR": "源数据读取失败,请检查只读连接和字段映射",
"STAGE_ERROR": "原始数据保存失败,请核对字段类型后重试",
"INTERRUPTED": "执行进程中断,可重新读取,已保存数据会自动去重",
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射",
"TYPE_UNRESOLVED": "关联核验所需的单据类型缺失或不唯一",
}
_ENTITY_TYPES = {item.entity_type for item in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS)}
def batch_out(batch, active_parents: set[str]) -> TPlusBatchOut:
"""只使用元数据字段组装输出,历史批次范围保持未知。"""
entities = batch.entity_types_json
scope_valid = (
isinstance(entities, list)
and bool(entities)
and all(isinstance(item, str) and item in _ENTITY_TYPES for item in entities)
)
return TPlusBatchOut(
batch_no=batch.batch_no,
task_type="AUDIT" if batch.sync_mode == "AUDIT" else "READ",
status=batch.status,
entity_types=entities if scope_valid else None,
source_record_count=batch.source_record_count,
staged_record_count=batch.staged_record_count,
skipped_record_count=batch.skipped_record_count,
failed_record_count=batch.failed_record_count,
parent_batch_no=batch.parent_batch_no,
create_time=batch.create_time,
started_at=batch.started_at,
finished_at=batch.finished_at,
can_retry=batch.status in ("FAILED", "PARTIAL")
and (batch.sync_mode == "AUDIT" or (batch.sync_mode == "FULL" and scope_valid))
and batch.batch_no not in active_parents,
error_summary="任务有异常,请查看异常明细"
if batch.status in ("FAILED", "PARTIAL")
else None,
)
def get_batch(session, number: str) -> TPlusBatchDetailOut:
"""按需加载报告,未找到时统一返回 404。"""
batch = repository.get_batch(session, number, with_result=True)
if batch is None:
raise NotFoundError("同步批次不存在")
out = batch_out(batch, repository.active_retry_parents(session, [number]))
return TPlusBatchDetailOut(
**out.model_dump(), audit_result=public_report(batch.audit_result_json)
)
def list_batches(session, query):
"""分页批次并批量查询重试状态。"""
rows, total = repository.list_batches(session, query)
active = repository.active_retry_parents(session, [row.batch_no for row in rows])
return PageResult[TPlusBatchOut](
records=[batch_out(row, active) for row in rows],
total=total,
page_num=query.page_num,
page_size=query.page_size,
)
def _safe_id(value) -> str | None:
if isinstance(value, str) and value.isascii() and value.isdigit() and len(value) <= 10:
return value if int(value) <= 2147483647 else None
return None
def list_errors(session, number, query):
"""错误按编码生成固定说明,避免驱动异常泄漏连接参数。"""
if repository.get_batch(session, number) is None:
raise NotFoundError("同步批次不存在")
rows, total = repository.list_errors(session, number, query)
records = [
TPlusErrorOut(
id=row.id,
entity_type=row.entity_type if row.entity_type in _ENTITY_TYPES else "system",
source_id=_safe_id(row.source_id),
error_code=row.error_code if row.error_code in ERROR_MESSAGES else "UNKNOWN",
error_message=ERROR_MESSAGES.get(row.error_code, "任务未完成,请核对配置后重试"),
retry_count=row.retry_count,
create_time=row.create_time,
)
for row in rows
]
return PageResult[TPlusErrorOut](
records=records, total=total, page_num=query.page_num, page_size=query.page_size
)
def overview(session) -> TPlusOverviewOut:
"""只校验配置,不在页面查询期间建立 T+ 连接。"""
try:
resolve_tplus_settings(task_service.get_settings())
configured, message = True, "只读配置已就绪,实际连接结果以任务执行为准"
except RuntimeError:
configured, message = False, "T+ 只读配置未就绪,请联系管理员配置"
latest = repository.latest_audit(session)
return TPlusOverviewOut(
is_configured=configured,
configuration_message=message,
status_counts=repository.status_counts(session),
total_staged_records=repository.total_staged_records(session),
latest_audit=public_report(latest.audit_result_json) if latest else None,
)
+95
View File
@@ -0,0 +1,95 @@
"""T+ 六类源表的只读读取器。"""
from collections.abc import Iterator
from datetime import datetime
from pathlib import Path
from typing import Any
from sqlalchemy import BigInteger, Integer, bindparam, text
from sqlalchemy.orm import Session
from app.services.tplus.mapping import TPLUS_MAPPINGS, get_mapping
_ENTITY_QUERY_FILES = {
"personnel": "personnel.sql",
"products": "products.sql",
"customers": "customers.sql",
"businesses": "businesses.sql",
"shipments": "shipments.sql",
"receipts": "receipts.sql",
"business_items": "business_items.sql",
"shipment_items": "shipment_items.sql",
"receipt_items": "receipt_items.sql",
}
class TPlusReader:
"""执行白名单 SQL 文件,禁止调用方传入任意表名或 SQL。"""
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
self.session = session
self.query_dir = query_dir or self._default_query_dir()
@staticmethod
def _default_query_dir() -> Path:
return Path(__file__).resolve().parents[4] / "db" / "queries" / "tplus"
def iter_entity(
self,
entity_type: str,
*,
watermark: datetime | None = None,
batch_size: int = 500,
) -> Iterator[dict[str, Any]]:
"""按实体流式读取,水位只通过绑定参数传入。"""
if entity_type not in _ENTITY_QUERY_FILES:
raise KeyError(f"不支持的 T+ 实体类型:{entity_type}")
if batch_size < 1 or batch_size > 5000:
raise ValueError("batch_size 必须在 1 到 5000 之间")
if watermark is not None:
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
get_mapping(entity_type)
sql_path = self.query_dir / _ENTITY_QUERY_FILES[entity_type]
if not sql_path.is_file():
raise FileNotFoundError(f"缺少 T+ 查询文件:{sql_path}")
statement = text(sql_path.read_text(encoding="utf-8")).bindparams(
bindparam("batch_size", type_=Integer()), bindparam("after_id", type_=BigInteger())
)
after_id = -2147483649 # 所有源表主键均已核验为 SQL Server int,包含其最小值。
while True:
result = self.session.execute(
statement, {"batch_size": batch_size, "after_id": after_id}
)
try:
rows = [dict(row) for row in result.mappings().all()]
finally:
result.close()
if not rows:
return
next_id = int(rows[-1]["id"])
if next_id <= after_id:
raise ValueError("T+ 源主键未递增,停止读取以避免重复循环")
yield from rows
after_id = next_id
def fetch_entity(
self,
entity_type: str,
*,
watermark: datetime | None = None,
max_rows: int = 5000,
) -> list[dict[str, Any]]:
"""读取有限数量的实体行,供核验命令和单元测试使用。"""
if max_rows < 1:
raise ValueError("max_rows 必须大于 0")
rows: list[dict[str, Any]] = []
for row in self.iter_entity(entity_type, watermark=watermark):
rows.append(row)
if len(rows) >= max_rows:
break
return rows
def inventory(self) -> tuple[str, ...]:
"""返回当前适配器支持的实体类型。"""
return tuple(mapping.entity_type for mapping in TPLUS_MAPPINGS)
@@ -0,0 +1,135 @@
"""读取实际系统字典并统计带单据类型的收款关联,不推断计提口径。"""
import re
from datetime import datetime
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.logging import scrub
from app.services.tplus.reader import TPlusReader
_COLUMNS = {
"source_version": ("source_version",),
"voucher_types": ("id", "code", "name"),
"state_enums": ("id", "code", "name", "enum_name"),
"receipt_kinds": ("voucher_type_code", "voucher_type_name", "receive_flag", "row_count"),
"receipt_line_kinds": ("voucher_type_id", "voucher_type_name", "receive_flag", "row_count"),
"sales_links": (
"row_count",
"delivery_matches",
"detail_matches",
"direct_order_matches",
"detail_order_matches",
"unambiguous_order_matches",
"invalid_order_ids",
"conflicting_order_ids",
),
"order_coverage": ("row_count", "without_order", "single_order", "multiple_orders"),
}
PENDING_CONFIRMATIONS = (
("cost_basis", "绩效成本取值", "订单最新成本或出库成本的类型、时点和含税口径需业务确认"),
("receipt_date", "到账日", "单据日、审核日和明细 arrivalDate 的业务含义需确认"),
("ship_date", "发货日", "回款天数应使用哪个发货日期需确认"),
("allocation", "多单分摊", "一笔收款涉及多张销货单或订单时,不能自动把整笔金额记给一单"),
("negative_events", "退货与红冲", "负数、冲销及已结算业务的调整方式需确认"),
)
def public_report(raw: dict | None) -> dict | None:
"""历史报告也按当前字段白名单输出,禁止透传未知嵌套数据。"""
if not isinstance(raw, dict):
return None
result = {
key: TPlusRelationshipAudit._safe_value(raw.get(key))
for key in (
"schema_version",
"captured_at",
"source_version",
"is_target_engine",
"relationship_status",
)
}
result["compatibility_status"] = "PENDING"
for section, columns in _COLUMNS.items():
if section in ("source_version", "voucher_types"):
continue
value = raw.get(section)
if isinstance(value, list):
result[section] = [
{key: TPlusRelationshipAudit._safe_value(row.get(key)) for key in columns}
for row in value[:200]
if isinstance(row, dict)
]
elif isinstance(value, dict):
result[section] = {
key: TPlusRelationshipAudit._safe_value(value.get(key)) for key in columns
}
else:
result[section] = None
result["pending_confirmations"] = [
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
]
return result
class TPlusRelationshipAudit:
"""执行固定核验查询;结果只包含系统字典和数量,不包含客户记录。"""
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
self.session = session
root = query_dir or TPlusReader._default_query_dir()
source = (root / "audit" / "queries.sql").read_text(encoding="utf-8")
sections = re.split(r"(?m)^-- @query (\w+)\s*$", source)
self.queries = dict(zip(sections[1::2], sections[2::2], strict=True))
if set(self.queries) != set(_COLUMNS):
raise ValueError("关联核验查询清单不完整")
def fetch(self, name: str, **params) -> list[dict]:
"""只接受登记的查询名与绑定参数,过滤非合同字段。"""
if name not in self.queries:
raise ValueError("不支持的核验查询")
result = self.session.execute(text(self.queries[name]), params)
try:
return [
{key: self._safe_value(row.get(key)) for key in _COLUMNS[name]}
for row in result.mappings().all()
]
finally:
result.close()
@staticmethod
def _safe_value(value):
if value is None or isinstance(value, (int, bool)):
return value
return scrub(re.sub(r"[\x00-\x1f]", " ", str(value)))[:200]
def run(self) -> dict:
"""生成事实报告;类型缺失或有歧义时不执行该关联链路。"""
version = self.fetch("source_version")[0]["source_version"]
types = self.fetch("voucher_types")
report = {
"schema_version": 1,
"captured_at": datetime.now().isoformat(),
"source_version": version,
"is_target_engine": str(version).startswith("10.50."),
"compatibility_status": "PENDING",
"state_enums": self.fetch("state_enums"),
"receipt_kinds": self.fetch("receipt_kinds"),
"receipt_line_kinds": self.fetch("receipt_line_kinds"),
"sales_links": None,
"order_coverage": None,
"relationship_status": "UNRESOLVED",
"relationship_basis": "主表按 isReceiveFlag=1(是否收款)筛选;明细按 SM_VoucherType 的 SA04 类型关联。主表 idvouchertype 是来源类型,不能据此筛本单类型。",
"pending_confirmations": [
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
],
}
sale = [r["id"] for r in types if r["code"] == "SA04"]
if len(sale) == 1:
params = {"sale_delivery_type_id": sale[0], "receive_flag": 1}
report["sales_links"] = self.fetch("sales_links", **params)[0]
report["order_coverage"] = self.fetch("order_coverage", **params)[0]
report["relationship_status"] = "OBSERVED"
return report
+123
View File
@@ -0,0 +1,123 @@
"""T+ 原始行 staging 服务。"""
import hashlib
import json
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import insert, select, tuple_
from sqlalchemy.orm import Session
from app.models import TPlusSyncRecord
def _json_default(value: Any) -> str:
"""将日期、Decimal 等源字段转换为稳定 JSON 文本。"""
if isinstance(value, (date, datetime)):
return value.isoformat()
if isinstance(value, Decimal):
return str(value)
if isinstance(value, (bytes, bytearray, memoryview)):
return "0x" + bytes(value).hex()
raise TypeError("T+ 源行包含无法序列化的数据类型")
def stage_row(
session: Session,
*,
batch_no: str,
entity_type: str,
source_table: str,
row: dict[str, Any],
) -> tuple[TPlusSyncRecord, bool]:
"""幂等写入单条原始行;返回记录和是否新插入。"""
values = prepare_row(batch_no, entity_type, source_table, row)
existing = session.execute(
select(TPlusSyncRecord).where(
TPlusSyncRecord.entity_type == entity_type,
TPlusSyncRecord.source_id == values["source_id"],
TPlusSyncRecord.row_hash == values["row_hash"],
)
).scalar_one_or_none()
if existing is not None:
return existing, False
record = TPlusSyncRecord(**values)
session.add(record)
session.flush()
return record, True
def prepare_row(batch_no: str, entity_type: str, source_table: str, row: dict) -> dict:
"""规范化原始值,生成与 JSON 列序列化一致的哈希。"""
source_id = str(row["id"]) if row.get("id") is not None else ""
if not source_id:
raise ValueError("T+ 源行缺少 id,无法建立幂等键")
source_no = row.get("code")
payload = json.dumps(
row, ensure_ascii=False, sort_keys=True, default=_json_default, allow_nan=False
)
return {
"batch_no": batch_no,
"entity_type": entity_type,
"source_table": source_table,
"source_id": source_id,
"source_no": str(source_no).strip() if source_no is not None else None,
"source_line_no": str(row.get("line_no")) if row.get("line_no") is not None else None,
"source_updated_at": _source_datetime(row),
"row_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
"payload_json": json.loads(payload),
}
def stage_rows(session: Session, batch_no: str, mapping, rows: list[dict]) -> tuple[int, list]:
"""批量去重和插入;数据库拒绝整批时逐条隔离,保留有效行。"""
records, errors = {}, []
for row in rows:
try:
values = prepare_row(batch_no, mapping.entity_type, mapping.source_table, row)
records[(values["source_id"], values["row_hash"])] = values
except (ValueError, TypeError) as exc:
errors.append((row, type(exc).__name__))
if not records:
return 0, errors
existing = set(
session.execute(
select(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).where(
TPlusSyncRecord.entity_type == mapping.entity_type,
tuple_(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).in_(list(records)),
)
).all()
)
pending = [v for k, v in records.items() if k not in existing]
if not pending:
return 0, errors
try:
with session.begin_nested():
session.execute(insert(TPlusSyncRecord), pending)
return len(pending), errors
except Exception:
# 唯一约束竞争或坏行触发回滚后,再逐条处理,不能丢掉同批有效行。
inserted = 0
for row in rows:
try:
with session.begin_nested():
_, is_new = stage_row(
session,
batch_no=batch_no,
entity_type=mapping.entity_type,
source_table=mapping.source_table,
row=row,
)
inserted += int(is_new)
except Exception as exc:
if not any(old is row for old, _ in errors):
errors.append((row, type(exc).__name__))
return inserted, errors
def _source_datetime(row: dict[str, Any]) -> datetime | None:
"""只接收真实时间值;SQL Server rowversion 不能当作日期。"""
value = row.get("source_updated_at") or row.get("updated")
return value if isinstance(value, datetime) else None
+131
View File
@@ -0,0 +1,131 @@
"""T+ 只读快照同步:批量 staging、幂等去重和失败隔离。"""
from collections.abc import Iterable
from datetime import datetime
from itertools import batched
from pathlib import Path
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.tplus_lock import sync_lock
from app.models import TPlusSyncBatch, TPlusSyncError
from app.services.tplus.mapping import TPlusMapping, get_mapping
from app.services.tplus.reader import TPlusReader
from app.services.tplus.staging import stage_rows
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running
class TPlusSyncService:
"""按分页提交原始快照,业务表始终不参与本服务。"""
def __init__(self, source_session: Session, target_session: Session) -> None:
self.source_session = source_session
self.target_session = target_session
configured = get_settings().t_plus_query_dir
self.reader = TPlusReader(source_session, Path(configured) if configured else None)
def run(
self,
*,
entity_types: Iterable[str],
watermark: datetime | None = None,
batch_no: str | None = None,
batch_size: int = 500,
) -> TPlusSyncBatch:
"""执行可重复的全量读取,提交每页进度;字段语义未定时拒绝时间增量。"""
entities = tuple(dict.fromkeys(entity_types))
if not entities or not 1 <= batch_size <= 5000:
raise ValueError("至少指定一个实体,batch_size 必须在 1 到 5000 之间")
if watermark is not None:
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
for entity in entities:
get_mapping(entity)
with sync_lock(self.target_session):
recover_running(self.target_session)
batch = self._start_direct(entities, batch_no)
try:
return self.execute_claimed(batch, batch_size)
except BaseException:
self.target_session.rollback()
fail_batch(self.target_session, batch, "INTERRUPTED")
raise
def _start_direct(self, entities, batch_no):
batch = TPlusSyncBatch(
batch_no=batch_no or new_batch_no(),
sync_mode="FULL",
status="RUNNING",
source_database=get_settings().t_plus_database or None,
started_at=datetime.now(),
source_record_count=0,
staged_record_count=0,
failed_record_count=0,
skipped_record_count=0,
entity_types_json=list(entities),
)
self.target_session.add(batch)
self.target_session.commit()
return batch
def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
"""处理已持锁且标记 RUNNING 的任务;仅由直接入口或 worker 调用。"""
if batch.status != "RUNNING" or not batch.entity_types_json:
raise ValueError("任务尚未领取或读取范围不完整")
mappings = [get_mapping(entity) for entity in batch.entity_types_json]
for mapping in mappings:
try:
self._read_entity(batch, mapping, batch_size)
except Exception as exc:
self.target_session.rollback()
self._record_error(batch, mapping, {}, "READ_ERROR", type(exc).__name__)
self.target_session.commit()
batch.status = "SUCCEEDED"
if batch.failed_record_count:
failed_rows = (
self.target_session.scalar(
select(func.count())
.select_from(TPlusSyncError)
.where(
TPlusSyncError.batch_no == batch.batch_no,
TPlusSyncError.error_code == "STAGE_ERROR",
)
)
or 0
)
batch.status = "PARTIAL" if batch.source_record_count > failed_rows else "FAILED"
batch.error_message = "部分数据未完成,请查询同步错误记录并重跑全量快照"
batch.finished_at = datetime.now()
batch.watermark_to = None
self.target_session.commit()
return batch
def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
"""批量写入一类数据;每页提交后即使进程中断也可幂等重跑。"""
source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
for chunk in batched(source, size):
inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
batch.source_record_count += len(chunk)
batch.staged_record_count += inserted
batch.skipped_record_count += len(chunk) - inserted - len(errors)
for row, reason in errors:
self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
self.target_session.commit()
def _record_error(
self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
) -> None:
"""只记录错误类别和源 ID,避免数据库异常携带个人信息或凭据。"""
self.target_session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type=mapping.entity_type,
source_table=mapping.source_table,
source_id=str(row["id"]) if row.get("id") is not None else None,
source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
error_code=code,
error_message=f"源数据处理失败({reason}),请核对连接和字段",
)
)
batch.failed_record_count += 1
+184
View File
@@ -0,0 +1,184 @@
"""持久化任务创建和重试,不在 HTTP 事务中读取源库。"""
import secrets
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.exceptions import BadRequestError, NotFoundError
from app.core.tplus_db import resolve_tplus_settings
from app.models import TPlusSyncBatch, TPlusSyncError, User
from app.repositories import tplus_repository as repository
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services import audit_service
from app.services.audit_service import AuditRecord, RequestMeta
@dataclass(frozen=True)
class TaskActor:
"""HTTP 操作者与请求元信息,CLI 调用不传。"""
user: User
meta: RequestMeta | None = None
def new_batch_no() -> str:
"""生成最长 24 字符的可追溯批次号。"""
return f"TP{datetime.now():%Y%m%d%H%M%S}{secrets.token_hex(4)}"
def _audit(session, actor, action, data):
if actor is None:
return nullcontext()
return audit_service.audit(
session,
operator=actor.user,
meta=actor.meta,
record=AuditRecord("T+数据同步", action, "T+ 数据任务" + action, data),
)
def _source_database() -> str:
try:
return resolve_tplus_settings(get_settings()).t_plus_database
except RuntimeError as exc:
raise BadRequestError(str(exc)) from None
def _same_intent(batch, mode, entities, parent):
if (
batch.sync_mode != mode
or sorted(batch.entity_types_json or []) != sorted(entities)
or batch.parent_batch_no != parent
):
raise BadRequestError("幂等键已用于不同的任务请求")
return batch
def _insert(session, values):
try:
with session.begin_nested():
batch = TPlusSyncBatch(**values)
session.add(batch)
session.flush()
return batch, True
except IntegrityError:
batch = repository.get_request(session, values["request_key"], for_update=True)
if batch is None:
raise
return _same_intent(
batch, values["sync_mode"], values["entity_types_json"], values["parent_batch_no"]
), False
def _values(payload, database, actor, parent=None):
return {
"batch_no": new_batch_no(),
"request_key": str(payload.request_key),
"status": "QUEUED",
"sync_mode": "AUDIT" if payload.task_type == "AUDIT" else "FULL",
"entity_types_json": sorted(payload.entity_types),
"source_database": database,
"requested_by": actor.user.id if actor else None,
"parent_batch_no": parent,
"started_at": None,
"source_record_count": 0,
"staged_record_count": 0,
"failed_record_count": 0,
"skipped_record_count": 0,
}
def create_task(session: Session, payload: TPlusCreateIn, actor: TaskActor | None = None):
"""原子创建任务;重放已存在的意图不依赖此刻的源连接状态。"""
with _audit(session, actor, "创建读取任务", payload.model_dump(mode="json")):
existing = repository.get_request(session, str(payload.request_key))
mode = "AUDIT" if payload.task_type == "AUDIT" else "FULL"
if existing:
batch = _same_intent(existing, mode, payload.entity_types, None)
else:
batch, _ = _insert(session, _values(payload, _source_database(), actor))
session.commit()
return batch
def retry_task(
session: Session, parent_no: str, payload: TPlusRetryIn, actor: TaskActor | None = None
):
"""锁父批次后创建一个新任务;不覆盖原批次及异常历史。"""
with _audit(
session,
actor,
"重试读取任务",
{"parent_batch_no": parent_no, "request_key": str(payload.request_key)},
):
existing = repository.get_request(session, str(payload.request_key))
if existing:
if existing.parent_batch_no != parent_no:
raise BadRequestError("幂等键已用于不同的重试请求")
session.commit()
batch = existing
else:
parent = repository.get_batch(session, parent_no, for_update=True)
if parent is None:
raise NotFoundError("同步批次不存在")
request = _retry_request(parent, payload)
active = repository.active_retry(session, parent_no)
if active:
if active.request_key != str(payload.request_key):
raise BadRequestError("该批次已有未完成的重试任务,请刷新列表查看")
batch = active
else:
batch, inserted = _insert(
session, _values(request, _source_database(), actor, parent_no)
)
if inserted:
repository.increment_retry_errors(session, parent_no)
session.commit()
return batch
def _retry_request(parent, payload):
if parent.status not in ("FAILED", "PARTIAL"):
raise BadRequestError("只能重试失败或部分失败的批次")
if parent.sync_mode not in ("FULL", "AUDIT"):
raise BadRequestError("该同步方式尚未启用,请重新选择读取任务")
if parent.sync_mode != "AUDIT" and parent.entity_types_json is None:
raise BadRequestError("历史批次未记录范围,请重新选择读取类型")
return TPlusCreateIn(
task_type="AUDIT" if parent.sync_mode == "AUDIT" else "READ",
entity_types=parent.entity_types_json or [],
request_key=payload.request_key,
)
def fail_batch(session: Session, batch: TPlusSyncBatch, code: str) -> None:
"""只保存固定错误摘要;即使原异常含连接密码也不直接落库。"""
messages = {
"INTERRUPTED": "执行进程中断,请重新读取;已暂存数据会自动去重",
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射后重试",
"TYPE_UNRESOLVED": "核验所需单据类型缺失或不唯一,请核对源字典",
}
message = messages.get(code, "任务未完成,请核对运行配置")
batch.status, batch.finished_at, batch.error_message = "FAILED", datetime.now(), message
batch.failed_record_count = (batch.failed_record_count or 0) + 1
session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type="system",
source_table="system",
error_code=code,
error_message=message,
)
)
session.commit()
def recover_running(session: Session) -> None:
"""仅在持有执行锁时调用;锁证明之前的执行者已经退出。"""
for batch in repository.running_batches(session):
fail_batch(session, batch, "INTERRUPTED")
+61
View File
@@ -0,0 +1,61 @@
"""持久化任务执行者;获取连接级锁后才恢复或领取任务。"""
from datetime import datetime
from pathlib import Path
from app.core.tplus_db import get_tplus_session_factory, resolve_tplus_settings
from app.core.tplus_lock import TPlusBusyError, sync_lock
from app.repositories import tplus_repository as repository
from app.services.tplus import task_service
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
from app.services.tplus.sync_service import TPlusSyncService
def run_once(session, source_factory=None):
"""返回本轮处理的批次号;忙锁或队列为空时不假称已执行。"""
try:
with sync_lock(session):
task_service.recover_running(session)
batch = repository.next_queued(session)
if batch is None:
session.rollback()
return None
batch.status, batch.started_at = "RUNNING", datetime.now()
session.commit()
_execute(session, batch, source_factory)
return batch.batch_no
except TPlusBusyError:
session.rollback()
return None
def _execute(session, batch, source_factory):
try:
settings = resolve_tplus_settings(task_service.get_settings())
if batch.source_database and batch.source_database != settings.t_plus_database:
raise RuntimeError("任务排队后的源账套配置已改变")
factory = source_factory or get_tplus_session_factory()
with factory() as source:
if batch.sync_mode == "AUDIT":
root = Path(settings.t_plus_query_dir) if settings.t_plus_query_dir else None
report = TPlusRelationshipAudit(source, root).run()
batch.audit_result_json = report
batch.source_record_count = sum(row["row_count"] for row in report["receipt_kinds"])
batch.finished_at, batch.status = datetime.now(), "SUCCEEDED"
if report["relationship_status"] != "OBSERVED":
task_service.fail_batch(session, batch, "TYPE_UNRESOLVED")
else:
session.commit()
else:
TPlusSyncService(source, session).execute_claimed(batch)
except BaseException as exc:
session.rollback()
task_service.fail_batch(
session,
batch,
"INTERRUPTED"
if isinstance(exc, (KeyboardInterrupt, SystemExit))
else "EXECUTION_ERROR",
)
if isinstance(exc, (KeyboardInterrupt, SystemExit)):
raise