feat(tplus): 新增 T+ 只读同步服务与同步管理

This commit is contained in:
liuwenxin 2026-10-28 11:42:15 +08:00
parent 78bef43c06
commit 4fe5916c2c
43 changed files with 3309 additions and 388 deletions

View File

@ -140,6 +140,19 @@ app/
(`warning.enabled` / `warning.receipt.overdue.days` / `warning.approval.timeout.days`,
V10 种子);⑥ T+ 同步未建,同步相关条目以 `sync_available=false` 显式降级。
### T+ 本地只读同步与管理
T+ 数据库是 SQL Server 只读源库,业务库仍为 MySQL。Docker 恢复 PoC、备份检查和 D 盘目录约定见 `docs/T+本地恢复与同步开发手册.md`。
- 连接配置在 `backend/.env` 的 `T_PLUS_*` 变量中,默认关闭,不配置时不影响主业务启动。
- `uv run python -m app.cli tplus-stage --entity personnel` 按主键分页,将源行写入 `tplus_sync_records` staging,并记录批次和失败信息。水位表已预留,时间增量未启用,`--watermark` 会明确拒绝。
- T+ 驱动通过 `uv sync --extra tplus` 安装到应用环境,ODBC 驱动名称必须显式配置;交付兼容目标为 SQL Server 2008 R2,2022 副本不能代替同版本验收。
- 查询目录 `db/queries/tplus/` 随部署交付,或使用 `T_PLUS_QUERY_DIR` 指定受控位置;支持六类主表和三类明细。
- T+ 字典中的候选字段在实际备份核验和客户确认前,不得直接覆盖岗位、工资标准、归属、业务状态或核算状态。
- 管理 API `/api/v1/system/tplus` 全部限管理员:概况、批次/异常分页、按需报告、创建与失败重试。HTTP 只创建 QUEUED 任务;`uv run python -m app.cli tplus-worker --watch --connection-file <项目外JSON路径>` 独立处理。
- 核心文件:`services/tplus/task_service.py`(幂等/重试)、`worker.py`(执行)、`relationship_audit.py`(字典与关联)、`query_service.py`(安全输出)、`core/tplus_lock.py`(跨提交执行锁)。原 CLI 读取与 worker 共用执行锁。
- V15 / Alembic `e5f607a8b9c0` 增加范围、跳过数、请求键、父批次、请求人和核验结果。未启用时间增量与工资业务导入;完整运行步骤和异常处理见本地恢复手册。
## 常用命令
```bash

View File

@ -26,6 +26,7 @@ from app.api.v1.system import logs as system_logs
from app.api.v1.system import params as system_params
from app.api.v1.system import roles as system_roles
from app.api.v1.system import settings as system_settings
from app.api.v1.system import tplus as system_tplus
from app.api.v1.system import users as system_users
api_router = APIRouter()
@ -47,6 +48,7 @@ system_router.include_router(system_settings.router)
system_router.include_router(system_approvals.router)
system_router.include_router(system_logs.operation_router)
system_router.include_router(system_logs.login_router)
system_router.include_router(system_tplus.router)
v1_router.include_router(system_router)
# 基础数据(需求 2.1):/api/v1/base/*,业务操作登录即可访问

View File

@ -0,0 +1,71 @@
"""管理员管理 T+ 只读任务;长任务交给独立 worker 执行。"""
from typing import Annotated
from fastapi import APIRouter, Path, Query
from app.api.deps import AdminUser, DbSession, RequestMetaDep
from app.core.response import ApiResponse, PageResult, ok
from app.schemas.tplus import (
TPlusBatchDetailOut,
TPlusBatchOut,
TPlusBatchQuery,
TPlusCreateIn,
TPlusErrorOut,
TPlusErrorQuery,
TPlusOverviewOut,
TPlusRetryIn,
)
from app.services.tplus import query_service, task_service
router = APIRouter(prefix="/tplus", tags=["系统管理 · T+ 数据同步"])
BatchNumber = Annotated[str, Path(min_length=1, max_length=32, pattern=r"^[A-Za-z0-9_-]+$")]
@router.get("/overview", response_model=ApiResponse[TPlusOverviewOut])
def overview(session: DbSession, _: AdminUser):
"""查询配置状态、任务统计和最近核验。"""
return ok(query_service.overview(session))
@router.get("/batches", response_model=ApiResponse[PageResult[TPlusBatchOut]])
def batches(session: DbSession, _: AdminUser, query: Annotated[TPlusBatchQuery, Query()]):
"""分页查询批次元数据。"""
return ok(query_service.list_batches(session, query))
@router.get("/batches/{batch_no}", response_model=ApiResponse[TPlusBatchDetailOut])
def detail(session: DbSession, _: AdminUser, batch_no: BatchNumber):
"""查询批次详情与受控的核验报告。"""
return ok(query_service.get_batch(session, batch_no))
@router.get("/batches/{batch_no}/errors", response_model=ApiResponse[PageResult[TPlusErrorOut]])
def errors(
session: DbSession,
_: AdminUser,
batch_no: BatchNumber,
query: Annotated[TPlusErrorQuery, Query()],
):
"""分页查询异常定位和固定错误说明。"""
return ok(query_service.list_errors(session, batch_no, query))
@router.post("/batches", response_model=ApiResponse[TPlusBatchDetailOut])
def create(session: DbSession, admin: AdminUser, meta: RequestMetaDep, payload: TPlusCreateIn):
"""将读取或核验任务持久化排队。"""
batch = task_service.create_task(session, payload, task_service.TaskActor(admin, meta))
return ok(query_service.get_batch(session, batch.batch_no))
@router.post("/batches/{batch_no}/retry", response_model=ApiResponse[TPlusBatchDetailOut])
def retry(
session: DbSession,
admin: AdminUser,
meta: RequestMetaDep,
batch_no: BatchNumber,
payload: TPlusRetryIn,
):
"""新建重试批次,保留原批次与异常历史。"""
batch = task_service.retry_task(session, batch_no, payload, task_service.TaskActor(admin, meta))
return ok(query_service.get_batch(session, batch.batch_no))

View File

@ -11,13 +11,20 @@
import argparse
import getpass
import sys
import time
from datetime import datetime
from sqlalchemy import select
from app.core.config import get_settings
from app.core.constants import PASSWORD_MAX_BYTES, PASSWORD_MIN_BYTES, ROLE_ADMIN
from app.core.db import dispose_engine, get_session_factory
from app.core.security import hash_password
from app.core.tplus_db import dispose_tplus_engine, get_tplus_session_factory
from app.models import Role, User, UserRole
from app.repositories.tplus_repository import get_batch
from app.services.tplus.sync_service import TPlusSyncService
from app.services.tplus.worker import run_once
def _prompt_password() -> str:
@ -80,6 +87,54 @@ def _create_admin(args: argparse.Namespace) -> int:
return 0
def _stage_tplus(args: argparse.Namespace) -> int:
"""读取 T+ 源库并写入 MySQL staging。"""
target_factory = get_session_factory()
source_factory = get_tplus_session_factory()
watermark = datetime.fromisoformat(args.watermark) if args.watermark else None
with source_factory() as source_session, target_factory() as target_session:
batch = TPlusSyncService(source_session, target_session).run(
entity_types=args.entity,
watermark=watermark,
batch_size=args.batch_size,
)
print(
f"T+ staging 完成:批次={batch.batch_no} 状态={batch.status} "
f"源记录={batch.source_record_count} 新增={batch.staged_record_count} "
f"失败={batch.failed_record_count}"
)
return 0 if batch.status == "SUCCEEDED" else 1
def _tplus_worker(args: argparse.Namespace) -> int:
"""独立消费持久化任务;HTTP 进程不承担长时间读取。"""
if args.connection_file:
settings = get_settings()
settings.t_plus_enabled, settings.t_plus_connection_file = True, args.connection_file
if not 2 <= args.poll_seconds <= 60:
print("轮询间隔必须在 2 到 60 秒之间", file=sys.stderr)
return 1
factory = get_session_factory()
try:
while True:
with factory() as session:
batch_no = run_once(session)
batch = get_batch(session, batch_no) if batch_no else None
status = batch.status if batch else None
if batch_no:
print(f"任务完成:{batch_no} 状态={status}", flush=True)
if not args.watch:
return 0 if status in (None, "SUCCEEDED") else 1
if batch_no is None or status != "SUCCEEDED":
time.sleep(args.poll_seconds)
except KeyboardInterrupt:
print("T+ 读取服务已停止")
return 130
except Exception as exc:
print(f"T+ 读取服务停止({type(exc).__name__}),请检查运行环境", file=sys.stderr)
return 1
def main() -> int:
"""命令行入口。"""
parser = argparse.ArgumentParser(
@ -96,12 +151,49 @@ def main() -> int:
help="密码。不传则交互式输入(推荐)——命令行传参会留在 shell 历史中",
)
p_tplus = sub.add_parser("tplus-stage", help="读取 T+ 源库并写入原始 staging")
p_tplus.add_argument(
"--entity",
action="append",
choices=[
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
"business_items",
"shipment_items",
"receipt_items",
],
required=True,
help="实体类型,可重复指定",
)
p_tplus.add_argument(
"--watermark",
default=None,
help="增量起始时间,ISO 8601 格式;不传表示全量读取",
)
p_tplus.add_argument("--batch-size", type=int, default=500, help="每次从源库读取的行数")
p_worker = sub.add_parser("tplus-worker", help="执行已排队的 T+ 读取或关联核验")
modes = p_worker.add_mutually_exclusive_group()
modes.add_argument("--once", action="store_true", help="执行一个任务后退出(默认)")
modes.add_argument("--watch", action="store_true", help="持续消费手动排队任务")
p_worker.add_argument("--connection-file", default=None, help="显式指定项目外只读连接 JSON")
p_worker.add_argument("--poll-seconds", type=int, default=3, help="空闲轮询间隔,2–60 秒")
args = parser.parse_args()
try:
return _create_admin(args)
if args.command == "create-admin":
return _create_admin(args)
if args.command == "tplus-worker":
return _tplus_worker(args)
return _stage_tplus(args)
finally:
dispose_engine()
dispose_tplus_engine()
if __name__ == "__main__":

View File

@ -58,6 +58,23 @@ class Settings(BaseSettings):
redis_password: str = ""
redis_db: int = 0
# ===== T+ 只读 SQL Server =====
# T+ 数据库默认关闭,避免开发者未配置驱动时影响主业务启动。
t_plus_enabled: bool = False
t_plus_host: str = "127.0.0.1"
t_plus_port: int = 1433
t_plus_database: str = ""
t_plus_user: str = ""
t_plus_password: str = Field(default="", repr=False)
t_plus_connection_file: str = ""
# 部署前必须指定已在 SQL Server 2008 R2 实测的驱动,不自动选择最新版。
t_plus_odbc_driver: str = ""
t_plus_encrypt: bool = False
t_plus_trust_server_certificate: bool = True
t_plus_connect_timeout: int = 10
t_plus_query_timeout: int = Field(default=120, ge=1, le=600)
t_plus_query_dir: str = ""
@field_validator("app_secret_key")
@classmethod
def _check_secret_key(cls, v: str) -> str:
@ -92,6 +109,31 @@ class Settings(BaseSettings):
auth = f":{self.redis_password}@" if self.redis_password else ""
return f"redis://{auth}{self.redis_host}:{self.redis_port}/{self.redis_db}"
@property
def t_plus_database_url(self) -> URL:
"""构造 T+ 只读 SQL Server 连接 URL。
连接 URL 只在启用 T+ 适配器时使用;密码由 SQLAlchemy URL 对象保存,
日志中调用 ``render_as_string()`` 时仍会默认隐藏密码。
"""
query = {
"driver": self.t_plus_odbc_driver,
"Encrypt": "yes" if self.t_plus_encrypt else "no",
"TrustServerCertificate": "yes"
if self.t_plus_trust_server_certificate
else "no",
"Connection Timeout": str(self.t_plus_connect_timeout),
}
return URL.create(
drivername="mssql+pyodbc",
username=self.t_plus_user,
password=self.t_plus_password,
host=self.t_plus_host,
port=self.t_plus_port,
database=self.t_plus_database,
query=query,
)
@property
def upload_dir(self) -> Path:
"""上传文件根目录(公司 logo 等):backend/uploads/。

View File

@ -0,0 +1,112 @@
"""T+ 只读 SQL Server 连接。
T+ 是外部源库,业务接口不直接依赖它。连接采用延迟初始化,未配置或未启用
时不会影响主业务服务启动;适配器执行查询时才建立连接池。
"""
import json
from collections.abc import Generator
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
from sqlalchemy import Engine, create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from app.core.config import Settings, get_settings
_engine: Engine | None = None
_session_factory: sessionmaker[Session] | None = None
class _ConnectionFile(BaseModel):
"""连接文件仅允许源库字段,密码不会出现在模型表示中。"""
model_config = ConfigDict(extra="forbid", strict=True)
host: str = Field(min_length=1, max_length=255)
port: int = Field(ge=1, le=65535)
database: str = Field(min_length=1, max_length=128)
user: str = Field(min_length=1, max_length=128)
password: SecretStr
driver: str = Field(min_length=1, max_length=128)
def resolve_tplus_settings(settings: Settings) -> Settings:
"""解析显式配置的只读连接,返回副本,不修改应用全局配置。"""
if not settings.t_plus_enabled:
raise RuntimeError("T+ 数据源未启用,请设置 T_PLUS_ENABLED=true")
resolved = settings
if settings.t_plus_connection_file:
try:
path = Path(settings.t_plus_connection_file)
if path.stat().st_size > 16384:
raise ValueError("连接文件过大")
config = _ConnectionFile.model_validate(
json.loads(path.read_text(encoding="utf-8-sig"))
)
except (OSError, ValueError, ValidationError):
raise RuntimeError("T+ 只读连接文件不可用或字段不正确") from None
values = {
"t_plus_" + key: getattr(config, key) for key in ("host", "port", "database", "user")
}
values.update(
t_plus_password=config.password.get_secret_value(), t_plus_odbc_driver=config.driver
)
resolved = settings.model_copy(update=values)
if resolved.t_plus_user.strip().casefold() == "sa":
raise RuntimeError("应用必须使用 T+ 专用只读账号,不能使用 sa")
if not resolved.t_plus_database or not resolved.t_plus_user or not resolved.t_plus_odbc_driver:
raise RuntimeError("T+ 只读连接缺少数据库、账号或已验证的 ODBC 驱动")
return resolved
def get_tplus_engine() -> Engine:
"""获取 T+ 只读引擎;未启用或缺少 ODBC 驱动时给出可读错误。"""
global _engine, _session_factory
if _engine is not None:
return _engine
settings = resolve_tplus_settings(get_settings())
try:
_engine = create_engine(
settings.t_plus_database_url,
pool_pre_ping=True,
pool_recycle=1800,
pool_timeout=settings.t_plus_connect_timeout,
echo=False,
hide_parameters=True,
)
except ModuleNotFoundError as exc:
raise RuntimeError(
"T+ 连接需要 pyodbc(uv sync --extra tplus)和已验证的 SQL Server ODBC 驱动"
) from exc
_session_factory = sessionmaker(_engine, expire_on_commit=False)
def configure_timeout(connection, _record):
connection.timeout = settings.t_plus_query_timeout
event.listen(_engine, "connect", configure_timeout)
return _engine
def get_tplus_session_factory() -> sessionmaker[Session]:
"""获取 T+ SQLAlchemy 会话工厂。"""
get_tplus_engine()
assert _session_factory is not None
return _session_factory
def get_tplus_db() -> Generator[Session, None, None]:
"""提供只读 T+ 会话;调用方不得 commit 或执行写操作。"""
factory = get_tplus_session_factory()
with factory() as session:
yield session
def dispose_tplus_engine() -> None:
"""释放 T+ 连接池。"""
global _engine, _session_factory
if _engine is not None:
_engine.dispose()
_engine = None
_session_factory = None

View File

@ -0,0 +1,48 @@
"""同账套任务串行锁;独立持有 MySQL 连接,跨分页 commit 仍保持有效。"""
import hashlib
from contextlib import contextmanager
from sqlalchemy import text
from sqlalchemy.orm import Session
class TPlusBusyError(RuntimeError):
"""已有任务持有执行锁。"""
def lock_name(session: Session) -> str:
"""生成不含地址、账号或密码的固定长度锁名。"""
database = session.get_bind().url.database or "default"
return "ys:tplus:read:" + hashlib.sha256(database.encode()).hexdigest()[:24]
@contextmanager
def sync_lock(session: Session):
"""只在取得锁后执行;释放失败时使连接失效,防止带锁回池。"""
engine = session.get_bind()
if engine.dialect.name != "mysql":
raise RuntimeError("T+ 执行协调需要 MySQL 命名锁")
connection = engine.connect()
acquired = False
try:
acquired = (
connection.execute(
text("SELECT GET_LOCK(:name, 0)"), {"name": lock_name(session)}
).scalar_one()
== 1
)
if not acquired:
raise TPlusBusyError("已有 T+ 任务正在执行,请稍后重试")
yield
finally:
if acquired:
try:
released = connection.execute(
text("SELECT RELEASE_LOCK(:name)"), {"name": lock_name(session)}
).scalar_one()
if released != 1:
connection.invalidate()
except Exception:
connection.invalidate()
connection.close()

View File

@ -15,6 +15,7 @@ from app.core.db import dispose_engine
from app.core.exception_handlers import register_exception_handlers
from app.core.logging import setup_logging
from app.core.redis_client import close_redis
from app.core.tplus_db import dispose_tplus_engine
logger = logging.getLogger(__name__)
@ -35,6 +36,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
)
yield
dispose_engine()
dispose_tplus_engine()
close_redis()
logger.info("应用已停止,连接池已释放")

View File

@ -26,6 +26,12 @@ from app.models.role import Permission, Role, RolePermission
from app.models.settlement import AdjustmentOrder, MonthlySettlement, SettlementItem
from app.models.shipment import Shipment, ShipmentItem
from app.models.system_setting import SystemSetting
from app.models.tplus_sync import (
TPlusSyncBatch,
TPlusSyncError,
TPlusSyncRecord,
TPlusSyncWatermark,
)
from app.models.user import User, UserRole
__all__ = [
@ -60,6 +66,10 @@ __all__ = [
"Shipment",
"ShipmentItem",
"SystemSetting",
"TPlusSyncBatch",
"TPlusSyncError",
"TPlusSyncRecord",
"TPlusSyncWatermark",
"TimestampMixin",
"User",
"UserRole",

View File

@ -91,6 +91,9 @@ class TPlusSyncMixin:
String(32), comment="同步批次号(同步模块写入)"
)
synced_at: Mapped[datetime | None] = mapped_column(DateTime, comment="最近同步时间")
source_updated_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="T+ 源数据更新时间(未确认时可为空)"
)
def mysql_table_args(comment: str, *extra: object) -> tuple[object, ...]:

View File

@ -0,0 +1,149 @@
"""T+ 同步批次、错误和原始 staging 记录。"""
from datetime import datetime
from sqlalchemy import JSON, DateTime, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import BIGINT_UNSIGNED, Base, TimestampMixin, mysql_table_args
class TPlusSyncBatch(Base, TimestampMixin):
"""一次 T+ 全量或增量同步的审计记录。"""
__tablename__ = "tplus_sync_batches"
__table_args__ = mysql_table_args(
"T+ 同步批次表(记录读取范围、结果和对账统计)",
UniqueConstraint("batch_no", name="uk_tplus_sync_batches_batch_no"),
UniqueConstraint("request_key", name="uk_tplus_sync_batches_request_key"),
Index("idx_tplus_sync_batches_status", "status"),
Index("idx_tplus_sync_batches_started_at", "started_at"),
Index("idx_tplus_sync_batches_parent_status", "parent_batch_no", "status"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
sync_mode: Mapped[str] = mapped_column(
String(16), nullable=False, comment="任务方式:FULL全量 AUDIT关联核验 INCREMENTAL预留"
)
status: Mapped[str] = mapped_column(
String(16), nullable=False, comment="状态:QUEUED/RUNNING/SUCCEEDED/PARTIAL/FAILED"
)
source_database: Mapped[str | None] = mapped_column(String(128), comment="T+ 源数据库名")
started_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="开始执行时间(排队时为空)"
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime, comment="结束时间")
watermark_from: Mapped[datetime | None] = mapped_column(DateTime, comment="增量起始水位")
watermark_to: Mapped[datetime | None] = mapped_column(DateTime, comment="本批次结束水位")
source_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="源记录数"
)
staged_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="写入 staging 的记录数"
)
failed_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="失败记录数"
)
error_message: Mapped[str | None] = mapped_column(Text, comment="批次错误摘要")
entity_types_json: Mapped[list[str] | None] = mapped_column(
JSON, comment="请求读取的实体类型;历史批次可为空"
)
skipped_record_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="重复快照跳过条数"
)
parent_batch_no: Mapped[str | None] = mapped_column(
String(32), comment="重试来源批次号,保留原批次记录"
)
request_key: Mapped[str | None] = mapped_column(
String(36), comment="请求幂等键,空值用于历史和直接CLI任务"
)
requested_by: Mapped[int | None] = mapped_column(
BIGINT_UNSIGNED, comment="请求用户ID,CLI任务为空,无物理外键"
)
audit_result_json: Mapped[dict | None] = mapped_column(
JSON, comment="关联核验结果,只含系统字典和汇总"
)
class TPlusSyncError(Base, TimestampMixin):
"""单条 T+ 数据读取、清洗或 staging 失败记录。"""
__tablename__ = "tplus_sync_errors"
__table_args__ = mysql_table_args(
"T+ 同步错误表(记录失败行和重试信息)",
Index("idx_tplus_sync_errors_batch_no", "batch_no"),
Index("idx_tplus_sync_errors_entity_type", "entity_type"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
source_id: Mapped[str | None] = mapped_column(String(64), comment="T+ 源主键")
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
error_code: Mapped[str] = mapped_column(String(32), nullable=False, comment="错误编码")
error_message: Mapped[str] = mapped_column(Text, nullable=False, comment="错误说明")
retry_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default="0", comment="已重试次数"
)
class TPlusSyncRecord(Base, TimestampMixin):
"""T+ 原始行 staging,保存映射未定前的可追溯数据。"""
__tablename__ = "tplus_sync_records"
__table_args__ = mysql_table_args(
"T+ 原始同步 staging 表(映射确认前只保存原始行,不直接改业务表)",
UniqueConstraint(
"entity_type", "source_id", "row_hash", name="uk_tplus_sync_records_entity_source_hash"
),
Index("idx_tplus_sync_records_batch_no", "batch_no"),
Index("idx_tplus_sync_records_source_no", "source_no"),
Index("idx_tplus_sync_records_state", "state"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
source_id: Mapped[str] = mapped_column(String(64), nullable=False, comment="T+ 源主键")
source_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 原始编号")
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime, comment="T+ 源更新时间")
row_hash: Mapped[str] = mapped_column(String(64), nullable=False, comment="原始行 SHA-256")
state: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default="PENDING",
comment="状态:PENDING/APPLIED/SKIPPED/ERROR",
)
payload_json: Mapped[dict] = mapped_column(JSON, nullable=False, comment="原始行 JSON")
class TPlusSyncWatermark(Base, TimestampMixin):
"""按实体保存增量同步水位。"""
__tablename__ = "tplus_sync_watermarks"
__table_args__ = mysql_table_args(
"T+ 增量同步水位表",
UniqueConstraint("entity_type", name="uk_tplus_sync_watermarks_entity_type"),
)
id: Mapped[int] = mapped_column(
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
)
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
last_source_updated_at: Mapped[datetime | None] = mapped_column(
DateTime, comment="最近源更新时间"
)
last_source_id: Mapped[str | None] = mapped_column(String(64), comment="同一时间点的最后源主键")
status: Mapped[str] = mapped_column(
String(16), nullable=False, server_default="READY", comment="状态:READY/BLOCKED"
)

View File

@ -0,0 +1,149 @@
"""T+ 管理库访问;只 flush,事务由 service 管理。"""
from sqlalchemy import func, select, update
from sqlalchemy.orm import Session, defer
from app.models import TPlusSyncBatch, TPlusSyncError
def get_batch(
session: Session, batch_no: str, *, for_update: bool = False, with_result: bool = False
):
"""按批次号查询;普通查询不加载核验大字段。"""
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == batch_no)
if not with_result:
statement = statement.options(defer(TPlusSyncBatch.audit_result_json))
if for_update:
statement = statement.with_for_update().execution_options(populate_existing=True)
return session.scalar(statement)
def get_request(session: Session, key: str, *, for_update: bool = False):
"""锁读用于唯一约束竞争后的当前读,避免旧事务快照看不到新任务。"""
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.request_key == key)
if for_update:
statement = statement.with_for_update().execution_options(populate_existing=True)
return session.scalar(statement)
def active_retry(session: Session, parent_no: str):
"""调用方先锁父批次,再用当前读判断是否已有未完成重试。"""
return session.scalar(
select(TPlusSyncBatch)
.where(
TPlusSyncBatch.parent_batch_no == parent_no,
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
)
.with_for_update()
)
def increment_retry_errors(session: Session, parent_no: str) -> None:
"""创建新重试时对原异常累计重试次数。"""
session.execute(
update(TPlusSyncError)
.where(TPlusSyncError.batch_no == parent_no)
.values(retry_count=TPlusSyncError.retry_count + 1)
)
session.flush()
def next_queued(session: Session):
"""调用方持有账套执行锁后领取最早排队任务。"""
return session.scalar(
select(TPlusSyncBatch)
.where(TPlusSyncBatch.status == "QUEUED")
.order_by(TPlusSyncBatch.id)
.limit(1)
.with_for_update()
)
def running_batches(session: Session) -> list[TPlusSyncBatch]:
"""只在持有执行锁时使用,不能依据时间戳判断任务已经停止。"""
return list(
session.scalars(
select(TPlusSyncBatch).where(TPlusSyncBatch.status == "RUNNING").with_for_update()
).all()
)
def list_batches(session: Session, query):
"""分页只读元数据,核验报告和原异常不随列表加载。"""
filters = []
if query.status:
filters.append(TPlusSyncBatch.status == query.status)
if query.task_type:
filters.append(
TPlusSyncBatch.sync_mode == ("AUDIT" if query.task_type == "AUDIT" else "FULL")
)
total = session.scalar(select(func.count()).select_from(TPlusSyncBatch).where(*filters))
rows = session.scalars(
select(TPlusSyncBatch)
.where(*filters)
.options(defer(TPlusSyncBatch.audit_result_json), defer(TPlusSyncBatch.error_message))
.order_by(TPlusSyncBatch.id.desc())
.offset(query.offset)
.limit(query.page_size)
).all()
return rows, total or 0
def active_retry_parents(session: Session, numbers: list[str]) -> set[str]:
"""整页一次获取未完成的重试,避免逐行查询。"""
if not numbers:
return set()
return set(
session.scalars(
select(TPlusSyncBatch.parent_batch_no).where(
TPlusSyncBatch.parent_batch_no.in_(numbers),
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
)
).all()
)
def list_errors(session: Session, number: str, query):
"""只读取当前批次的异常定位元数据。"""
filters = [TPlusSyncError.batch_no == number]
if query.entity_type:
filters.append(TPlusSyncError.entity_type == query.entity_type)
total = session.scalar(select(func.count()).select_from(TPlusSyncError).where(*filters))
rows = session.scalars(
select(TPlusSyncError)
.where(*filters)
.options(defer(TPlusSyncError.error_message))
.order_by(TPlusSyncError.id.desc())
.offset(query.offset)
.limit(query.page_size)
).all()
return rows, total or 0
def status_counts(session: Session) -> dict[str, int]:
"""汇总任务表,不扫描原始快照。"""
return dict(
session.execute(
select(TPlusSyncBatch.status, func.count()).group_by(TPlusSyncBatch.status)
).all()
)
def latest_audit(session: Session):
"""最近完成的核验;返回批次供查询层过滤报告。"""
return session.scalar(
select(TPlusSyncBatch)
.where(
TPlusSyncBatch.sync_mode == "AUDIT",
TPlusSyncBatch.finished_at.is_not(None),
)
.order_by(TPlusSyncBatch.id.desc())
.limit(1)
)
def total_staged_records(session: Session) -> int:
"""从批次元数据汇总累计新增数,避免扫描原始快照。"""
return (
session.scalar(select(func.coalesce(func.sum(TPlusSyncBatch.staged_record_count), 0))) or 0
)

View File

@ -0,0 +1,109 @@
"""T+ 管理接口合同;只暴露范围、数量和受控核验结果。"""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field, model_validator
from app.core.response import PageQuery
EntityType = Literal[
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
"business_items",
"shipment_items",
"receipt_items",
]
TaskType = Literal["READ", "AUDIT"]
TaskStatus = Literal["QUEUED", "RUNNING", "SUCCEEDED", "PARTIAL", "FAILED"]
class TPlusCreateIn(BaseModel):
"""创建长任务;幂等键是调用意图的一部分,必须由调用方保留。"""
task_type: TaskType = "READ"
entity_types: list[EntityType] = Field(default_factory=list, max_length=9)
request_key: UUID
@model_validator(mode="after")
def check_scope(self):
if self.task_type == "READ" and not self.entity_types:
raise ValueError("请选择至少一种读取数据")
if self.task_type == "AUDIT" and self.entity_types:
raise ValueError("关联核验不接受自定义读取范围")
if len(set(self.entity_types)) != len(self.entity_types):
raise ValueError("读取类型不能重复")
return self
class TPlusRetryIn(BaseModel):
"""失败批次重试不接受任意新范围。"""
request_key: UUID
class TPlusBatchQuery(PageQuery):
"""批次筛选,与读取状态和任务类型保持一致。"""
status: TaskStatus | None = None
task_type: TaskType | None = None
class TPlusErrorQuery(PageQuery):
"""异常只能在指定批次内查询。"""
entity_type: EntityType | None = None
class TPlusBatchOut(BaseModel):
"""批次列表不包含原始数据、源配置或数据库异常。"""
batch_no: str
task_type: TaskType
status: str
entity_types: list[str] | None
source_record_count: int
staged_record_count: int
skipped_record_count: int
failed_record_count: int
parent_batch_no: str | None
create_time: datetime
started_at: datetime | None
finished_at: datetime | None
can_retry: bool
error_summary: str | None = None
class TPlusBatchDetailOut(TPlusBatchOut):
"""详情按需携带受控的关联核验结果。"""
audit_result: dict | None = None
class TPlusErrorOut(BaseModel):
"""异常使用固定说明,源定位仅开放整数 ID。"""
id: int
entity_type: str
source_id: str | None
error_code: str
error_message: str
retry_count: int
create_time: datetime
class TPlusOverviewOut(BaseModel):
"""配置就绪不等同于源库连通或兼容验收通过。"""
is_configured: bool
configuration_message: str
status_counts: dict[str, int]
total_staged_records: int = 0
compatibility_status: str = "PENDING"
business_import_enabled: bool = False
latest_audit: dict | None = None

View File

@ -0,0 +1,21 @@
# T+ 只读同步
> AI 摘要:读取外部 SQL Server 的原始快照,并管理持久化任务。关键文件为 task_service、worker、sync_service、relationship_audit 和 query_service;管理员 API 为 `/api/v1/system/tplus`。
## 职责与边界
- `task_service` 创建/重试任务,控制请求幂等和事务,保留原批次与异常;HTTP 不执行源库长查询。
- `worker.run_once` 取得 MySQL 执行锁后恢复中断任务、领取一个任务;读取和关联核验共用锁。
- `sync_service` 主键分页读取白名单实体,每页保存原始快照,相同源 ID 和行哈希去重。
- `relationship_audit` 执行 `db/queries/tplus/audit/queries.sql`,只返回系统字典和数量;按收款方向与单据类型核对关联。
- `query_service` 分页元数据,按需加载白名单报告;不返回原始 payload、驱动异常或源配置。
## 主要入口
管理员 API 提供概况、批次/异常分页、详情、创建与失败重试。CLI 为 `python -m app.cli tplus-worker --once/--watch`;原 `tplus-stage --entity personnel` 可继续直接读取。
连接配置、启动命令和异常处理参见项目 `docs/T+本地恢复与同步开发手册.md`。后端与 worker 必须指向同一 MySQL 业务库和 T+ 账套,查询文件随部署交付。
## 验收限制
当前只写 MySQL 暂存与管理表,不写工资业务表、不推进时间水位。关联、成本、日期和负向事件的业务规则待客户确认;SQL Server 2022 的实测不能替代客户 2008 R2(10.50.x)的兼容验收。

View File

@ -0,0 +1,6 @@
"""T+ 只读读取、字段映射和同步 staging。"""
from app.services.tplus.reader import TPlusReader
from app.services.tplus.sync_service import TPlusSyncService
__all__ = ["TPlusReader", "TPlusSyncService"]

View File

@ -0,0 +1,136 @@
"""T+ 字段映射元数据。
这里记录候选字段和确认状态,不把字典中的候选直接当成业务契约。
高风险口径在客户确认前保持 ``PENDING``,同步器只允许写入原始 staging。
"""
from dataclasses import dataclass
from enum import StrEnum
class MappingStatus(StrEnum):
"""字段映射的证据状态。"""
CONFIRMED = "CONFIRMED"
CANDIDATE = "CANDIDATE"
MISSING = "MISSING"
PENDING = "PENDING"
@dataclass(frozen=True, slots=True)
class TPlusMapping:
"""一个实体的源表、候选字段和目标表说明。"""
entity_type: str
source_table: str
source_fields: tuple[str, ...]
target_table: str
status: MappingStatus
unresolved: tuple[str, ...] = ()
version: str = "0.2"
schema_evidence: str = "ceshi20261026.bak,2026-10-27 恢复副本结构核验"
TPLUS_MAPPINGS: tuple[TPlusMapping, ...] = (
TPlusMapping(
"personnel",
"AA_Person",
("id", "code", "name", "disabled", "mobilePhoneNo", "officePhoneNo", "iddepartment"),
"personnel",
MappingStatus.CANDIDATE,
("工号是否等于 code", "经理隶属关系", "入职日期", "试用期"),
),
TPlusMapping(
"products",
"AA_Inventory",
("id", "code", "name", "specification", "idunit", "idinventoryclass"),
"products",
MappingStatus.CANDIDATE,
("库存数量语义", "参考价来源", "系统产品分类"),
),
TPlusMapping(
"customers",
"AA_Partner",
("id", "code", "name", "Contact", "representative", "disabled", "updated"),
"customers",
MappingStatus.CANDIDATE,
("客户唯一键", "改名/合并规则", "系统客户分类"),
),
TPlusMapping(
"businesses",
"SA_SaleOrder",
("id", "code", "voucherdate", "idcustomer", "idclerk", "idbusinesstype", "updated"),
"businesses",
MappingStatus.PENDING,
("成本口径", "业务负责人覆盖规则", "含税口径"),
),
TPlusMapping(
"shipments",
"SA_SaleDelivery",
(
"id",
"code",
"voucherdate",
"SendDate",
"deliveryDate",
"auditeddate",
"SourceVoucherCode",
"updated",
),
"shipments",
MappingStatus.PENDING,
("发货日期", "订单关联路径", "退货/红冲规则", "运费来源"),
),
TPlusMapping(
"receipts",
"ARAP_ReceivePayment",
(
"id",
"code",
"voucherDate",
"auditedDate",
"createdTime",
"SaleOrderCode",
"SaleOrderID",
"ts",
),
"receipts",
MappingStatus.PENDING,
("到账日期", "收款方式", "核销状态", "回款与发货关联"),
),
)
DETAIL_MAPPINGS: tuple[TPlusMapping, ...] = (
TPlusMapping(
"business_items",
"SA_SaleOrder_b",
("id", "idSaleOrderDTO", "LatestCost"),
"business_items",
MappingStatus.PENDING,
("成本口径", "含税口径"),
),
TPlusMapping(
"shipment_items",
"SA_SaleDelivery_b",
("ID", "idSaleDeliveryDTO", "saleOrderId"),
"shipment_items",
MappingStatus.PENDING,
("发货归属", "红冲和退货"),
),
TPlusMapping(
"receipt_items",
"ARAP_ReceivePayment_b",
("id", "idArapReceivePaymentDTO", "saleorderid"),
"receipts",
MappingStatus.PENDING,
("回款拆分", "核销语义", "增量水位"),
),
)
def get_mapping(entity_type: str) -> TPlusMapping:
"""按实体类型取得映射;未知实体直接拒绝。"""
for mapping in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS):
if mapping.entity_type == entity_type:
return mapping
raise KeyError(f"未知 T+ 实体类型:{entity_type}")

View File

@ -0,0 +1,122 @@
"""同步管理只读查询;只公开受控元数据,不公开源数据和原异常。"""
from app.core.exceptions import NotFoundError
from app.core.response import PageResult
from app.core.tplus_db import resolve_tplus_settings
from app.repositories import tplus_repository as repository
from app.schemas.tplus import (
TPlusBatchDetailOut,
TPlusBatchOut,
TPlusErrorOut,
TPlusOverviewOut,
)
from app.services.tplus import task_service
from app.services.tplus.mapping import DETAIL_MAPPINGS, TPLUS_MAPPINGS
from app.services.tplus.relationship_audit import public_report
ERROR_MESSAGES = {
"READ_ERROR": "源数据读取失败,请检查只读连接和字段映射",
"STAGE_ERROR": "原始数据保存失败,请核对字段类型后重试",
"INTERRUPTED": "执行进程中断,可重新读取,已保存数据会自动去重",
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射",
"TYPE_UNRESOLVED": "关联核验所需的单据类型缺失或不唯一",
}
_ENTITY_TYPES = {item.entity_type for item in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS)}
def batch_out(batch, active_parents: set[str]) -> TPlusBatchOut:
"""只使用元数据字段组装输出,历史批次范围保持未知。"""
entities = batch.entity_types_json
scope_valid = (
isinstance(entities, list)
and bool(entities)
and all(isinstance(item, str) and item in _ENTITY_TYPES for item in entities)
)
return TPlusBatchOut(
batch_no=batch.batch_no,
task_type="AUDIT" if batch.sync_mode == "AUDIT" else "READ",
status=batch.status,
entity_types=entities if scope_valid else None,
source_record_count=batch.source_record_count,
staged_record_count=batch.staged_record_count,
skipped_record_count=batch.skipped_record_count,
failed_record_count=batch.failed_record_count,
parent_batch_no=batch.parent_batch_no,
create_time=batch.create_time,
started_at=batch.started_at,
finished_at=batch.finished_at,
can_retry=batch.status in ("FAILED", "PARTIAL")
and (batch.sync_mode == "AUDIT" or (batch.sync_mode == "FULL" and scope_valid))
and batch.batch_no not in active_parents,
error_summary="任务有异常,请查看异常明细"
if batch.status in ("FAILED", "PARTIAL")
else None,
)
def get_batch(session, number: str) -> TPlusBatchDetailOut:
"""按需加载报告,未找到时统一返回 404。"""
batch = repository.get_batch(session, number, with_result=True)
if batch is None:
raise NotFoundError("同步批次不存在")
out = batch_out(batch, repository.active_retry_parents(session, [number]))
return TPlusBatchDetailOut(
**out.model_dump(), audit_result=public_report(batch.audit_result_json)
)
def list_batches(session, query):
"""分页批次并批量查询重试状态。"""
rows, total = repository.list_batches(session, query)
active = repository.active_retry_parents(session, [row.batch_no for row in rows])
return PageResult[TPlusBatchOut](
records=[batch_out(row, active) for row in rows],
total=total,
page_num=query.page_num,
page_size=query.page_size,
)
def _safe_id(value) -> str | None:
if isinstance(value, str) and value.isascii() and value.isdigit() and len(value) <= 10:
return value if int(value) <= 2147483647 else None
return None
def list_errors(session, number, query):
"""错误按编码生成固定说明,避免驱动异常泄漏连接参数。"""
if repository.get_batch(session, number) is None:
raise NotFoundError("同步批次不存在")
rows, total = repository.list_errors(session, number, query)
records = [
TPlusErrorOut(
id=row.id,
entity_type=row.entity_type if row.entity_type in _ENTITY_TYPES else "system",
source_id=_safe_id(row.source_id),
error_code=row.error_code if row.error_code in ERROR_MESSAGES else "UNKNOWN",
error_message=ERROR_MESSAGES.get(row.error_code, "任务未完成,请核对配置后重试"),
retry_count=row.retry_count,
create_time=row.create_time,
)
for row in rows
]
return PageResult[TPlusErrorOut](
records=records, total=total, page_num=query.page_num, page_size=query.page_size
)
def overview(session) -> TPlusOverviewOut:
"""只校验配置,不在页面查询期间建立 T+ 连接。"""
try:
resolve_tplus_settings(task_service.get_settings())
configured, message = True, "只读配置已就绪,实际连接结果以任务执行为准"
except RuntimeError:
configured, message = False, "T+ 只读配置未就绪,请联系管理员配置"
latest = repository.latest_audit(session)
return TPlusOverviewOut(
is_configured=configured,
configuration_message=message,
status_counts=repository.status_counts(session),
total_staged_records=repository.total_staged_records(session),
latest_audit=public_report(latest.audit_result_json) if latest else None,
)

View File

@ -0,0 +1,95 @@
"""T+ 六类源表的只读读取器。"""
from collections.abc import Iterator
from datetime import datetime
from pathlib import Path
from typing import Any
from sqlalchemy import BigInteger, Integer, bindparam, text
from sqlalchemy.orm import Session
from app.services.tplus.mapping import TPLUS_MAPPINGS, get_mapping
_ENTITY_QUERY_FILES = {
"personnel": "personnel.sql",
"products": "products.sql",
"customers": "customers.sql",
"businesses": "businesses.sql",
"shipments": "shipments.sql",
"receipts": "receipts.sql",
"business_items": "business_items.sql",
"shipment_items": "shipment_items.sql",
"receipt_items": "receipt_items.sql",
}
class TPlusReader:
"""执行白名单 SQL 文件,禁止调用方传入任意表名或 SQL。"""
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
self.session = session
self.query_dir = query_dir or self._default_query_dir()
@staticmethod
def _default_query_dir() -> Path:
return Path(__file__).resolve().parents[4] / "db" / "queries" / "tplus"
def iter_entity(
self,
entity_type: str,
*,
watermark: datetime | None = None,
batch_size: int = 500,
) -> Iterator[dict[str, Any]]:
"""按实体流式读取,水位只通过绑定参数传入。"""
if entity_type not in _ENTITY_QUERY_FILES:
raise KeyError(f"不支持的 T+ 实体类型:{entity_type}")
if batch_size < 1 or batch_size > 5000:
raise ValueError("batch_size 必须在 1 到 5000 之间")
if watermark is not None:
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
get_mapping(entity_type)
sql_path = self.query_dir / _ENTITY_QUERY_FILES[entity_type]
if not sql_path.is_file():
raise FileNotFoundError(f"缺少 T+ 查询文件:{sql_path}")
statement = text(sql_path.read_text(encoding="utf-8")).bindparams(
bindparam("batch_size", type_=Integer()), bindparam("after_id", type_=BigInteger())
)
after_id = -2147483649 # 所有源表主键均已核验为 SQL Server int,包含其最小值。
while True:
result = self.session.execute(
statement, {"batch_size": batch_size, "after_id": after_id}
)
try:
rows = [dict(row) for row in result.mappings().all()]
finally:
result.close()
if not rows:
return
next_id = int(rows[-1]["id"])
if next_id <= after_id:
raise ValueError("T+ 源主键未递增,停止读取以避免重复循环")
yield from rows
after_id = next_id
def fetch_entity(
self,
entity_type: str,
*,
watermark: datetime | None = None,
max_rows: int = 5000,
) -> list[dict[str, Any]]:
"""读取有限数量的实体行,供核验命令和单元测试使用。"""
if max_rows < 1:
raise ValueError("max_rows 必须大于 0")
rows: list[dict[str, Any]] = []
for row in self.iter_entity(entity_type, watermark=watermark):
rows.append(row)
if len(rows) >= max_rows:
break
return rows
def inventory(self) -> tuple[str, ...]:
"""返回当前适配器支持的实体类型。"""
return tuple(mapping.entity_type for mapping in TPLUS_MAPPINGS)

View File

@ -0,0 +1,135 @@
"""读取实际系统字典并统计带单据类型的收款关联,不推断计提口径。"""
import re
from datetime import datetime
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.logging import scrub
from app.services.tplus.reader import TPlusReader
_COLUMNS = {
"source_version": ("source_version",),
"voucher_types": ("id", "code", "name"),
"state_enums": ("id", "code", "name", "enum_name"),
"receipt_kinds": ("voucher_type_code", "voucher_type_name", "receive_flag", "row_count"),
"receipt_line_kinds": ("voucher_type_id", "voucher_type_name", "receive_flag", "row_count"),
"sales_links": (
"row_count",
"delivery_matches",
"detail_matches",
"direct_order_matches",
"detail_order_matches",
"unambiguous_order_matches",
"invalid_order_ids",
"conflicting_order_ids",
),
"order_coverage": ("row_count", "without_order", "single_order", "multiple_orders"),
}
PENDING_CONFIRMATIONS = (
("cost_basis", "绩效成本取值", "订单最新成本或出库成本的类型、时点和含税口径需业务确认"),
("receipt_date", "到账日", "单据日、审核日和明细 arrivalDate 的业务含义需确认"),
("ship_date", "发货日", "回款天数应使用哪个发货日期需确认"),
("allocation", "多单分摊", "一笔收款涉及多张销货单或订单时,不能自动把整笔金额记给一单"),
("negative_events", "退货与红冲", "负数、冲销及已结算业务的调整方式需确认"),
)
def public_report(raw: dict | None) -> dict | None:
"""历史报告也按当前字段白名单输出,禁止透传未知嵌套数据。"""
if not isinstance(raw, dict):
return None
result = {
key: TPlusRelationshipAudit._safe_value(raw.get(key))
for key in (
"schema_version",
"captured_at",
"source_version",
"is_target_engine",
"relationship_status",
)
}
result["compatibility_status"] = "PENDING"
for section, columns in _COLUMNS.items():
if section in ("source_version", "voucher_types"):
continue
value = raw.get(section)
if isinstance(value, list):
result[section] = [
{key: TPlusRelationshipAudit._safe_value(row.get(key)) for key in columns}
for row in value[:200]
if isinstance(row, dict)
]
elif isinstance(value, dict):
result[section] = {
key: TPlusRelationshipAudit._safe_value(value.get(key)) for key in columns
}
else:
result[section] = None
result["pending_confirmations"] = [
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
]
return result
class TPlusRelationshipAudit:
"""执行固定核验查询;结果只包含系统字典和数量,不包含客户记录。"""
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
self.session = session
root = query_dir or TPlusReader._default_query_dir()
source = (root / "audit" / "queries.sql").read_text(encoding="utf-8")
sections = re.split(r"(?m)^-- @query (\w+)\s*$", source)
self.queries = dict(zip(sections[1::2], sections[2::2], strict=True))
if set(self.queries) != set(_COLUMNS):
raise ValueError("关联核验查询清单不完整")
def fetch(self, name: str, **params) -> list[dict]:
"""只接受登记的查询名与绑定参数,过滤非合同字段。"""
if name not in self.queries:
raise ValueError("不支持的核验查询")
result = self.session.execute(text(self.queries[name]), params)
try:
return [
{key: self._safe_value(row.get(key)) for key in _COLUMNS[name]}
for row in result.mappings().all()
]
finally:
result.close()
@staticmethod
def _safe_value(value):
if value is None or isinstance(value, (int, bool)):
return value
return scrub(re.sub(r"[\x00-\x1f]", " ", str(value)))[:200]
def run(self) -> dict:
"""生成事实报告;类型缺失或有歧义时不执行该关联链路。"""
version = self.fetch("source_version")[0]["source_version"]
types = self.fetch("voucher_types")
report = {
"schema_version": 1,
"captured_at": datetime.now().isoformat(),
"source_version": version,
"is_target_engine": str(version).startswith("10.50."),
"compatibility_status": "PENDING",
"state_enums": self.fetch("state_enums"),
"receipt_kinds": self.fetch("receipt_kinds"),
"receipt_line_kinds": self.fetch("receipt_line_kinds"),
"sales_links": None,
"order_coverage": None,
"relationship_status": "UNRESOLVED",
"relationship_basis": "主表按 isReceiveFlag=1(是否收款)筛选;明细按 SM_VoucherType 的 SA04 类型关联。主表 idvouchertype 是来源类型,不能据此筛本单类型。",
"pending_confirmations": [
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
],
}
sale = [r["id"] for r in types if r["code"] == "SA04"]
if len(sale) == 1:
params = {"sale_delivery_type_id": sale[0], "receive_flag": 1}
report["sales_links"] = self.fetch("sales_links", **params)[0]
report["order_coverage"] = self.fetch("order_coverage", **params)[0]
report["relationship_status"] = "OBSERVED"
return report

View File

@ -0,0 +1,123 @@
"""T+ 原始行 staging 服务。"""
import hashlib
import json
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import insert, select, tuple_
from sqlalchemy.orm import Session
from app.models import TPlusSyncRecord
def _json_default(value: Any) -> str:
"""将日期、Decimal 等源字段转换为稳定 JSON 文本。"""
if isinstance(value, (date, datetime)):
return value.isoformat()
if isinstance(value, Decimal):
return str(value)
if isinstance(value, (bytes, bytearray, memoryview)):
return "0x" + bytes(value).hex()
raise TypeError("T+ 源行包含无法序列化的数据类型")
def stage_row(
session: Session,
*,
batch_no: str,
entity_type: str,
source_table: str,
row: dict[str, Any],
) -> tuple[TPlusSyncRecord, bool]:
"""幂等写入单条原始行;返回记录和是否新插入。"""
values = prepare_row(batch_no, entity_type, source_table, row)
existing = session.execute(
select(TPlusSyncRecord).where(
TPlusSyncRecord.entity_type == entity_type,
TPlusSyncRecord.source_id == values["source_id"],
TPlusSyncRecord.row_hash == values["row_hash"],
)
).scalar_one_or_none()
if existing is not None:
return existing, False
record = TPlusSyncRecord(**values)
session.add(record)
session.flush()
return record, True
def prepare_row(batch_no: str, entity_type: str, source_table: str, row: dict) -> dict:
"""规范化原始值,生成与 JSON 列序列化一致的哈希。"""
source_id = str(row["id"]) if row.get("id") is not None else ""
if not source_id:
raise ValueError("T+ 源行缺少 id,无法建立幂等键")
source_no = row.get("code")
payload = json.dumps(
row, ensure_ascii=False, sort_keys=True, default=_json_default, allow_nan=False
)
return {
"batch_no": batch_no,
"entity_type": entity_type,
"source_table": source_table,
"source_id": source_id,
"source_no": str(source_no).strip() if source_no is not None else None,
"source_line_no": str(row.get("line_no")) if row.get("line_no") is not None else None,
"source_updated_at": _source_datetime(row),
"row_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
"payload_json": json.loads(payload),
}
def stage_rows(session: Session, batch_no: str, mapping, rows: list[dict]) -> tuple[int, list]:
"""批量去重和插入;数据库拒绝整批时逐条隔离,保留有效行。"""
records, errors = {}, []
for row in rows:
try:
values = prepare_row(batch_no, mapping.entity_type, mapping.source_table, row)
records[(values["source_id"], values["row_hash"])] = values
except (ValueError, TypeError) as exc:
errors.append((row, type(exc).__name__))
if not records:
return 0, errors
existing = set(
session.execute(
select(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).where(
TPlusSyncRecord.entity_type == mapping.entity_type,
tuple_(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).in_(list(records)),
)
).all()
)
pending = [v for k, v in records.items() if k not in existing]
if not pending:
return 0, errors
try:
with session.begin_nested():
session.execute(insert(TPlusSyncRecord), pending)
return len(pending), errors
except Exception:
# 唯一约束竞争或坏行触发回滚后,再逐条处理,不能丢掉同批有效行。
inserted = 0
for row in rows:
try:
with session.begin_nested():
_, is_new = stage_row(
session,
batch_no=batch_no,
entity_type=mapping.entity_type,
source_table=mapping.source_table,
row=row,
)
inserted += int(is_new)
except Exception as exc:
if not any(old is row for old, _ in errors):
errors.append((row, type(exc).__name__))
return inserted, errors
def _source_datetime(row: dict[str, Any]) -> datetime | None:
"""只接收真实时间值;SQL Server rowversion 不能当作日期。"""
value = row.get("source_updated_at") or row.get("updated")
return value if isinstance(value, datetime) else None

View File

@ -0,0 +1,131 @@
"""T+ 只读快照同步:批量 staging、幂等去重和失败隔离。"""
from collections.abc import Iterable
from datetime import datetime
from itertools import batched
from pathlib import Path
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.tplus_lock import sync_lock
from app.models import TPlusSyncBatch, TPlusSyncError
from app.services.tplus.mapping import TPlusMapping, get_mapping
from app.services.tplus.reader import TPlusReader
from app.services.tplus.staging import stage_rows
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running
class TPlusSyncService:
"""按分页提交原始快照,业务表始终不参与本服务。"""
def __init__(self, source_session: Session, target_session: Session) -> None:
self.source_session = source_session
self.target_session = target_session
configured = get_settings().t_plus_query_dir
self.reader = TPlusReader(source_session, Path(configured) if configured else None)
def run(
self,
*,
entity_types: Iterable[str],
watermark: datetime | None = None,
batch_no: str | None = None,
batch_size: int = 500,
) -> TPlusSyncBatch:
"""执行可重复的全量读取,提交每页进度;字段语义未定时拒绝时间增量。"""
entities = tuple(dict.fromkeys(entity_types))
if not entities or not 1 <= batch_size <= 5000:
raise ValueError("至少指定一个实体,batch_size 必须在 1 到 5000 之间")
if watermark is not None:
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
for entity in entities:
get_mapping(entity)
with sync_lock(self.target_session):
recover_running(self.target_session)
batch = self._start_direct(entities, batch_no)
try:
return self.execute_claimed(batch, batch_size)
except BaseException:
self.target_session.rollback()
fail_batch(self.target_session, batch, "INTERRUPTED")
raise
def _start_direct(self, entities, batch_no):
batch = TPlusSyncBatch(
batch_no=batch_no or new_batch_no(),
sync_mode="FULL",
status="RUNNING",
source_database=get_settings().t_plus_database or None,
started_at=datetime.now(),
source_record_count=0,
staged_record_count=0,
failed_record_count=0,
skipped_record_count=0,
entity_types_json=list(entities),
)
self.target_session.add(batch)
self.target_session.commit()
return batch
def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
"""处理已持锁且标记 RUNNING 的任务;仅由直接入口或 worker 调用。"""
if batch.status != "RUNNING" or not batch.entity_types_json:
raise ValueError("任务尚未领取或读取范围不完整")
mappings = [get_mapping(entity) for entity in batch.entity_types_json]
for mapping in mappings:
try:
self._read_entity(batch, mapping, batch_size)
except Exception as exc:
self.target_session.rollback()
self._record_error(batch, mapping, {}, "READ_ERROR", type(exc).__name__)
self.target_session.commit()
batch.status = "SUCCEEDED"
if batch.failed_record_count:
failed_rows = (
self.target_session.scalar(
select(func.count())
.select_from(TPlusSyncError)
.where(
TPlusSyncError.batch_no == batch.batch_no,
TPlusSyncError.error_code == "STAGE_ERROR",
)
)
or 0
)
batch.status = "PARTIAL" if batch.source_record_count > failed_rows else "FAILED"
batch.error_message = "部分数据未完成,请查询同步错误记录并重跑全量快照"
batch.finished_at = datetime.now()
batch.watermark_to = None
self.target_session.commit()
return batch
def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
"""批量写入一类数据;每页提交后即使进程中断也可幂等重跑。"""
source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
for chunk in batched(source, size):
inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
batch.source_record_count += len(chunk)
batch.staged_record_count += inserted
batch.skipped_record_count += len(chunk) - inserted - len(errors)
for row, reason in errors:
self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
self.target_session.commit()
def _record_error(
self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
) -> None:
"""只记录错误类别和源 ID,避免数据库异常携带个人信息或凭据。"""
self.target_session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type=mapping.entity_type,
source_table=mapping.source_table,
source_id=str(row["id"]) if row.get("id") is not None else None,
source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
error_code=code,
error_message=f"源数据处理失败({reason}),请核对连接和字段",
)
)
batch.failed_record_count += 1

View File

@ -0,0 +1,184 @@
"""持久化任务创建和重试,不在 HTTP 事务中读取源库。"""
import secrets
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.exceptions import BadRequestError, NotFoundError
from app.core.tplus_db import resolve_tplus_settings
from app.models import TPlusSyncBatch, TPlusSyncError, User
from app.repositories import tplus_repository as repository
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services import audit_service
from app.services.audit_service import AuditRecord, RequestMeta
@dataclass(frozen=True)
class TaskActor:
"""HTTP 操作者与请求元信息,CLI 调用不传。"""
user: User
meta: RequestMeta | None = None
def new_batch_no() -> str:
"""生成最长 24 字符的可追溯批次号。"""
return f"TP{datetime.now():%Y%m%d%H%M%S}{secrets.token_hex(4)}"
def _audit(session, actor, action, data):
if actor is None:
return nullcontext()
return audit_service.audit(
session,
operator=actor.user,
meta=actor.meta,
record=AuditRecord("T+数据同步", action, "T+ 数据任务" + action, data),
)
def _source_database() -> str:
try:
return resolve_tplus_settings(get_settings()).t_plus_database
except RuntimeError as exc:
raise BadRequestError(str(exc)) from None
def _same_intent(batch, mode, entities, parent):
if (
batch.sync_mode != mode
or sorted(batch.entity_types_json or []) != sorted(entities)
or batch.parent_batch_no != parent
):
raise BadRequestError("幂等键已用于不同的任务请求")
return batch
def _insert(session, values):
try:
with session.begin_nested():
batch = TPlusSyncBatch(**values)
session.add(batch)
session.flush()
return batch, True
except IntegrityError:
batch = repository.get_request(session, values["request_key"], for_update=True)
if batch is None:
raise
return _same_intent(
batch, values["sync_mode"], values["entity_types_json"], values["parent_batch_no"]
), False
def _values(payload, database, actor, parent=None):
return {
"batch_no": new_batch_no(),
"request_key": str(payload.request_key),
"status": "QUEUED",
"sync_mode": "AUDIT" if payload.task_type == "AUDIT" else "FULL",
"entity_types_json": sorted(payload.entity_types),
"source_database": database,
"requested_by": actor.user.id if actor else None,
"parent_batch_no": parent,
"started_at": None,
"source_record_count": 0,
"staged_record_count": 0,
"failed_record_count": 0,
"skipped_record_count": 0,
}
def create_task(session: Session, payload: TPlusCreateIn, actor: TaskActor | None = None):
"""原子创建任务;重放已存在的意图不依赖此刻的源连接状态。"""
with _audit(session, actor, "创建读取任务", payload.model_dump(mode="json")):
existing = repository.get_request(session, str(payload.request_key))
mode = "AUDIT" if payload.task_type == "AUDIT" else "FULL"
if existing:
batch = _same_intent(existing, mode, payload.entity_types, None)
else:
batch, _ = _insert(session, _values(payload, _source_database(), actor))
session.commit()
return batch
def retry_task(
session: Session, parent_no: str, payload: TPlusRetryIn, actor: TaskActor | None = None
):
"""锁父批次后创建一个新任务;不覆盖原批次及异常历史。"""
with _audit(
session,
actor,
"重试读取任务",
{"parent_batch_no": parent_no, "request_key": str(payload.request_key)},
):
existing = repository.get_request(session, str(payload.request_key))
if existing:
if existing.parent_batch_no != parent_no:
raise BadRequestError("幂等键已用于不同的重试请求")
session.commit()
batch = existing
else:
parent = repository.get_batch(session, parent_no, for_update=True)
if parent is None:
raise NotFoundError("同步批次不存在")
request = _retry_request(parent, payload)
active = repository.active_retry(session, parent_no)
if active:
if active.request_key != str(payload.request_key):
raise BadRequestError("该批次已有未完成的重试任务,请刷新列表查看")
batch = active
else:
batch, inserted = _insert(
session, _values(request, _source_database(), actor, parent_no)
)
if inserted:
repository.increment_retry_errors(session, parent_no)
session.commit()
return batch
def _retry_request(parent, payload):
if parent.status not in ("FAILED", "PARTIAL"):
raise BadRequestError("只能重试失败或部分失败的批次")
if parent.sync_mode not in ("FULL", "AUDIT"):
raise BadRequestError("该同步方式尚未启用,请重新选择读取任务")
if parent.sync_mode != "AUDIT" and parent.entity_types_json is None:
raise BadRequestError("历史批次未记录范围,请重新选择读取类型")
return TPlusCreateIn(
task_type="AUDIT" if parent.sync_mode == "AUDIT" else "READ",
entity_types=parent.entity_types_json or [],
request_key=payload.request_key,
)
def fail_batch(session: Session, batch: TPlusSyncBatch, code: str) -> None:
"""只保存固定错误摘要;即使原异常含连接密码也不直接落库。"""
messages = {
"INTERRUPTED": "执行进程中断,请重新读取;已暂存数据会自动去重",
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射后重试",
"TYPE_UNRESOLVED": "核验所需单据类型缺失或不唯一,请核对源字典",
}
message = messages.get(code, "任务未完成,请核对运行配置")
batch.status, batch.finished_at, batch.error_message = "FAILED", datetime.now(), message
batch.failed_record_count = (batch.failed_record_count or 0) + 1
session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type="system",
source_table="system",
error_code=code,
error_message=message,
)
)
session.commit()
def recover_running(session: Session) -> None:
"""仅在持有执行锁时调用;锁证明之前的执行者已经退出。"""
for batch in repository.running_batches(session):
fail_batch(session, batch, "INTERRUPTED")

View File

@ -0,0 +1,61 @@
"""持久化任务执行者;获取连接级锁后才恢复或领取任务。"""
from datetime import datetime
from pathlib import Path
from app.core.tplus_db import get_tplus_session_factory, resolve_tplus_settings
from app.core.tplus_lock import TPlusBusyError, sync_lock
from app.repositories import tplus_repository as repository
from app.services.tplus import task_service
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
from app.services.tplus.sync_service import TPlusSyncService
def run_once(session, source_factory=None):
"""返回本轮处理的批次号;忙锁或队列为空时不假称已执行。"""
try:
with sync_lock(session):
task_service.recover_running(session)
batch = repository.next_queued(session)
if batch is None:
session.rollback()
return None
batch.status, batch.started_at = "RUNNING", datetime.now()
session.commit()
_execute(session, batch, source_factory)
return batch.batch_no
except TPlusBusyError:
session.rollback()
return None
def _execute(session, batch, source_factory):
try:
settings = resolve_tplus_settings(task_service.get_settings())
if batch.source_database and batch.source_database != settings.t_plus_database:
raise RuntimeError("任务排队后的源账套配置已改变")
factory = source_factory or get_tplus_session_factory()
with factory() as source:
if batch.sync_mode == "AUDIT":
root = Path(settings.t_plus_query_dir) if settings.t_plus_query_dir else None
report = TPlusRelationshipAudit(source, root).run()
batch.audit_result_json = report
batch.source_record_count = sum(row["row_count"] for row in report["receipt_kinds"])
batch.finished_at, batch.status = datetime.now(), "SUCCEEDED"
if report["relationship_status"] != "OBSERVED":
task_service.fail_batch(session, batch, "TYPE_UNRESOLVED")
else:
session.commit()
else:
TPlusSyncService(source, session).execute_claimed(batch)
except BaseException as exc:
session.rollback()
task_service.fail_batch(
session,
batch,
"INTERRUPTED"
if isinstance(exc, (KeyboardInterrupt, SystemExit))
else "EXECUTION_ERROR",
)
if isinstance(exc, (KeyboardInterrupt, SystemExit)):
raise

View File

@ -38,6 +38,11 @@ dependencies = [
"python-multipart>=0.0.32",
]
[project.optional-dependencies]
tplus = [
"pyodbc>=5.2,<6",
]
[dependency-groups]
dev = [
"pytest>=8.3",

View File

@ -1,6 +1,21 @@
# 后端测试
> AI摘要:本项目后端的测试用例清单与运行记录。按「基础设施层随实现即时测、业务模块整模块完成时统一测」分层组织。当前 697 个用例全绿(2026-10-16,M3 增量批次后)。
> AI 摘要:本轮 T+ 管理已完成真实 MySQL、只读源库与浏览器验证。最终单元测试 602 项通过;管理/并发/结构定点 18 项通过。真正的 2008 R2 验收仍未执行。
## T+ 本轮核验(2026-10-27)
- 完整回归在独立 MySQL `payroll_tplus_validation_20261027`、Redis 15 库运行:984 passed、2 failed。两项失败均为配置单测预期默认 Redis 0 库,而进程继承了集成用的 `REDIS_DB=15`;没有修改默认配置或削弱断言。
- 后续将单元测试放在不设置上述集成覆盖值的独立进程中运行:602 passed。最终管理 API、真实并发和结构一致性定点测试:18 passed,包含新增加的配置关闭及创建/重试审计。
- 真实并发:相同请求只生成一个批次;不同键同时重试同父批次只有一个成功,无死锁;执行锁跨 ORM commit 仍有效,退出后可再获取。
- V15 在隔离库 downgrade 到 V14,再 upgrade 到 head 通过;当前 head `e5f607a8b9c0`。模型、Alembic、留存 SQL 和初始化脚本的结构检查通过;此前 V1–V14 留存 SQL 已从空库执行通过。
- 人员重读 847 条,新增 0、重复跳过 847、失败 0;关联报告覆盖 103,318 条销货类收款明细,50,044 条有明确订单关联,六类业务表行数不变。独立证据位于 D 盘 reports/management_verification_2026-10-27.json。
- Chrome 真实接口检查通过:排队、受控失败、异常查看、页面重试、人员去重、关联检查、筛选、普通用户拒绝访问,以及 320/768/1024/1440 宽度。无页面运行错误;临时账号和批次已清理。
- 加载、空态、配置关闭、概况失败另外使用浏览器响应模拟验证,不能与真实源链路混记。证据位于 .ai/archive/ai-aux/preview/tplus/。
- 320 像素主内容可操作,现有全局顶栏仍有文字挤压;未扩大本轮范围修改共享布局。宽表保留横向滚动,截图和限制见同步管理验收报告。
- 前端类型检查、生产构建和 Ruff 通过;构建保留既有的大分块提醒,测试有现存 Starlette TestClient 弃用提醒。
- 源查询保持参数化与旧版语法,时间增量和业务表写入均关闭。合法 Windows/SQL Server 2008 R2 安装介质仍未取得,不能把 2022 的结果标记为客户版本兼容。
单元测试与集成测试建议分独立终端运行:单元测试不设置集成覆盖值;集成测试使用隔离 MySQL 和 Redis。通用集成夹具会清理数据,不能指向开发或客户业务库。Windows 可设置 `PYTEST_DEBUG_TEMPROOT` 到项目 D 盘的 ai-aux/pytest-tmp,避免不同权限令牌共享系统临时目录。
## 一、测试约定

View File

@ -4,6 +4,36 @@ import pytest
from fastapi.testclient import TestClient
@pytest.fixture
def tplus_session(monkeypatch):
"""T+ 管理逻辑的真实内存事务;MySQL 并发锁另做集成验证。"""
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, text
from sqlalchemy.orm import Session
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
engine = create_engine("sqlite:///:memory:")
metadata = MetaData()
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
table = model.__table__.to_metadata(metadata)
table.c.id.type = Integer()
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
metadata.create_all(engine)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_database="unit_tplus",
t_plus_user="reader",
t_plus_odbc_driver="unit",
)
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
with Session(engine, expire_on_commit=False) as session:
yield session
engine.dispose()
@pytest.fixture(scope="session")
def client() -> TestClient:
"""构造应用测试客户端(会话级复用,避免重复构建应用)。

View File

@ -324,3 +324,36 @@ def normal(make_user, auth_headers) -> dict:
user = make_user([ROLE_USER])
user["headers"] = auth_headers(user)
return user
@pytest.fixture
def tplus_tasks(monkeypatch):
"""隔离测试任务,仅清理此夹具创建的批次和异常。"""
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_database="unit_tplus",
t_plus_user="reader",
t_plus_odbc_driver="unit",
)
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
keys = []
def payload():
key = str(uuid.uuid4())
keys.append(key)
return {"task_type": "READ", "entity_types": ["personnel"], "request_key": key}
yield payload
with get_session_factory()() as db:
numbers = list(
db.scalars(select(TPlusSyncBatch.batch_no).where(TPlusSyncBatch.request_key.in_(keys)))
)
if numbers:
db.execute(delete(TPlusSyncError).where(TPlusSyncError.batch_no.in_(numbers)))
db.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.batch_no.in_(numbers)))
db.commit()

View File

@ -487,8 +487,18 @@ class TestManagerAggregate:
assert len([i for i in out["items"] if i["calc_type"] == 2]) == 1
assert len([i for i in out["items"] if i["calc_type"] == 1]) == 1
def test_月末取参_预约变更影响本月聚合(self, client: TestClient, admin: dict) -> None:
def test_月末取参_预约变更影响本月聚合(
self, client: TestClient, admin: dict, monkeypatch: pytest.MonkeyPatch
) -> None:
"""取参锚点=所属月月末:10-25 生效的预约变更应影响 2026-10 月结。"""
class ScenarioDate(date):
@classmethod
def today(cls):
return cls(2026, 10, 20)
# 固定预约场景的申请日,避免现实日期超过 10-25 后误测成“禁止追溯修改”。
monkeypatch.setattr("app.services.param_service.date", ScenarioDate)
resp = client.post(
"/api/v1/system/params/changes",
json={

View File

@ -0,0 +1,90 @@
"""在隔离 MySQL 中验证幂等竞争、父批次串行化和连接级锁。"""
from concurrent.futures import ThreadPoolExecutor
from threading import Barrier
from uuid import uuid4
import pytest
from sqlalchemy import delete, func, select, text
from app.core.db import get_session_factory
from app.core.exceptions import BadRequestError
from app.core.tplus_lock import TPlusBusyError, sync_lock
from app.models import TPlusSyncBatch
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services.tplus import task_service
pytestmark = pytest.mark.integration
def _parallel(action):
barrier = Barrier(2)
def run():
with get_session_factory()() as session:
barrier.wait(timeout=10)
return action(session)
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(run) for _ in range(2)]
return [future.result(timeout=20) for future in futures]
def test_并发相同请求只创建一个批次(tplus_tasks):
payload = TPlusCreateIn(**tplus_tasks())
numbers = _parallel(lambda session: task_service.create_task(session, payload).batch_no)
assert numbers[0] == numbers[1]
with get_session_factory()() as session:
assert (
session.scalar(
select(func.count())
.select_from(TPlusSyncBatch)
.where(TPlusSyncBatch.request_key == str(payload.request_key))
)
== 1
)
def test_并发不同请求重试同父批次只允许一个(tplus_tasks):
with get_session_factory()() as session:
parent = task_service.create_task(session, TPlusCreateIn(**tplus_tasks()))
parent.status = "FAILED"
session.commit()
number = parent.batch_no
def retry(session):
try:
return task_service.retry_task(
session, number, TPlusRetryIn(request_key=uuid4())
).batch_no
except BadRequestError:
session.rollback()
return "REJECTED"
try:
numbers = _parallel(retry)
assert numbers.count("REJECTED") == 1
with get_session_factory()() as session:
assert (
session.scalar(
select(func.count())
.select_from(TPlusSyncBatch)
.where(TPlusSyncBatch.parent_batch_no == number)
)
== 1
)
finally:
with get_session_factory()() as session:
session.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.parent_batch_no == number))
session.commit()
def test_执行锁跨提交保持且释放后可获取():
with get_session_factory()() as owner, get_session_factory()() as other:
with sync_lock(owner):
owner.execute(text("SELECT 1"))
owner.commit()
with pytest.raises(TPlusBusyError), sync_lock(other):
pytest.fail("另一执行者不应取得锁")
with sync_lock(other):
assert other.scalar(text("SELECT 1")) == 1

View File

@ -0,0 +1,129 @@
"""管理员可管理原始读取任务;权限、幂等和异常脱敏由真实接口验证。"""
import pytest
from sqlalchemy import select
from app.core.db import get_session_factory
from app.models import TPlusSyncBatch, TPlusSyncError
pytestmark = pytest.mark.integration
BASE = "/api/v1/system/tplus"
def test_未登录和普通用户不能访问管理(client, admin, normal, tplus_tasks):
assert client.get(BASE + "/overview").status_code == 401
assert client.get(BASE + "/overview", headers=normal["headers"]).status_code == 403
assert (
client.post(BASE + "/batches", json=tplus_tasks(), headers=normal["headers"]).status_code
== 403
)
def test_创建重复请求返回同一批次并可分页查看(client, admin, tplus_tasks):
payload = tplus_tasks()
first = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
assert first.status_code == 200
second = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
assert second.status_code == 200
number = first.json()["data"]["batch_no"]
assert second.json()["data"]["batch_no"] == number
page = client.get(
BASE + "/batches", params={"pageSize": 1, "status": "QUEUED"}, headers=admin["headers"]
)
assert page.status_code == 200 and page.json()["data"]["pageSize"] == 1
detail = client.get(BASE + "/batches/" + number, headers=admin["headers"])
assert detail.status_code == 200
assert detail.json()["data"]["status"] == "QUEUED"
def test_异常返回固定说明不泄漏数据库参数(client, admin, tplus_tasks):
response = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"])
assert response.status_code == 200
number = response.json()["data"]["batch_no"]
with get_session_factory()() as db:
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
batch.status, batch.error_message = "FAILED", "password=private-value phone=13800000000"
db.add(
TPlusSyncError(
batch_no=number,
entity_type="personnel",
source_table="AA_Person",
source_id="12",
error_code="READ_ERROR",
error_message=batch.error_message,
)
)
db.commit()
for path in (f"/batches/{number}", f"/batches/{number}/errors"):
result = client.get(BASE + path, headers=admin["headers"])
assert result.status_code == 200
assert "private-value" not in result.text and "13800000000" not in result.text
def test_不支持的实体和不存在的批次有明确错误(client, admin, tplus_tasks):
payload = tplus_tasks()
payload["entity_types"] = ["users;drop"]
assert client.post(BASE + "/batches", json=payload, headers=admin["headers"]).status_code == 400
assert client.get(BASE + "/batches/TPmissing", headers=admin["headers"]).status_code == 404
def test_配置关闭时可查询历史但不能排队(client, admin, tplus_tasks, monkeypatch):
from app.services.tplus import task_service
settings = task_service.get_settings().model_copy(update={"t_plus_enabled": False})
monkeypatch.setattr(task_service, "get_settings", lambda: settings)
response = client.get(BASE + "/overview", headers=admin["headers"])
assert response.status_code == 200
assert response.json()["data"]["is_configured"] is False
assert response.json()["data"]["business_import_enabled"] is False
assert response.json()["data"]["compatibility_status"] == "PENDING"
assert (
client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).status_code
== 400
)
def test_失败批次重试保留异常并返回新批次(client, admin, tplus_tasks):
from app.models import OperationLog
parent = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).json()[
"data"
]
number = parent["batch_no"]
with get_session_factory()() as db:
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
batch.status, batch.failed_record_count = "FAILED", 1
db.add(
TPlusSyncError(
batch_no=number,
entity_type="personnel",
source_table="AA_Person",
source_id="1",
error_code="READ_ERROR",
error_message="读取失败",
)
)
db.commit()
request = {"request_key": tplus_tasks()["request_key"]}
first = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
replay = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
assert first.status_code == replay.status_code == 200
child = first.json()["data"]
assert child["parent_batch_no"] == number and child["batch_no"] != number
assert replay.json()["data"]["batch_no"] == child["batch_no"]
detail = client.get(BASE + f"/batches/{number}", headers=admin["headers"]).json()["data"]
assert detail["can_retry"] is False
with get_session_factory()() as db:
assert (
db.scalar(select(TPlusSyncError.retry_count).where(TPlusSyncError.batch_no == number))
== 1
)
assert db.scalar(
select(OperationLog.id)
.where(
OperationLog.user_id == admin["id"],
OperationLog.module == "T+数据同步",
OperationLog.action == "重试读取任务",
)
.limit(1)
)

View File

@ -13,7 +13,12 @@ VALID_SECRET = "x" * 32
def _settings(**overrides: object) -> Settings:
"""构造一个不读取 .env 的配置实例。"""
params: dict[str, object] = {"_env_file": None, "app_secret_key": VALID_SECRET}
# _env_file=None 只禁用文件读取;显式值避免隔离测试库的环境变量影响默认值断言。
params: dict[str, object] = {
"_env_file": None,
"app_secret_key": VALID_SECRET,
"mysql_database": "payroll",
}
params.update(overrides)
return Settings(**params) # type: ignore[arg-type]

View File

@ -44,6 +44,10 @@ EXPECTED_TABLES = {
"monthly_settlements",
"settlement_items",
"adjustment_orders",
"tplus_sync_batches",
"tplus_sync_errors",
"tplus_sync_records",
"tplus_sync_watermarks",
}
# 单复数同形的集合名词:本身即复数语义,不受「以 s 结尾」检查约束

View File

@ -0,0 +1,58 @@
"""T+ 配置和读取适配层的纯单元测试。"""
from pathlib import Path
import pytest
from app.core.config import Settings
from app.services.tplus.mapping import MappingStatus, get_mapping
from app.services.tplus.reader import TPlusReader
def _settings(**overrides: object) -> Settings:
params: dict[str, object] = {"_env_file": None, "app_secret_key": "x" * 32}
params.update(overrides)
return Settings(**params) # type: ignore[arg-type]
def test_T加数据库URL使用mssql_pyodbc并隐藏密码() -> None:
settings = _settings(
t_plus_enabled=True,
t_plus_database="UFTData472115_000001",
t_plus_user="reader",
t_plus_password="p@ss:w/ord",
)
url = settings.t_plus_database_url
assert url.drivername == "mssql+pyodbc"
assert url.database == "UFTData472115_000001"
assert "p@ss:w/ord" not in url.render_as_string()
def test_T加未启用时读取器不建立连接() -> None:
settings = _settings(t_plus_enabled=False)
assert settings.t_plus_enabled is False
def test_T加查询文件覆盖六类实体() -> None:
reader = TPlusReader.__new__(TPlusReader)
reader.query_dir = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
assert reader.inventory() == (
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
)
for entity in reader.inventory():
assert (reader.query_dir / f"{entity}.sql").is_file()
def test_T加高风险映射保持待确认() -> None:
assert get_mapping("businesses").status is MappingStatus.PENDING
assert "成本口径" in get_mapping("businesses").unresolved
def test_未知T加实体被拒绝() -> None:
with pytest.raises(KeyError):
get_mapping("unknown")

View File

@ -0,0 +1,72 @@
"""显式连接文件不应污染应用配置,也不能启用未授权的数据源。"""
import json
import pytest
from app.core.config import Settings
from app.core.tplus_db import resolve_tplus_settings
def test_未开启连接时不读取配置文件(tmp_path):
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_connection_file=str(tmp_path / "missing.json"),
)
with pytest.raises(RuntimeError, match="未启用"):
resolve_tplus_settings(settings)
def test_文件连接只覆盖源字段且原Settings不变(tmp_path):
path = tmp_path / "reader.json"
path.write_text(
json.dumps(
{
"host": "127.0.0.1",
"port": 1433,
"database": "UFTData",
"user": "reader",
"password": "test-only-value",
"driver": "SQL Server",
}
),
encoding="utf-8",
)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_connection_file=str(path),
)
resolved = resolve_tplus_settings(settings)
assert resolved.t_plus_database == "UFTData"
assert resolved.mysql_database == settings.mysql_database
assert settings.t_plus_password == ""
assert "test-only-value" not in str(resolved.t_plus_database_url)
def test_连接文件不接受sa或应用配置覆盖(tmp_path):
path = tmp_path / "bad.json"
path.write_text(
json.dumps(
{
"host": "127.0.0.1",
"port": 1433,
"database": "UFTData",
"user": "sa",
"password": "test-only-value",
"driver": "SQL Server",
"app_secret_key": "bad",
}
),
encoding="utf-8",
)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_connection_file=str(path),
)
with pytest.raises(RuntimeError):
resolve_tplus_settings(settings)

View File

@ -0,0 +1,43 @@
"""对外元数据必须过滤历史原异常和未知报告字段。"""
from uuid import uuid4
from app.models import TPlusSyncError
from app.schemas.tplus import TPlusCreateIn, TPlusErrorQuery
from app.services.tplus import query_service, task_service
from app.services.tplus.relationship_audit import public_report
def test_历史报告只开放白名单并保持兼容待验():
result = public_report(
{
"source_version": "16.0",
"compatibility_status": "PASSED",
"password": "sensitive",
"sales_links": {"row_count": 1, "payload": "secret"},
"state_enums": [{"name": "password=hidden", "extra": "secret"}],
}
)
assert result["compatibility_status"] == "PENDING"
assert "password" not in result and "payload" not in result["sales_links"]
assert "hidden" not in str(result) and "secret" not in str(result)
def test_异常不返回手机号或原始错误文本(tplus_session):
batch = task_service.create_task(
tplus_session, TPlusCreateIn(entity_types=["personnel"], request_key=uuid4())
)
tplus_session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type="personnel",
source_table="AA_Person",
source_id="13800000000",
error_code="READ_ERROR",
error_message="password=private",
)
)
tplus_session.commit()
page = query_service.list_errors(tplus_session, batch.batch_no, TPlusErrorQuery())
assert page.records[0].source_id is None
assert "private" not in page.model_dump_json()

View File

@ -0,0 +1,97 @@
"""关联核验必须使用真实类型字典,不能依靠恰好相同的跨表 ID。"""
from unittest.mock import Mock
import pytest
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
def _audit(types=None):
source = Mock()
rows = {
"source_version": [{"source_version": "16.0.4165.4"}],
"voucher_types": types
if types is not None
else [
{"id": 9104, "code": "SA04", "name": "销货单"},
{"id": 9092, "code": "ARAP01", "name": "收款单"},
],
"state_enums": [{"id": 189, "code": "01", "name": "生效", "enum_name": "VoucherState"}],
"receipt_kinds": [
{
"voucher_type_code": "ARAP01",
"voucher_type_name": "收款单",
"receive_flag": 1,
"row_count": 10,
}
],
"receipt_line_kinds": [
{"voucher_type_id": 9104, "voucher_type_name": "销货单", "row_count": 12}
],
"sales_links": [
{
"row_count": 12,
"delivery_matches": 12,
"detail_matches": 7,
"direct_order_matches": 4,
"invalid_order_ids": 0,
}
],
"order_coverage": [
{"row_count": 12, "without_order": 2, "single_order": 8, "multiple_orders": 2}
],
}
audit = TPlusRelationshipAudit(source)
audit.fetch = Mock(side_effect=lambda name, **kw: rows[name])
return audit
def test_类型标识从字典读取而不是硬编码104():
audit = _audit()
result = audit.run()
args = next(call.kwargs for call in audit.fetch.call_args_list if call.args[0] == "sales_links")
assert args == {"sale_delivery_type_id": 9104, "receive_flag": 1}
assert result["sales_links"]["delivery_matches"] == 12
def test_枚举名称按源库保留且2022不通过2008验收():
result = _audit().run()
assert result["state_enums"][0]["name"] == "生效"
assert result["is_target_engine"] is False
assert result["compatibility_status"] == "PENDING"
def test_同类型字典有歧义就停止该链路统计不猜ID():
audit = _audit(
[
{"id": 1, "code": "SA04", "name": "A"},
{"id": 2, "code": "SA04", "name": "B"},
{"id": 3, "code": "ARAP01", "name": "收款单"},
]
)
result = audit.run()
assert result["relationship_status"] == "UNRESOLVED"
assert result["sales_links"] is None
assert not any(call.args[0] == "sales_links" for call in audit.fetch.call_args_list)
def test_多订单和成本日期仍需业务确认不自动选择():
result = _audit().run()
assert result["order_coverage"]["multiple_orders"] == 2
keys = {item["key"] for item in result["pending_confirmations"]}
assert {"cost_basis", "receipt_date", "ship_date", "allocation"} <= keys
def test_SQL查询有固定白名单且不接受自定义表名():
audit = TPlusRelationshipAudit(Mock())
with pytest.raises(ValueError, match="核验查询"):
audit.fetch("SELECT * FROM customers")
def test_主表来源单据类型为空不妨碍按是否收款核验():
# 字典明确 idvouchertype 是“来源单据类型”,不是当前收款单的类型。
audit = _audit([{"id": 9104, "code": "SA04", "name": "销货单"}])
assert audit.run()["relationship_status"] == "OBSERVED"
assert "p.isReceiveFlag = :receive_flag" in audit.queries["sales_links"]
assert "p.idvouchertype =" not in audit.queries["sales_links"]

View File

@ -0,0 +1,40 @@
"""限制 T+ 源查询语法;真实 2008 R2 的驱动/引擎实测仍是最终验收条件。"""
import re
from pathlib import Path
import pytest
QUERY_ROOT = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
@pytest.mark.parametrize("path", sorted(QUERY_ROOT.glob("*.sql")), ids=lambda p: p.stem)
def test_源查询只使用旧版本分页并保持只读(path):
sql = re.sub(r"--[^\n]*", "", path.read_text(encoding="utf-8"))
assert not re.search(
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
sql,
re.I,
)
assert "TOP (:batch_size)" in sql
assert ":after_id" in sql
assert not re.search(r"SELECT\s+\*", sql, re.I)
def test_收付款主表没有更新时间时不虚构水位():
sql = (QUERY_ROOT / "receipts.sql").read_text(encoding="utf-8")
assert "CAST(NULL AS datetime) AS source_updated_at" in sql
assert not re.search(r"h\.updated\b", sql, re.I)
def test_关联核验保持只读且不使用新版函数():
sql = re.sub(r"--[^\n]*", "", (QUERY_ROOT / "audit/queries.sql").read_text(encoding="utf-8"))
assert not re.search(
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
sql,
re.I,
)
assert not re.search(r"SELECT\s+\*", sql, re.I)
assert ":sale_delivery_type_id" in sql and ":receive_flag" in sql

View File

@ -0,0 +1,126 @@
"""T+ staging 的幂等和原始行保护测试。"""
import json
from datetime import datetime
from decimal import Decimal
import pytest
from app.models import TPlusSyncRecord
from app.services.tplus.staging import stage_row
class _Result:
def __init__(self, value: TPlusSyncRecord | None) -> None:
self.value = value
def scalar_one_or_none(self) -> TPlusSyncRecord | None:
return self.value
class _Session:
def __init__(self, existing: TPlusSyncRecord | None = None) -> None:
self.existing = existing
self.added: list[object] = []
self.flushed = False
def execute(self, _statement: object) -> _Result:
return _Result(self.existing)
def add(self, value: object) -> None:
self.added.append(value)
def flush(self) -> None:
self.flushed = True
def _row() -> dict[str, object]:
return {
"id": 1001,
"code": "SO-1001",
"updated": datetime(2026, 10, 27, 10, 30),
"amount": 12.5,
}
def test_stage_row_首次写入保留源编号更新时间和哈希() -> None:
session = _Session()
record, inserted = stage_row(
session,
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
row=_row(),
)
assert inserted is True
assert isinstance(session.added[0], TPlusSyncRecord)
assert record.source_id == "1001"
assert record.source_no == "SO-1001"
assert record.source_updated_at == datetime(2026, 10, 27, 10, 30)
assert len(record.row_hash) == 64
assert session.flushed is True
def test_stage_row_同一行重复执行返回已有记录() -> None:
existing = TPlusSyncRecord(
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
source_id="1001",
row_hash="x" * 64,
payload_json={},
)
session = _Session(existing)
record, inserted = stage_row(
session,
batch_no="TP20261027103000EFGH",
entity_type="businesses",
source_table="SA_SaleOrder",
row=_row(),
)
assert inserted is False
assert record is existing
assert session.added == []
def test_stage_row_缺少源主键时拒绝写入() -> None:
with pytest.raises(ValueError, match="缺少 id"):
stage_row(
_Session(),
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
row={"code": "SO-1001"},
)
def test_stage_row_日期金额和二进制行版本能够实际序列化() -> None:
row = {"id": 0, "code": "A", "updated": b"\x00\x01", "amount": Decimal("12.50")}
record, _ = stage_row(
_Session(), batch_no="TP1", entity_type="products", source_table="AA_Inventory", row=row
)
assert record.source_id == "0"
assert record.source_updated_at is None
assert record.payload_json["updated"] == "0x0001"
assert record.payload_json["amount"] == "12.50"
assert json.loads(json.dumps(record.payload_json)) == record.payload_json
def test_stage_row_键顺序不影响原始行哈希() -> None:
first, _ = stage_row(
_Session(),
batch_no="TP1",
entity_type="products",
source_table="AA_Inventory",
row={"id": 1, "code": "A"},
)
second, _ = stage_row(
_Session(),
batch_no="TP2",
entity_type="products",
source_table="AA_Inventory",
row={"code": "A", "id": 1},
)
assert first.row_hash == second.row_hash

View File

@ -0,0 +1,110 @@
"""用真实事务验证 T+ staging 重跑、失败隔离和敏感信息保护。"""
from contextlib import nullcontext
from unittest.mock import Mock
import pytest
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, func, select, text
from sqlalchemy.orm import Session
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
from app.services.tplus.sync_service import TPlusSyncService
@pytest.fixture
def target(monkeypatch):
"""以 SQLite 执行 staging 事务,MySQL 专属 DDL 仍由迁移测试覆盖。"""
engine = create_engine("sqlite:///:memory:")
metadata = MetaData()
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
table = model.__table__.to_metadata(metadata)
table.c.id.type = Integer()
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
metadata.create_all(engine)
settings = Settings(_env_file=None, app_secret_key="x" * 32)
monkeypatch.setattr("app.services.tplus.sync_service.get_settings", lambda: settings)
monkeypatch.setattr("app.services.tplus.sync_service.sync_lock", lambda session: nullcontext())
with Session(engine, expire_on_commit=False) as session:
yield session
engine.dispose()
def _service(target, rows):
service = TPlusSyncService(Mock(), target)
service.reader = Mock()
service.reader.iter_entity.side_effect = lambda *a, **kw: iter(rows)
return service
def test_重复全量读取不增加记录且业务表未被引用(target):
rows = [{"id": 1, "code": "A"}, {"id": 2, "code": "B"}]
service = _service(target, rows)
first = service.run(entity_types=["products"])
second = service.run(entity_types=["products"])
assert first.status == second.status == "SUCCEEDED"
assert first.staged_record_count == 2
assert second.staged_record_count == 0
assert second.source_record_count == 2
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
assert second.watermark_to is None
def test_单条缺失主键不会回滚其他有效行(target):
service = _service(target, [{"id": 1, "code": "A"}, {"code": "bad"}, {"id": 2}])
batch = service.run(entity_types=["products"])
assert batch.status == "PARTIAL"
assert batch.staged_record_count == 2
assert batch.failed_record_count == 1
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
def test_数据库约束失败后仍可写入下一条(target):
service = _service(target, [{"id": 1, "code": "A"}])
# 用数据库真实触发器拒绝坏行,验证 executemany 失败后的回滚和逐条隔离。
service.reader.iter_entity.side_effect = lambda *a, **kw: iter([{"id": 1}, {"id": 2}])
target.execute(
text(
"CREATE TRIGGER reject_first BEFORE INSERT ON tplus_sync_records "
"WHEN NEW.source_id='1' BEGIN SELECT RAISE(ABORT, 'invalid row'); END"
)
)
batch = service.run(entity_types=["products"])
assert batch.status == "PARTIAL"
assert batch.staged_record_count == 1
assert target.scalar(select(TPlusSyncRecord.source_id)) == "2"
def test_读取失败有审计且异常中的敏感参数不落日志(target):
service = _service(target, [])
service.reader.iter_entity.side_effect = RuntimeError("password=secret phone=13800000000")
batch = service.run(entity_types=["products"])
assert batch.status == "FAILED"
error = target.scalar(select(TPlusSyncError))
assert error.error_code == "READ_ERROR"
assert "secret" not in error.error_message
assert "13800000000" not in error.error_message
def test_成功一类后另一类读取失败应标记部分成功(target):
service = _service(target, [])
def source(entity, **kwargs):
if entity == "products":
return iter([{"id": 1}])
raise RuntimeError("读取中断")
service.reader.iter_entity.side_effect = source
batch = service.run(entity_types=["products", "customers"])
assert batch.status == "PARTIAL"
assert batch.source_record_count == batch.staged_record_count == 1
assert batch.failed_record_count == 1
def test_未确认时间增量不能创建误导性批次(target):
from datetime import datetime
service = _service(target, [])
with pytest.raises(ValueError, match="增量水位语义尚未确认"):
service.run(entity_types=["receipts"], watermark=datetime(2026, 10, 27))
assert target.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 0

View File

@ -0,0 +1,21 @@
"""持久化任务需要完整的幂等和重试证据,不能只有运行中的内存对象。"""
from app.models import TPlusSyncBatch
def test_批次记录范围重试幂等与报告():
table = TPlusSyncBatch.__table__
assert {
"entity_types_json",
"skipped_record_count",
"parent_batch_no",
"request_key",
"requested_by",
"audit_result_json",
} <= set(table.c.keys())
assert table.c.started_at.nullable is True
def test_请求键由数据库唯一约束保护():
constraints = TPlusSyncBatch.__table__.constraints
assert any(c.name == "uk_tplus_sync_batches_request_key" for c in constraints)

View File

@ -0,0 +1,93 @@
"""同一意图不可重复排队,重试必须保留原始批次和范围。"""
from uuid import uuid4
import pytest
from sqlalchemy import func, select
from app.core.exceptions import BadRequestError
from app.models import TPlusSyncBatch, TPlusSyncError
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services.tplus import task_service
def _create(**kwargs):
return TPlusCreateIn(request_key=uuid4(), entity_types=["products"], **kwargs)
def test_同键同意图复用同一批次(tplus_session):
request = _create()
first = task_service.create_task(tplus_session, request)
second = task_service.create_task(tplus_session, request)
assert first.id == second.id
assert first.status == "QUEUED"
assert first.started_at is None
assert first.entity_types_json == ["products"]
assert tplus_session.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 1
def test_同键不同范围拒绝(tplus_session):
request = _create()
task_service.create_task(tplus_session, request)
changed = TPlusCreateIn(request_key=request.request_key, entity_types=["personnel"])
with pytest.raises(BadRequestError, match="幂等"):
task_service.create_task(tplus_session, changed)
def test_失败重试创建新批次并保留原错误(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "FAILED"
tplus_session.add(
TPlusSyncError(
batch_no=parent.batch_no,
entity_type="products",
source_table="AA_Inventory",
error_code="READ_ERROR",
error_message="读取失败",
)
)
tplus_session.commit()
retry = TPlusRetryIn(request_key=uuid4())
child = task_service.retry_task(tplus_session, parent.batch_no, retry)
replay = task_service.retry_task(tplus_session, parent.batch_no, retry)
assert child.id == replay.id and child.id != parent.id
assert child.parent_batch_no == parent.batch_no
assert parent.status == "FAILED"
assert tplus_session.scalar(select(TPlusSyncError.retry_count)) == 1
def test_已有未完成重试时不重复排队(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "FAILED"
tplus_session.commit()
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
with pytest.raises(BadRequestError, match="重试任务"):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
def test_成功或未知历史范围不能自动重试(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "SUCCEEDED"
tplus_session.commit()
with pytest.raises(BadRequestError):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
parent.status, parent.entity_types_json = "FAILED", None
tplus_session.commit()
with pytest.raises(BadRequestError, match="范围"):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
def test_关联核验不需要读取实体范围(tplus_session):
batch = task_service.create_task(
tplus_session, TPlusCreateIn(task_type="AUDIT", request_key=uuid4())
)
assert batch.sync_mode == "AUDIT"
assert batch.entity_types_json == []
def test_预留增量批次不得静默改为全量重试(tplus_session):
batch = task_service.create_task(tplus_session, _create())
batch.status, batch.sync_mode = "FAILED", "INCREMENTAL"
tplus_session.commit()
with pytest.raises(BadRequestError, match="方式"):
task_service.retry_task(tplus_session, batch.batch_no, TPlusRetryIn(request_key=uuid4()))

View File

@ -0,0 +1,100 @@
"""排队任务要有实际执行者;忙锁、断连和进程中断不能被误记成成功。"""
from contextlib import contextmanager, nullcontext
from unittest.mock import Mock
from uuid import uuid4
from sqlalchemy import select
from app.core.tplus_lock import TPlusBusyError
from app.models import TPlusSyncError
from app.schemas.tplus import TPlusCreateIn
from app.services.tplus import task_service, worker
def _queued(session, audit=False):
return task_service.create_task(
session,
TPlusCreateIn(
request_key=uuid4(),
task_type="AUDIT" if audit else "READ",
entity_types=[] if audit else ["products"],
),
)
def _worker_setup(monkeypatch):
monkeypatch.setattr(worker, "sync_lock", lambda session: nullcontext())
monkeypatch.setattr(
"app.services.tplus.reader.TPlusReader.iter_entity",
lambda *a, **kw: iter([{"id": 1, "code": "A"}]),
)
return lambda: nullcontext(Mock())
def test_读取任务由worker处理并可观察重复数(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
first = _queued(tplus_session)
assert worker.run_once(tplus_session, source_factory=source) == first.batch_no
assert first.status == "SUCCEEDED" and first.started_at is not None
second = _queued(tplus_session)
worker.run_once(tplus_session, source_factory=source)
assert second.staged_record_count == 0
assert second.skipped_record_count == 1
def test_忙锁时不领取任务也不恢复其他运行记录(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session)
@contextmanager
def busy(_session):
raise TPlusBusyError("已有任务")
yield
monkeypatch.setattr(worker, "sync_lock", busy)
assert worker.run_once(tplus_session, source_factory=source) is None
assert batch.status == "QUEUED"
def test_源连接失败写入固定错误不泄漏凭据(tplus_session, monkeypatch):
_worker_setup(monkeypatch)
batch = _queued(tplus_session)
def unavailable():
raise RuntimeError("password=private-secret")
worker.run_once(tplus_session, source_factory=unavailable)
assert batch.status == "FAILED"
error = tplus_session.scalar(select(TPlusSyncError))
assert "private-secret" not in error.error_message
assert error.error_code == "EXECUTION_ERROR"
def test_取得锁后把遗留运行中任务记为中断失败(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session)
batch.status = "RUNNING"
tplus_session.commit()
assert worker.run_once(tplus_session, source_factory=source) is None
assert batch.status == "FAILED"
assert tplus_session.scalar(select(TPlusSyncError.error_code)) == "INTERRUPTED"
def test_核验任务持久化报告但不修改业务数据(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session, audit=True)
monkeypatch.setattr(
worker.TPlusRelationshipAudit,
"run",
lambda self: {
"relationship_status": "OBSERVED",
"receipt_kinds": [{"row_count": 10}],
"compatibility_status": "PENDING",
"is_target_engine": False,
},
)
worker.run_once(tplus_session, source_factory=source)
assert batch.status == "SUCCEEDED"
assert batch.audit_result_json["compatibility_status"] == "PENDING"
assert batch.staged_record_count == 0

792
backend/uv.lock generated

File diff suppressed because it is too large Load Diff