feat(tplus): 新增 T+ 只读同步服务与同步管理
This commit is contained in:
parent
78bef43c06
commit
4fe5916c2c
@ -140,6 +140,19 @@ app/
|
||||
(`warning.enabled` / `warning.receipt.overdue.days` / `warning.approval.timeout.days`,
|
||||
V10 种子);⑥ T+ 同步未建,同步相关条目以 `sync_available=false` 显式降级。
|
||||
|
||||
### T+ 本地只读同步与管理
|
||||
|
||||
T+ 数据库是 SQL Server 只读源库,业务库仍为 MySQL。Docker 恢复 PoC、备份检查和 D 盘目录约定见 `docs/T+本地恢复与同步开发手册.md`。
|
||||
|
||||
- 连接配置在 `backend/.env` 的 `T_PLUS_*` 变量中,默认关闭,不配置时不影响主业务启动。
|
||||
- `uv run python -m app.cli tplus-stage --entity personnel` 按主键分页,将源行写入 `tplus_sync_records` staging,并记录批次和失败信息。水位表已预留,时间增量未启用,`--watermark` 会明确拒绝。
|
||||
- T+ 驱动通过 `uv sync --extra tplus` 安装到应用环境,ODBC 驱动名称必须显式配置;交付兼容目标为 SQL Server 2008 R2,2022 副本不能代替同版本验收。
|
||||
- 查询目录 `db/queries/tplus/` 随部署交付,或使用 `T_PLUS_QUERY_DIR` 指定受控位置;支持六类主表和三类明细。
|
||||
- T+ 字典中的候选字段在实际备份核验和客户确认前,不得直接覆盖岗位、工资标准、归属、业务状态或核算状态。
|
||||
- 管理 API `/api/v1/system/tplus` 全部限管理员:概况、批次/异常分页、按需报告、创建与失败重试。HTTP 只创建 QUEUED 任务;`uv run python -m app.cli tplus-worker --watch --connection-file <项目外JSON路径>` 独立处理。
|
||||
- 核心文件:`services/tplus/task_service.py`(幂等/重试)、`worker.py`(执行)、`relationship_audit.py`(字典与关联)、`query_service.py`(安全输出)、`core/tplus_lock.py`(跨提交执行锁)。原 CLI 读取与 worker 共用执行锁。
|
||||
- V15 / Alembic `e5f607a8b9c0` 增加范围、跳过数、请求键、父批次、请求人和核验结果。未启用时间增量与工资业务导入;完整运行步骤和异常处理见本地恢复手册。
|
||||
|
||||
## 常用命令
|
||||
|
||||
```bash
|
||||
|
||||
@ -26,6 +26,7 @@ from app.api.v1.system import logs as system_logs
|
||||
from app.api.v1.system import params as system_params
|
||||
from app.api.v1.system import roles as system_roles
|
||||
from app.api.v1.system import settings as system_settings
|
||||
from app.api.v1.system import tplus as system_tplus
|
||||
from app.api.v1.system import users as system_users
|
||||
|
||||
api_router = APIRouter()
|
||||
@ -47,6 +48,7 @@ system_router.include_router(system_settings.router)
|
||||
system_router.include_router(system_approvals.router)
|
||||
system_router.include_router(system_logs.operation_router)
|
||||
system_router.include_router(system_logs.login_router)
|
||||
system_router.include_router(system_tplus.router)
|
||||
v1_router.include_router(system_router)
|
||||
|
||||
# 基础数据(需求 2.1):/api/v1/base/*,业务操作登录即可访问
|
||||
|
||||
71
backend/app/api/v1/system/tplus.py
Normal file
71
backend/app/api/v1/system/tplus.py
Normal file
@ -0,0 +1,71 @@
|
||||
"""管理员管理 T+ 只读任务;长任务交给独立 worker 执行。"""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Path, Query
|
||||
|
||||
from app.api.deps import AdminUser, DbSession, RequestMetaDep
|
||||
from app.core.response import ApiResponse, PageResult, ok
|
||||
from app.schemas.tplus import (
|
||||
TPlusBatchDetailOut,
|
||||
TPlusBatchOut,
|
||||
TPlusBatchQuery,
|
||||
TPlusCreateIn,
|
||||
TPlusErrorOut,
|
||||
TPlusErrorQuery,
|
||||
TPlusOverviewOut,
|
||||
TPlusRetryIn,
|
||||
)
|
||||
from app.services.tplus import query_service, task_service
|
||||
|
||||
router = APIRouter(prefix="/tplus", tags=["系统管理 · T+ 数据同步"])
|
||||
BatchNumber = Annotated[str, Path(min_length=1, max_length=32, pattern=r"^[A-Za-z0-9_-]+$")]
|
||||
|
||||
|
||||
@router.get("/overview", response_model=ApiResponse[TPlusOverviewOut])
|
||||
def overview(session: DbSession, _: AdminUser):
|
||||
"""查询配置状态、任务统计和最近核验。"""
|
||||
return ok(query_service.overview(session))
|
||||
|
||||
|
||||
@router.get("/batches", response_model=ApiResponse[PageResult[TPlusBatchOut]])
|
||||
def batches(session: DbSession, _: AdminUser, query: Annotated[TPlusBatchQuery, Query()]):
|
||||
"""分页查询批次元数据。"""
|
||||
return ok(query_service.list_batches(session, query))
|
||||
|
||||
|
||||
@router.get("/batches/{batch_no}", response_model=ApiResponse[TPlusBatchDetailOut])
|
||||
def detail(session: DbSession, _: AdminUser, batch_no: BatchNumber):
|
||||
"""查询批次详情与受控的核验报告。"""
|
||||
return ok(query_service.get_batch(session, batch_no))
|
||||
|
||||
|
||||
@router.get("/batches/{batch_no}/errors", response_model=ApiResponse[PageResult[TPlusErrorOut]])
|
||||
def errors(
|
||||
session: DbSession,
|
||||
_: AdminUser,
|
||||
batch_no: BatchNumber,
|
||||
query: Annotated[TPlusErrorQuery, Query()],
|
||||
):
|
||||
"""分页查询异常定位和固定错误说明。"""
|
||||
return ok(query_service.list_errors(session, batch_no, query))
|
||||
|
||||
|
||||
@router.post("/batches", response_model=ApiResponse[TPlusBatchDetailOut])
|
||||
def create(session: DbSession, admin: AdminUser, meta: RequestMetaDep, payload: TPlusCreateIn):
|
||||
"""将读取或核验任务持久化排队。"""
|
||||
batch = task_service.create_task(session, payload, task_service.TaskActor(admin, meta))
|
||||
return ok(query_service.get_batch(session, batch.batch_no))
|
||||
|
||||
|
||||
@router.post("/batches/{batch_no}/retry", response_model=ApiResponse[TPlusBatchDetailOut])
|
||||
def retry(
|
||||
session: DbSession,
|
||||
admin: AdminUser,
|
||||
meta: RequestMetaDep,
|
||||
batch_no: BatchNumber,
|
||||
payload: TPlusRetryIn,
|
||||
):
|
||||
"""新建重试批次,保留原批次与异常历史。"""
|
||||
batch = task_service.retry_task(session, batch_no, payload, task_service.TaskActor(admin, meta))
|
||||
return ok(query_service.get_batch(session, batch.batch_no))
|
||||
@ -11,13 +11,20 @@
|
||||
import argparse
|
||||
import getpass
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.constants import PASSWORD_MAX_BYTES, PASSWORD_MIN_BYTES, ROLE_ADMIN
|
||||
from app.core.db import dispose_engine, get_session_factory
|
||||
from app.core.security import hash_password
|
||||
from app.core.tplus_db import dispose_tplus_engine, get_tplus_session_factory
|
||||
from app.models import Role, User, UserRole
|
||||
from app.repositories.tplus_repository import get_batch
|
||||
from app.services.tplus.sync_service import TPlusSyncService
|
||||
from app.services.tplus.worker import run_once
|
||||
|
||||
|
||||
def _prompt_password() -> str:
|
||||
@ -80,6 +87,54 @@ def _create_admin(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _stage_tplus(args: argparse.Namespace) -> int:
|
||||
"""读取 T+ 源库并写入 MySQL staging。"""
|
||||
target_factory = get_session_factory()
|
||||
source_factory = get_tplus_session_factory()
|
||||
watermark = datetime.fromisoformat(args.watermark) if args.watermark else None
|
||||
with source_factory() as source_session, target_factory() as target_session:
|
||||
batch = TPlusSyncService(source_session, target_session).run(
|
||||
entity_types=args.entity,
|
||||
watermark=watermark,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
print(
|
||||
f"T+ staging 完成:批次={batch.batch_no} 状态={batch.status} "
|
||||
f"源记录={batch.source_record_count} 新增={batch.staged_record_count} "
|
||||
f"失败={batch.failed_record_count}"
|
||||
)
|
||||
return 0 if batch.status == "SUCCEEDED" else 1
|
||||
|
||||
|
||||
def _tplus_worker(args: argparse.Namespace) -> int:
|
||||
"""独立消费持久化任务;HTTP 进程不承担长时间读取。"""
|
||||
if args.connection_file:
|
||||
settings = get_settings()
|
||||
settings.t_plus_enabled, settings.t_plus_connection_file = True, args.connection_file
|
||||
if not 2 <= args.poll_seconds <= 60:
|
||||
print("轮询间隔必须在 2 到 60 秒之间", file=sys.stderr)
|
||||
return 1
|
||||
factory = get_session_factory()
|
||||
try:
|
||||
while True:
|
||||
with factory() as session:
|
||||
batch_no = run_once(session)
|
||||
batch = get_batch(session, batch_no) if batch_no else None
|
||||
status = batch.status if batch else None
|
||||
if batch_no:
|
||||
print(f"任务完成:{batch_no} 状态={status}", flush=True)
|
||||
if not args.watch:
|
||||
return 0 if status in (None, "SUCCEEDED") else 1
|
||||
if batch_no is None or status != "SUCCEEDED":
|
||||
time.sleep(args.poll_seconds)
|
||||
except KeyboardInterrupt:
|
||||
print("T+ 读取服务已停止")
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"T+ 读取服务停止({type(exc).__name__}),请检查运行环境", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""命令行入口。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
@ -96,12 +151,49 @@ def main() -> int:
|
||||
help="密码。不传则交互式输入(推荐)——命令行传参会留在 shell 历史中",
|
||||
)
|
||||
|
||||
p_tplus = sub.add_parser("tplus-stage", help="读取 T+ 源库并写入原始 staging")
|
||||
p_tplus.add_argument(
|
||||
"--entity",
|
||||
action="append",
|
||||
choices=[
|
||||
"personnel",
|
||||
"products",
|
||||
"customers",
|
||||
"businesses",
|
||||
"shipments",
|
||||
"receipts",
|
||||
"business_items",
|
||||
"shipment_items",
|
||||
"receipt_items",
|
||||
],
|
||||
required=True,
|
||||
help="实体类型,可重复指定",
|
||||
)
|
||||
p_tplus.add_argument(
|
||||
"--watermark",
|
||||
default=None,
|
||||
help="增量起始时间,ISO 8601 格式;不传表示全量读取",
|
||||
)
|
||||
p_tplus.add_argument("--batch-size", type=int, default=500, help="每次从源库读取的行数")
|
||||
|
||||
p_worker = sub.add_parser("tplus-worker", help="执行已排队的 T+ 读取或关联核验")
|
||||
modes = p_worker.add_mutually_exclusive_group()
|
||||
modes.add_argument("--once", action="store_true", help="执行一个任务后退出(默认)")
|
||||
modes.add_argument("--watch", action="store_true", help="持续消费手动排队任务")
|
||||
p_worker.add_argument("--connection-file", default=None, help="显式指定项目外只读连接 JSON")
|
||||
p_worker.add_argument("--poll-seconds", type=int, default=3, help="空闲轮询间隔,2–60 秒")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
return _create_admin(args)
|
||||
if args.command == "create-admin":
|
||||
return _create_admin(args)
|
||||
if args.command == "tplus-worker":
|
||||
return _tplus_worker(args)
|
||||
return _stage_tplus(args)
|
||||
finally:
|
||||
dispose_engine()
|
||||
dispose_tplus_engine()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -58,6 +58,23 @@ class Settings(BaseSettings):
|
||||
redis_password: str = ""
|
||||
redis_db: int = 0
|
||||
|
||||
# ===== T+ 只读 SQL Server =====
|
||||
# T+ 数据库默认关闭,避免开发者未配置驱动时影响主业务启动。
|
||||
t_plus_enabled: bool = False
|
||||
t_plus_host: str = "127.0.0.1"
|
||||
t_plus_port: int = 1433
|
||||
t_plus_database: str = ""
|
||||
t_plus_user: str = ""
|
||||
t_plus_password: str = Field(default="", repr=False)
|
||||
t_plus_connection_file: str = ""
|
||||
# 部署前必须指定已在 SQL Server 2008 R2 实测的驱动,不自动选择最新版。
|
||||
t_plus_odbc_driver: str = ""
|
||||
t_plus_encrypt: bool = False
|
||||
t_plus_trust_server_certificate: bool = True
|
||||
t_plus_connect_timeout: int = 10
|
||||
t_plus_query_timeout: int = Field(default=120, ge=1, le=600)
|
||||
t_plus_query_dir: str = ""
|
||||
|
||||
@field_validator("app_secret_key")
|
||||
@classmethod
|
||||
def _check_secret_key(cls, v: str) -> str:
|
||||
@ -92,6 +109,31 @@ class Settings(BaseSettings):
|
||||
auth = f":{self.redis_password}@" if self.redis_password else ""
|
||||
return f"redis://{auth}{self.redis_host}:{self.redis_port}/{self.redis_db}"
|
||||
|
||||
@property
|
||||
def t_plus_database_url(self) -> URL:
|
||||
"""构造 T+ 只读 SQL Server 连接 URL。
|
||||
|
||||
连接 URL 只在启用 T+ 适配器时使用;密码由 SQLAlchemy URL 对象保存,
|
||||
日志中调用 ``render_as_string()`` 时仍会默认隐藏密码。
|
||||
"""
|
||||
query = {
|
||||
"driver": self.t_plus_odbc_driver,
|
||||
"Encrypt": "yes" if self.t_plus_encrypt else "no",
|
||||
"TrustServerCertificate": "yes"
|
||||
if self.t_plus_trust_server_certificate
|
||||
else "no",
|
||||
"Connection Timeout": str(self.t_plus_connect_timeout),
|
||||
}
|
||||
return URL.create(
|
||||
drivername="mssql+pyodbc",
|
||||
username=self.t_plus_user,
|
||||
password=self.t_plus_password,
|
||||
host=self.t_plus_host,
|
||||
port=self.t_plus_port,
|
||||
database=self.t_plus_database,
|
||||
query=query,
|
||||
)
|
||||
|
||||
@property
|
||||
def upload_dir(self) -> Path:
|
||||
"""上传文件根目录(公司 logo 等):backend/uploads/。
|
||||
|
||||
112
backend/app/core/tplus_db.py
Normal file
112
backend/app/core/tplus_db.py
Normal file
@ -0,0 +1,112 @@
|
||||
"""T+ 只读 SQL Server 连接。
|
||||
|
||||
T+ 是外部源库,业务接口不直接依赖它。连接采用延迟初始化,未配置或未启用
|
||||
时不会影响主业务服务启动;适配器执行查询时才建立连接池。
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
|
||||
from sqlalchemy import Engine, create_engine, event
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
|
||||
_engine: Engine | None = None
|
||||
_session_factory: sessionmaker[Session] | None = None
|
||||
|
||||
|
||||
class _ConnectionFile(BaseModel):
|
||||
"""连接文件仅允许源库字段,密码不会出现在模型表示中。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
host: str = Field(min_length=1, max_length=255)
|
||||
port: int = Field(ge=1, le=65535)
|
||||
database: str = Field(min_length=1, max_length=128)
|
||||
user: str = Field(min_length=1, max_length=128)
|
||||
password: SecretStr
|
||||
driver: str = Field(min_length=1, max_length=128)
|
||||
|
||||
|
||||
def resolve_tplus_settings(settings: Settings) -> Settings:
|
||||
"""解析显式配置的只读连接,返回副本,不修改应用全局配置。"""
|
||||
if not settings.t_plus_enabled:
|
||||
raise RuntimeError("T+ 数据源未启用,请设置 T_PLUS_ENABLED=true")
|
||||
resolved = settings
|
||||
if settings.t_plus_connection_file:
|
||||
try:
|
||||
path = Path(settings.t_plus_connection_file)
|
||||
if path.stat().st_size > 16384:
|
||||
raise ValueError("连接文件过大")
|
||||
config = _ConnectionFile.model_validate(
|
||||
json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
)
|
||||
except (OSError, ValueError, ValidationError):
|
||||
raise RuntimeError("T+ 只读连接文件不可用或字段不正确") from None
|
||||
values = {
|
||||
"t_plus_" + key: getattr(config, key) for key in ("host", "port", "database", "user")
|
||||
}
|
||||
values.update(
|
||||
t_plus_password=config.password.get_secret_value(), t_plus_odbc_driver=config.driver
|
||||
)
|
||||
resolved = settings.model_copy(update=values)
|
||||
if resolved.t_plus_user.strip().casefold() == "sa":
|
||||
raise RuntimeError("应用必须使用 T+ 专用只读账号,不能使用 sa")
|
||||
if not resolved.t_plus_database or not resolved.t_plus_user or not resolved.t_plus_odbc_driver:
|
||||
raise RuntimeError("T+ 只读连接缺少数据库、账号或已验证的 ODBC 驱动")
|
||||
return resolved
|
||||
|
||||
|
||||
def get_tplus_engine() -> Engine:
|
||||
"""获取 T+ 只读引擎;未启用或缺少 ODBC 驱动时给出可读错误。"""
|
||||
global _engine, _session_factory
|
||||
if _engine is not None:
|
||||
return _engine
|
||||
|
||||
settings = resolve_tplus_settings(get_settings())
|
||||
|
||||
try:
|
||||
_engine = create_engine(
|
||||
settings.t_plus_database_url,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
pool_timeout=settings.t_plus_connect_timeout,
|
||||
echo=False,
|
||||
hide_parameters=True,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
"T+ 连接需要 pyodbc(uv sync --extra tplus)和已验证的 SQL Server ODBC 驱动"
|
||||
) from exc
|
||||
_session_factory = sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
def configure_timeout(connection, _record):
|
||||
connection.timeout = settings.t_plus_query_timeout
|
||||
|
||||
event.listen(_engine, "connect", configure_timeout)
|
||||
return _engine
|
||||
|
||||
|
||||
def get_tplus_session_factory() -> sessionmaker[Session]:
|
||||
"""获取 T+ SQLAlchemy 会话工厂。"""
|
||||
get_tplus_engine()
|
||||
assert _session_factory is not None
|
||||
return _session_factory
|
||||
|
||||
|
||||
def get_tplus_db() -> Generator[Session, None, None]:
|
||||
"""提供只读 T+ 会话;调用方不得 commit 或执行写操作。"""
|
||||
factory = get_tplus_session_factory()
|
||||
with factory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
def dispose_tplus_engine() -> None:
|
||||
"""释放 T+ 连接池。"""
|
||||
global _engine, _session_factory
|
||||
if _engine is not None:
|
||||
_engine.dispose()
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
48
backend/app/core/tplus_lock.py
Normal file
48
backend/app/core/tplus_lock.py
Normal file
@ -0,0 +1,48 @@
|
||||
"""同账套任务串行锁;独立持有 MySQL 连接,跨分页 commit 仍保持有效。"""
|
||||
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
class TPlusBusyError(RuntimeError):
|
||||
"""已有任务持有执行锁。"""
|
||||
|
||||
|
||||
def lock_name(session: Session) -> str:
|
||||
"""生成不含地址、账号或密码的固定长度锁名。"""
|
||||
database = session.get_bind().url.database or "default"
|
||||
return "ys:tplus:read:" + hashlib.sha256(database.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def sync_lock(session: Session):
|
||||
"""只在取得锁后执行;释放失败时使连接失效,防止带锁回池。"""
|
||||
engine = session.get_bind()
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError("T+ 执行协调需要 MySQL 命名锁")
|
||||
connection = engine.connect()
|
||||
acquired = False
|
||||
try:
|
||||
acquired = (
|
||||
connection.execute(
|
||||
text("SELECT GET_LOCK(:name, 0)"), {"name": lock_name(session)}
|
||||
).scalar_one()
|
||||
== 1
|
||||
)
|
||||
if not acquired:
|
||||
raise TPlusBusyError("已有 T+ 任务正在执行,请稍后重试")
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
try:
|
||||
released = connection.execute(
|
||||
text("SELECT RELEASE_LOCK(:name)"), {"name": lock_name(session)}
|
||||
).scalar_one()
|
||||
if released != 1:
|
||||
connection.invalidate()
|
||||
except Exception:
|
||||
connection.invalidate()
|
||||
connection.close()
|
||||
@ -15,6 +15,7 @@ from app.core.db import dispose_engine
|
||||
from app.core.exception_handlers import register_exception_handlers
|
||||
from app.core.logging import setup_logging
|
||||
from app.core.redis_client import close_redis
|
||||
from app.core.tplus_db import dispose_tplus_engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -35,6 +36,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
)
|
||||
yield
|
||||
dispose_engine()
|
||||
dispose_tplus_engine()
|
||||
close_redis()
|
||||
logger.info("应用已停止,连接池已释放")
|
||||
|
||||
|
||||
@ -26,6 +26,12 @@ from app.models.role import Permission, Role, RolePermission
|
||||
from app.models.settlement import AdjustmentOrder, MonthlySettlement, SettlementItem
|
||||
from app.models.shipment import Shipment, ShipmentItem
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.tplus_sync import (
|
||||
TPlusSyncBatch,
|
||||
TPlusSyncError,
|
||||
TPlusSyncRecord,
|
||||
TPlusSyncWatermark,
|
||||
)
|
||||
from app.models.user import User, UserRole
|
||||
|
||||
__all__ = [
|
||||
@ -60,6 +66,10 @@ __all__ = [
|
||||
"Shipment",
|
||||
"ShipmentItem",
|
||||
"SystemSetting",
|
||||
"TPlusSyncBatch",
|
||||
"TPlusSyncError",
|
||||
"TPlusSyncRecord",
|
||||
"TPlusSyncWatermark",
|
||||
"TimestampMixin",
|
||||
"User",
|
||||
"UserRole",
|
||||
|
||||
@ -91,6 +91,9 @@ class TPlusSyncMixin:
|
||||
String(32), comment="同步批次号(同步模块写入)"
|
||||
)
|
||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime, comment="最近同步时间")
|
||||
source_updated_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime, comment="T+ 源数据更新时间(未确认时可为空)"
|
||||
)
|
||||
|
||||
|
||||
def mysql_table_args(comment: str, *extra: object) -> tuple[object, ...]:
|
||||
|
||||
149
backend/app/models/tplus_sync.py
Normal file
149
backend/app/models/tplus_sync.py
Normal file
@ -0,0 +1,149 @@
|
||||
"""T+ 同步批次、错误和原始 staging 记录。"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import BIGINT_UNSIGNED, Base, TimestampMixin, mysql_table_args
|
||||
|
||||
|
||||
class TPlusSyncBatch(Base, TimestampMixin):
|
||||
"""一次 T+ 全量或增量同步的审计记录。"""
|
||||
|
||||
__tablename__ = "tplus_sync_batches"
|
||||
__table_args__ = mysql_table_args(
|
||||
"T+ 同步批次表(记录读取范围、结果和对账统计)",
|
||||
UniqueConstraint("batch_no", name="uk_tplus_sync_batches_batch_no"),
|
||||
UniqueConstraint("request_key", name="uk_tplus_sync_batches_request_key"),
|
||||
Index("idx_tplus_sync_batches_status", "status"),
|
||||
Index("idx_tplus_sync_batches_started_at", "started_at"),
|
||||
Index("idx_tplus_sync_batches_parent_status", "parent_batch_no", "status"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(
|
||||
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
|
||||
)
|
||||
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
|
||||
sync_mode: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, comment="任务方式:FULL全量 AUDIT关联核验 INCREMENTAL预留"
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, comment="状态:QUEUED/RUNNING/SUCCEEDED/PARTIAL/FAILED"
|
||||
)
|
||||
source_database: Mapped[str | None] = mapped_column(String(128), comment="T+ 源数据库名")
|
||||
started_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime, comment="开始执行时间(排队时为空)"
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime, comment="结束时间")
|
||||
watermark_from: Mapped[datetime | None] = mapped_column(DateTime, comment="增量起始水位")
|
||||
watermark_to: Mapped[datetime | None] = mapped_column(DateTime, comment="本批次结束水位")
|
||||
source_record_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default="0", comment="源记录数"
|
||||
)
|
||||
staged_record_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default="0", comment="写入 staging 的记录数"
|
||||
)
|
||||
failed_record_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default="0", comment="失败记录数"
|
||||
)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, comment="批次错误摘要")
|
||||
entity_types_json: Mapped[list[str] | None] = mapped_column(
|
||||
JSON, comment="请求读取的实体类型;历史批次可为空"
|
||||
)
|
||||
skipped_record_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default="0", comment="重复快照跳过条数"
|
||||
)
|
||||
parent_batch_no: Mapped[str | None] = mapped_column(
|
||||
String(32), comment="重试来源批次号,保留原批次记录"
|
||||
)
|
||||
request_key: Mapped[str | None] = mapped_column(
|
||||
String(36), comment="请求幂等键,空值用于历史和直接CLI任务"
|
||||
)
|
||||
requested_by: Mapped[int | None] = mapped_column(
|
||||
BIGINT_UNSIGNED, comment="请求用户ID,CLI任务为空,无物理外键"
|
||||
)
|
||||
audit_result_json: Mapped[dict | None] = mapped_column(
|
||||
JSON, comment="关联核验结果,只含系统字典和汇总"
|
||||
)
|
||||
|
||||
|
||||
class TPlusSyncError(Base, TimestampMixin):
|
||||
"""单条 T+ 数据读取、清洗或 staging 失败记录。"""
|
||||
|
||||
__tablename__ = "tplus_sync_errors"
|
||||
__table_args__ = mysql_table_args(
|
||||
"T+ 同步错误表(记录失败行和重试信息)",
|
||||
Index("idx_tplus_sync_errors_batch_no", "batch_no"),
|
||||
Index("idx_tplus_sync_errors_entity_type", "entity_type"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(
|
||||
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
|
||||
)
|
||||
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
|
||||
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
|
||||
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
|
||||
source_id: Mapped[str | None] = mapped_column(String(64), comment="T+ 源主键")
|
||||
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
|
||||
error_code: Mapped[str] = mapped_column(String(32), nullable=False, comment="错误编码")
|
||||
error_message: Mapped[str] = mapped_column(Text, nullable=False, comment="错误说明")
|
||||
retry_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default="0", comment="已重试次数"
|
||||
)
|
||||
|
||||
|
||||
class TPlusSyncRecord(Base, TimestampMixin):
|
||||
"""T+ 原始行 staging,保存映射未定前的可追溯数据。"""
|
||||
|
||||
__tablename__ = "tplus_sync_records"
|
||||
__table_args__ = mysql_table_args(
|
||||
"T+ 原始同步 staging 表(映射确认前只保存原始行,不直接改业务表)",
|
||||
UniqueConstraint(
|
||||
"entity_type", "source_id", "row_hash", name="uk_tplus_sync_records_entity_source_hash"
|
||||
),
|
||||
Index("idx_tplus_sync_records_batch_no", "batch_no"),
|
||||
Index("idx_tplus_sync_records_source_no", "source_no"),
|
||||
Index("idx_tplus_sync_records_state", "state"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(
|
||||
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
|
||||
)
|
||||
batch_no: Mapped[str] = mapped_column(String(32), nullable=False, comment="同步批次号")
|
||||
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
|
||||
source_table: Mapped[str] = mapped_column(String(128), nullable=False, comment="T+ 源表")
|
||||
source_id: Mapped[str] = mapped_column(String(64), nullable=False, comment="T+ 源主键")
|
||||
source_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 原始编号")
|
||||
source_line_no: Mapped[str | None] = mapped_column(String(64), comment="T+ 明细行号或明细主键")
|
||||
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime, comment="T+ 源更新时间")
|
||||
row_hash: Mapped[str] = mapped_column(String(64), nullable=False, comment="原始行 SHA-256")
|
||||
state: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
server_default="PENDING",
|
||||
comment="状态:PENDING/APPLIED/SKIPPED/ERROR",
|
||||
)
|
||||
payload_json: Mapped[dict] = mapped_column(JSON, nullable=False, comment="原始行 JSON")
|
||||
|
||||
|
||||
class TPlusSyncWatermark(Base, TimestampMixin):
|
||||
"""按实体保存增量同步水位。"""
|
||||
|
||||
__tablename__ = "tplus_sync_watermarks"
|
||||
__table_args__ = mysql_table_args(
|
||||
"T+ 增量同步水位表",
|
||||
UniqueConstraint("entity_type", name="uk_tplus_sync_watermarks_entity_type"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(
|
||||
BIGINT_UNSIGNED, primary_key=True, autoincrement=True, comment="主键ID"
|
||||
)
|
||||
entity_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="业务实体类型")
|
||||
last_source_updated_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime, comment="最近源更新时间"
|
||||
)
|
||||
last_source_id: Mapped[str | None] = mapped_column(String(64), comment="同一时间点的最后源主键")
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, server_default="READY", comment="状态:READY/BLOCKED"
|
||||
)
|
||||
149
backend/app/repositories/tplus_repository.py
Normal file
149
backend/app/repositories/tplus_repository.py
Normal file
@ -0,0 +1,149 @@
|
||||
"""T+ 管理库访问;只 flush,事务由 service 管理。"""
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session, defer
|
||||
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
|
||||
|
||||
def get_batch(
|
||||
session: Session, batch_no: str, *, for_update: bool = False, with_result: bool = False
|
||||
):
|
||||
"""按批次号查询;普通查询不加载核验大字段。"""
|
||||
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == batch_no)
|
||||
if not with_result:
|
||||
statement = statement.options(defer(TPlusSyncBatch.audit_result_json))
|
||||
if for_update:
|
||||
statement = statement.with_for_update().execution_options(populate_existing=True)
|
||||
return session.scalar(statement)
|
||||
|
||||
|
||||
def get_request(session: Session, key: str, *, for_update: bool = False):
|
||||
"""锁读用于唯一约束竞争后的当前读,避免旧事务快照看不到新任务。"""
|
||||
statement = select(TPlusSyncBatch).where(TPlusSyncBatch.request_key == key)
|
||||
if for_update:
|
||||
statement = statement.with_for_update().execution_options(populate_existing=True)
|
||||
return session.scalar(statement)
|
||||
|
||||
|
||||
def active_retry(session: Session, parent_no: str):
|
||||
"""调用方先锁父批次,再用当前读判断是否已有未完成重试。"""
|
||||
return session.scalar(
|
||||
select(TPlusSyncBatch)
|
||||
.where(
|
||||
TPlusSyncBatch.parent_batch_no == parent_no,
|
||||
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
|
||||
|
||||
def increment_retry_errors(session: Session, parent_no: str) -> None:
|
||||
"""创建新重试时对原异常累计重试次数。"""
|
||||
session.execute(
|
||||
update(TPlusSyncError)
|
||||
.where(TPlusSyncError.batch_no == parent_no)
|
||||
.values(retry_count=TPlusSyncError.retry_count + 1)
|
||||
)
|
||||
session.flush()
|
||||
|
||||
|
||||
def next_queued(session: Session):
|
||||
"""调用方持有账套执行锁后领取最早排队任务。"""
|
||||
return session.scalar(
|
||||
select(TPlusSyncBatch)
|
||||
.where(TPlusSyncBatch.status == "QUEUED")
|
||||
.order_by(TPlusSyncBatch.id)
|
||||
.limit(1)
|
||||
.with_for_update()
|
||||
)
|
||||
|
||||
|
||||
def running_batches(session: Session) -> list[TPlusSyncBatch]:
|
||||
"""只在持有执行锁时使用,不能依据时间戳判断任务已经停止。"""
|
||||
return list(
|
||||
session.scalars(
|
||||
select(TPlusSyncBatch).where(TPlusSyncBatch.status == "RUNNING").with_for_update()
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def list_batches(session: Session, query):
|
||||
"""分页只读元数据,核验报告和原异常不随列表加载。"""
|
||||
filters = []
|
||||
if query.status:
|
||||
filters.append(TPlusSyncBatch.status == query.status)
|
||||
if query.task_type:
|
||||
filters.append(
|
||||
TPlusSyncBatch.sync_mode == ("AUDIT" if query.task_type == "AUDIT" else "FULL")
|
||||
)
|
||||
total = session.scalar(select(func.count()).select_from(TPlusSyncBatch).where(*filters))
|
||||
rows = session.scalars(
|
||||
select(TPlusSyncBatch)
|
||||
.where(*filters)
|
||||
.options(defer(TPlusSyncBatch.audit_result_json), defer(TPlusSyncBatch.error_message))
|
||||
.order_by(TPlusSyncBatch.id.desc())
|
||||
.offset(query.offset)
|
||||
.limit(query.page_size)
|
||||
).all()
|
||||
return rows, total or 0
|
||||
|
||||
|
||||
def active_retry_parents(session: Session, numbers: list[str]) -> set[str]:
|
||||
"""整页一次获取未完成的重试,避免逐行查询。"""
|
||||
if not numbers:
|
||||
return set()
|
||||
return set(
|
||||
session.scalars(
|
||||
select(TPlusSyncBatch.parent_batch_no).where(
|
||||
TPlusSyncBatch.parent_batch_no.in_(numbers),
|
||||
TPlusSyncBatch.status.in_(("QUEUED", "RUNNING")),
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def list_errors(session: Session, number: str, query):
|
||||
"""只读取当前批次的异常定位元数据。"""
|
||||
filters = [TPlusSyncError.batch_no == number]
|
||||
if query.entity_type:
|
||||
filters.append(TPlusSyncError.entity_type == query.entity_type)
|
||||
total = session.scalar(select(func.count()).select_from(TPlusSyncError).where(*filters))
|
||||
rows = session.scalars(
|
||||
select(TPlusSyncError)
|
||||
.where(*filters)
|
||||
.options(defer(TPlusSyncError.error_message))
|
||||
.order_by(TPlusSyncError.id.desc())
|
||||
.offset(query.offset)
|
||||
.limit(query.page_size)
|
||||
).all()
|
||||
return rows, total or 0
|
||||
|
||||
|
||||
def status_counts(session: Session) -> dict[str, int]:
|
||||
"""汇总任务表,不扫描原始快照。"""
|
||||
return dict(
|
||||
session.execute(
|
||||
select(TPlusSyncBatch.status, func.count()).group_by(TPlusSyncBatch.status)
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def latest_audit(session: Session):
|
||||
"""最近完成的核验;返回批次供查询层过滤报告。"""
|
||||
return session.scalar(
|
||||
select(TPlusSyncBatch)
|
||||
.where(
|
||||
TPlusSyncBatch.sync_mode == "AUDIT",
|
||||
TPlusSyncBatch.finished_at.is_not(None),
|
||||
)
|
||||
.order_by(TPlusSyncBatch.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
|
||||
def total_staged_records(session: Session) -> int:
|
||||
"""从批次元数据汇总累计新增数,避免扫描原始快照。"""
|
||||
return (
|
||||
session.scalar(select(func.coalesce(func.sum(TPlusSyncBatch.staged_record_count), 0))) or 0
|
||||
)
|
||||
109
backend/app/schemas/tplus.py
Normal file
109
backend/app/schemas/tplus.py
Normal file
@ -0,0 +1,109 @@
|
||||
"""T+ 管理接口合同;只暴露范围、数量和受控核验结果。"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.core.response import PageQuery
|
||||
|
||||
EntityType = Literal[
|
||||
"personnel",
|
||||
"products",
|
||||
"customers",
|
||||
"businesses",
|
||||
"shipments",
|
||||
"receipts",
|
||||
"business_items",
|
||||
"shipment_items",
|
||||
"receipt_items",
|
||||
]
|
||||
TaskType = Literal["READ", "AUDIT"]
|
||||
TaskStatus = Literal["QUEUED", "RUNNING", "SUCCEEDED", "PARTIAL", "FAILED"]
|
||||
|
||||
|
||||
class TPlusCreateIn(BaseModel):
|
||||
"""创建长任务;幂等键是调用意图的一部分,必须由调用方保留。"""
|
||||
|
||||
task_type: TaskType = "READ"
|
||||
entity_types: list[EntityType] = Field(default_factory=list, max_length=9)
|
||||
request_key: UUID
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_scope(self):
|
||||
if self.task_type == "READ" and not self.entity_types:
|
||||
raise ValueError("请选择至少一种读取数据")
|
||||
if self.task_type == "AUDIT" and self.entity_types:
|
||||
raise ValueError("关联核验不接受自定义读取范围")
|
||||
if len(set(self.entity_types)) != len(self.entity_types):
|
||||
raise ValueError("读取类型不能重复")
|
||||
return self
|
||||
|
||||
|
||||
class TPlusRetryIn(BaseModel):
|
||||
"""失败批次重试不接受任意新范围。"""
|
||||
|
||||
request_key: UUID
|
||||
|
||||
|
||||
class TPlusBatchQuery(PageQuery):
|
||||
"""批次筛选,与读取状态和任务类型保持一致。"""
|
||||
|
||||
status: TaskStatus | None = None
|
||||
task_type: TaskType | None = None
|
||||
|
||||
|
||||
class TPlusErrorQuery(PageQuery):
|
||||
"""异常只能在指定批次内查询。"""
|
||||
|
||||
entity_type: EntityType | None = None
|
||||
|
||||
|
||||
class TPlusBatchOut(BaseModel):
|
||||
"""批次列表不包含原始数据、源配置或数据库异常。"""
|
||||
|
||||
batch_no: str
|
||||
task_type: TaskType
|
||||
status: str
|
||||
entity_types: list[str] | None
|
||||
source_record_count: int
|
||||
staged_record_count: int
|
||||
skipped_record_count: int
|
||||
failed_record_count: int
|
||||
parent_batch_no: str | None
|
||||
create_time: datetime
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
can_retry: bool
|
||||
error_summary: str | None = None
|
||||
|
||||
|
||||
class TPlusBatchDetailOut(TPlusBatchOut):
|
||||
"""详情按需携带受控的关联核验结果。"""
|
||||
|
||||
audit_result: dict | None = None
|
||||
|
||||
|
||||
class TPlusErrorOut(BaseModel):
|
||||
"""异常使用固定说明,源定位仅开放整数 ID。"""
|
||||
|
||||
id: int
|
||||
entity_type: str
|
||||
source_id: str | None
|
||||
error_code: str
|
||||
error_message: str
|
||||
retry_count: int
|
||||
create_time: datetime
|
||||
|
||||
|
||||
class TPlusOverviewOut(BaseModel):
|
||||
"""配置就绪不等同于源库连通或兼容验收通过。"""
|
||||
|
||||
is_configured: bool
|
||||
configuration_message: str
|
||||
status_counts: dict[str, int]
|
||||
total_staged_records: int = 0
|
||||
compatibility_status: str = "PENDING"
|
||||
business_import_enabled: bool = False
|
||||
latest_audit: dict | None = None
|
||||
21
backend/app/services/tplus/README.md
Normal file
21
backend/app/services/tplus/README.md
Normal file
@ -0,0 +1,21 @@
|
||||
# T+ 只读同步
|
||||
|
||||
> AI 摘要:读取外部 SQL Server 的原始快照,并管理持久化任务。关键文件为 task_service、worker、sync_service、relationship_audit 和 query_service;管理员 API 为 `/api/v1/system/tplus`。
|
||||
|
||||
## 职责与边界
|
||||
|
||||
- `task_service` 创建/重试任务,控制请求幂等和事务,保留原批次与异常;HTTP 不执行源库长查询。
|
||||
- `worker.run_once` 取得 MySQL 执行锁后恢复中断任务、领取一个任务;读取和关联核验共用锁。
|
||||
- `sync_service` 主键分页读取白名单实体,每页保存原始快照,相同源 ID 和行哈希去重。
|
||||
- `relationship_audit` 执行 `db/queries/tplus/audit/queries.sql`,只返回系统字典和数量;按收款方向与单据类型核对关联。
|
||||
- `query_service` 分页元数据,按需加载白名单报告;不返回原始 payload、驱动异常或源配置。
|
||||
|
||||
## 主要入口
|
||||
|
||||
管理员 API 提供概况、批次/异常分页、详情、创建与失败重试。CLI 为 `python -m app.cli tplus-worker --once/--watch`;原 `tplus-stage --entity personnel` 可继续直接读取。
|
||||
|
||||
连接配置、启动命令和异常处理参见项目 `docs/T+本地恢复与同步开发手册.md`。后端与 worker 必须指向同一 MySQL 业务库和 T+ 账套,查询文件随部署交付。
|
||||
|
||||
## 验收限制
|
||||
|
||||
当前只写 MySQL 暂存与管理表,不写工资业务表、不推进时间水位。关联、成本、日期和负向事件的业务规则待客户确认;SQL Server 2022 的实测不能替代客户 2008 R2(10.50.x)的兼容验收。
|
||||
6
backend/app/services/tplus/__init__.py
Normal file
6
backend/app/services/tplus/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
"""T+ 只读读取、字段映射和同步 staging。"""
|
||||
|
||||
from app.services.tplus.reader import TPlusReader
|
||||
from app.services.tplus.sync_service import TPlusSyncService
|
||||
|
||||
__all__ = ["TPlusReader", "TPlusSyncService"]
|
||||
136
backend/app/services/tplus/mapping.py
Normal file
136
backend/app/services/tplus/mapping.py
Normal file
@ -0,0 +1,136 @@
|
||||
"""T+ 字段映射元数据。
|
||||
|
||||
这里记录候选字段和确认状态,不把字典中的候选直接当成业务契约。
|
||||
高风险口径在客户确认前保持 ``PENDING``,同步器只允许写入原始 staging。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class MappingStatus(StrEnum):
|
||||
"""字段映射的证据状态。"""
|
||||
|
||||
CONFIRMED = "CONFIRMED"
|
||||
CANDIDATE = "CANDIDATE"
|
||||
MISSING = "MISSING"
|
||||
PENDING = "PENDING"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TPlusMapping:
|
||||
"""一个实体的源表、候选字段和目标表说明。"""
|
||||
|
||||
entity_type: str
|
||||
source_table: str
|
||||
source_fields: tuple[str, ...]
|
||||
target_table: str
|
||||
status: MappingStatus
|
||||
unresolved: tuple[str, ...] = ()
|
||||
version: str = "0.2"
|
||||
schema_evidence: str = "ceshi20261026.bak,2026-10-27 恢复副本结构核验"
|
||||
|
||||
|
||||
TPLUS_MAPPINGS: tuple[TPlusMapping, ...] = (
|
||||
TPlusMapping(
|
||||
"personnel",
|
||||
"AA_Person",
|
||||
("id", "code", "name", "disabled", "mobilePhoneNo", "officePhoneNo", "iddepartment"),
|
||||
"personnel",
|
||||
MappingStatus.CANDIDATE,
|
||||
("工号是否等于 code", "经理隶属关系", "入职日期", "试用期"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"products",
|
||||
"AA_Inventory",
|
||||
("id", "code", "name", "specification", "idunit", "idinventoryclass"),
|
||||
"products",
|
||||
MappingStatus.CANDIDATE,
|
||||
("库存数量语义", "参考价来源", "系统产品分类"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"customers",
|
||||
"AA_Partner",
|
||||
("id", "code", "name", "Contact", "representative", "disabled", "updated"),
|
||||
"customers",
|
||||
MappingStatus.CANDIDATE,
|
||||
("客户唯一键", "改名/合并规则", "系统客户分类"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"businesses",
|
||||
"SA_SaleOrder",
|
||||
("id", "code", "voucherdate", "idcustomer", "idclerk", "idbusinesstype", "updated"),
|
||||
"businesses",
|
||||
MappingStatus.PENDING,
|
||||
("成本口径", "业务负责人覆盖规则", "含税口径"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"shipments",
|
||||
"SA_SaleDelivery",
|
||||
(
|
||||
"id",
|
||||
"code",
|
||||
"voucherdate",
|
||||
"SendDate",
|
||||
"deliveryDate",
|
||||
"auditeddate",
|
||||
"SourceVoucherCode",
|
||||
"updated",
|
||||
),
|
||||
"shipments",
|
||||
MappingStatus.PENDING,
|
||||
("发货日期", "订单关联路径", "退货/红冲规则", "运费来源"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"receipts",
|
||||
"ARAP_ReceivePayment",
|
||||
(
|
||||
"id",
|
||||
"code",
|
||||
"voucherDate",
|
||||
"auditedDate",
|
||||
"createdTime",
|
||||
"SaleOrderCode",
|
||||
"SaleOrderID",
|
||||
"ts",
|
||||
),
|
||||
"receipts",
|
||||
MappingStatus.PENDING,
|
||||
("到账日期", "收款方式", "核销状态", "回款与发货关联"),
|
||||
),
|
||||
)
|
||||
|
||||
DETAIL_MAPPINGS: tuple[TPlusMapping, ...] = (
|
||||
TPlusMapping(
|
||||
"business_items",
|
||||
"SA_SaleOrder_b",
|
||||
("id", "idSaleOrderDTO", "LatestCost"),
|
||||
"business_items",
|
||||
MappingStatus.PENDING,
|
||||
("成本口径", "含税口径"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"shipment_items",
|
||||
"SA_SaleDelivery_b",
|
||||
("ID", "idSaleDeliveryDTO", "saleOrderId"),
|
||||
"shipment_items",
|
||||
MappingStatus.PENDING,
|
||||
("发货归属", "红冲和退货"),
|
||||
),
|
||||
TPlusMapping(
|
||||
"receipt_items",
|
||||
"ARAP_ReceivePayment_b",
|
||||
("id", "idArapReceivePaymentDTO", "saleorderid"),
|
||||
"receipts",
|
||||
MappingStatus.PENDING,
|
||||
("回款拆分", "核销语义", "增量水位"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_mapping(entity_type: str) -> TPlusMapping:
|
||||
"""按实体类型取得映射;未知实体直接拒绝。"""
|
||||
for mapping in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS):
|
||||
if mapping.entity_type == entity_type:
|
||||
return mapping
|
||||
raise KeyError(f"未知 T+ 实体类型:{entity_type}")
|
||||
122
backend/app/services/tplus/query_service.py
Normal file
122
backend/app/services/tplus/query_service.py
Normal file
@ -0,0 +1,122 @@
|
||||
"""同步管理只读查询;只公开受控元数据,不公开源数据和原异常。"""
|
||||
|
||||
from app.core.exceptions import NotFoundError
|
||||
from app.core.response import PageResult
|
||||
from app.core.tplus_db import resolve_tplus_settings
|
||||
from app.repositories import tplus_repository as repository
|
||||
from app.schemas.tplus import (
|
||||
TPlusBatchDetailOut,
|
||||
TPlusBatchOut,
|
||||
TPlusErrorOut,
|
||||
TPlusOverviewOut,
|
||||
)
|
||||
from app.services.tplus import task_service
|
||||
from app.services.tplus.mapping import DETAIL_MAPPINGS, TPLUS_MAPPINGS
|
||||
from app.services.tplus.relationship_audit import public_report
|
||||
|
||||
ERROR_MESSAGES = {
|
||||
"READ_ERROR": "源数据读取失败,请检查只读连接和字段映射",
|
||||
"STAGE_ERROR": "原始数据保存失败,请核对字段类型后重试",
|
||||
"INTERRUPTED": "执行进程中断,可重新读取,已保存数据会自动去重",
|
||||
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射",
|
||||
"TYPE_UNRESOLVED": "关联核验所需的单据类型缺失或不唯一",
|
||||
}
|
||||
_ENTITY_TYPES = {item.entity_type for item in (*TPLUS_MAPPINGS, *DETAIL_MAPPINGS)}
|
||||
|
||||
|
||||
def batch_out(batch, active_parents: set[str]) -> TPlusBatchOut:
|
||||
"""只使用元数据字段组装输出,历史批次范围保持未知。"""
|
||||
entities = batch.entity_types_json
|
||||
scope_valid = (
|
||||
isinstance(entities, list)
|
||||
and bool(entities)
|
||||
and all(isinstance(item, str) and item in _ENTITY_TYPES for item in entities)
|
||||
)
|
||||
return TPlusBatchOut(
|
||||
batch_no=batch.batch_no,
|
||||
task_type="AUDIT" if batch.sync_mode == "AUDIT" else "READ",
|
||||
status=batch.status,
|
||||
entity_types=entities if scope_valid else None,
|
||||
source_record_count=batch.source_record_count,
|
||||
staged_record_count=batch.staged_record_count,
|
||||
skipped_record_count=batch.skipped_record_count,
|
||||
failed_record_count=batch.failed_record_count,
|
||||
parent_batch_no=batch.parent_batch_no,
|
||||
create_time=batch.create_time,
|
||||
started_at=batch.started_at,
|
||||
finished_at=batch.finished_at,
|
||||
can_retry=batch.status in ("FAILED", "PARTIAL")
|
||||
and (batch.sync_mode == "AUDIT" or (batch.sync_mode == "FULL" and scope_valid))
|
||||
and batch.batch_no not in active_parents,
|
||||
error_summary="任务有异常,请查看异常明细"
|
||||
if batch.status in ("FAILED", "PARTIAL")
|
||||
else None,
|
||||
)
|
||||
|
||||
|
||||
def get_batch(session, number: str) -> TPlusBatchDetailOut:
|
||||
"""按需加载报告,未找到时统一返回 404。"""
|
||||
batch = repository.get_batch(session, number, with_result=True)
|
||||
if batch is None:
|
||||
raise NotFoundError("同步批次不存在")
|
||||
out = batch_out(batch, repository.active_retry_parents(session, [number]))
|
||||
return TPlusBatchDetailOut(
|
||||
**out.model_dump(), audit_result=public_report(batch.audit_result_json)
|
||||
)
|
||||
|
||||
|
||||
def list_batches(session, query):
|
||||
"""分页批次并批量查询重试状态。"""
|
||||
rows, total = repository.list_batches(session, query)
|
||||
active = repository.active_retry_parents(session, [row.batch_no for row in rows])
|
||||
return PageResult[TPlusBatchOut](
|
||||
records=[batch_out(row, active) for row in rows],
|
||||
total=total,
|
||||
page_num=query.page_num,
|
||||
page_size=query.page_size,
|
||||
)
|
||||
|
||||
|
||||
def _safe_id(value) -> str | None:
|
||||
if isinstance(value, str) and value.isascii() and value.isdigit() and len(value) <= 10:
|
||||
return value if int(value) <= 2147483647 else None
|
||||
return None
|
||||
|
||||
|
||||
def list_errors(session, number, query):
|
||||
"""错误按编码生成固定说明,避免驱动异常泄漏连接参数。"""
|
||||
if repository.get_batch(session, number) is None:
|
||||
raise NotFoundError("同步批次不存在")
|
||||
rows, total = repository.list_errors(session, number, query)
|
||||
records = [
|
||||
TPlusErrorOut(
|
||||
id=row.id,
|
||||
entity_type=row.entity_type if row.entity_type in _ENTITY_TYPES else "system",
|
||||
source_id=_safe_id(row.source_id),
|
||||
error_code=row.error_code if row.error_code in ERROR_MESSAGES else "UNKNOWN",
|
||||
error_message=ERROR_MESSAGES.get(row.error_code, "任务未完成,请核对配置后重试"),
|
||||
retry_count=row.retry_count,
|
||||
create_time=row.create_time,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
return PageResult[TPlusErrorOut](
|
||||
records=records, total=total, page_num=query.page_num, page_size=query.page_size
|
||||
)
|
||||
|
||||
|
||||
def overview(session) -> TPlusOverviewOut:
|
||||
"""只校验配置,不在页面查询期间建立 T+ 连接。"""
|
||||
try:
|
||||
resolve_tplus_settings(task_service.get_settings())
|
||||
configured, message = True, "只读配置已就绪,实际连接结果以任务执行为准"
|
||||
except RuntimeError:
|
||||
configured, message = False, "T+ 只读配置未就绪,请联系管理员配置"
|
||||
latest = repository.latest_audit(session)
|
||||
return TPlusOverviewOut(
|
||||
is_configured=configured,
|
||||
configuration_message=message,
|
||||
status_counts=repository.status_counts(session),
|
||||
total_staged_records=repository.total_staged_records(session),
|
||||
latest_audit=public_report(latest.audit_result_json) if latest else None,
|
||||
)
|
||||
95
backend/app/services/tplus/reader.py
Normal file
95
backend/app/services/tplus/reader.py
Normal file
@ -0,0 +1,95 @@
|
||||
"""T+ 六类源表的只读读取器。"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import BigInteger, Integer, bindparam, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.tplus.mapping import TPLUS_MAPPINGS, get_mapping
|
||||
|
||||
_ENTITY_QUERY_FILES = {
|
||||
"personnel": "personnel.sql",
|
||||
"products": "products.sql",
|
||||
"customers": "customers.sql",
|
||||
"businesses": "businesses.sql",
|
||||
"shipments": "shipments.sql",
|
||||
"receipts": "receipts.sql",
|
||||
"business_items": "business_items.sql",
|
||||
"shipment_items": "shipment_items.sql",
|
||||
"receipt_items": "receipt_items.sql",
|
||||
}
|
||||
|
||||
|
||||
class TPlusReader:
|
||||
"""执行白名单 SQL 文件,禁止调用方传入任意表名或 SQL。"""
|
||||
|
||||
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
|
||||
self.session = session
|
||||
self.query_dir = query_dir or self._default_query_dir()
|
||||
|
||||
@staticmethod
|
||||
def _default_query_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[4] / "db" / "queries" / "tplus"
|
||||
|
||||
def iter_entity(
|
||||
self,
|
||||
entity_type: str,
|
||||
*,
|
||||
watermark: datetime | None = None,
|
||||
batch_size: int = 500,
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
"""按实体流式读取,水位只通过绑定参数传入。"""
|
||||
if entity_type not in _ENTITY_QUERY_FILES:
|
||||
raise KeyError(f"不支持的 T+ 实体类型:{entity_type}")
|
||||
if batch_size < 1 or batch_size > 5000:
|
||||
raise ValueError("batch_size 必须在 1 到 5000 之间")
|
||||
if watermark is not None:
|
||||
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
|
||||
get_mapping(entity_type)
|
||||
sql_path = self.query_dir / _ENTITY_QUERY_FILES[entity_type]
|
||||
if not sql_path.is_file():
|
||||
raise FileNotFoundError(f"缺少 T+ 查询文件:{sql_path}")
|
||||
|
||||
statement = text(sql_path.read_text(encoding="utf-8")).bindparams(
|
||||
bindparam("batch_size", type_=Integer()), bindparam("after_id", type_=BigInteger())
|
||||
)
|
||||
after_id = -2147483649 # 所有源表主键均已核验为 SQL Server int,包含其最小值。
|
||||
while True:
|
||||
result = self.session.execute(
|
||||
statement, {"batch_size": batch_size, "after_id": after_id}
|
||||
)
|
||||
try:
|
||||
rows = [dict(row) for row in result.mappings().all()]
|
||||
finally:
|
||||
result.close()
|
||||
if not rows:
|
||||
return
|
||||
next_id = int(rows[-1]["id"])
|
||||
if next_id <= after_id:
|
||||
raise ValueError("T+ 源主键未递增,停止读取以避免重复循环")
|
||||
yield from rows
|
||||
after_id = next_id
|
||||
|
||||
def fetch_entity(
|
||||
self,
|
||||
entity_type: str,
|
||||
*,
|
||||
watermark: datetime | None = None,
|
||||
max_rows: int = 5000,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""读取有限数量的实体行,供核验命令和单元测试使用。"""
|
||||
if max_rows < 1:
|
||||
raise ValueError("max_rows 必须大于 0")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for row in self.iter_entity(entity_type, watermark=watermark):
|
||||
rows.append(row)
|
||||
if len(rows) >= max_rows:
|
||||
break
|
||||
return rows
|
||||
|
||||
def inventory(self) -> tuple[str, ...]:
|
||||
"""返回当前适配器支持的实体类型。"""
|
||||
return tuple(mapping.entity_type for mapping in TPLUS_MAPPINGS)
|
||||
135
backend/app/services/tplus/relationship_audit.py
Normal file
135
backend/app/services/tplus/relationship_audit.py
Normal file
@ -0,0 +1,135 @@
|
||||
"""读取实际系统字典并统计带单据类型的收款关联,不推断计提口径。"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.logging import scrub
|
||||
from app.services.tplus.reader import TPlusReader
|
||||
|
||||
_COLUMNS = {
|
||||
"source_version": ("source_version",),
|
||||
"voucher_types": ("id", "code", "name"),
|
||||
"state_enums": ("id", "code", "name", "enum_name"),
|
||||
"receipt_kinds": ("voucher_type_code", "voucher_type_name", "receive_flag", "row_count"),
|
||||
"receipt_line_kinds": ("voucher_type_id", "voucher_type_name", "receive_flag", "row_count"),
|
||||
"sales_links": (
|
||||
"row_count",
|
||||
"delivery_matches",
|
||||
"detail_matches",
|
||||
"direct_order_matches",
|
||||
"detail_order_matches",
|
||||
"unambiguous_order_matches",
|
||||
"invalid_order_ids",
|
||||
"conflicting_order_ids",
|
||||
),
|
||||
"order_coverage": ("row_count", "without_order", "single_order", "multiple_orders"),
|
||||
}
|
||||
PENDING_CONFIRMATIONS = (
|
||||
("cost_basis", "绩效成本取值", "订单最新成本或出库成本的类型、时点和含税口径需业务确认"),
|
||||
("receipt_date", "到账日", "单据日、审核日和明细 arrivalDate 的业务含义需确认"),
|
||||
("ship_date", "发货日", "回款天数应使用哪个发货日期需确认"),
|
||||
("allocation", "多单分摊", "一笔收款涉及多张销货单或订单时,不能自动把整笔金额记给一单"),
|
||||
("negative_events", "退货与红冲", "负数、冲销及已结算业务的调整方式需确认"),
|
||||
)
|
||||
|
||||
|
||||
def public_report(raw: dict | None) -> dict | None:
|
||||
"""历史报告也按当前字段白名单输出,禁止透传未知嵌套数据。"""
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
result = {
|
||||
key: TPlusRelationshipAudit._safe_value(raw.get(key))
|
||||
for key in (
|
||||
"schema_version",
|
||||
"captured_at",
|
||||
"source_version",
|
||||
"is_target_engine",
|
||||
"relationship_status",
|
||||
)
|
||||
}
|
||||
result["compatibility_status"] = "PENDING"
|
||||
for section, columns in _COLUMNS.items():
|
||||
if section in ("source_version", "voucher_types"):
|
||||
continue
|
||||
value = raw.get(section)
|
||||
if isinstance(value, list):
|
||||
result[section] = [
|
||||
{key: TPlusRelationshipAudit._safe_value(row.get(key)) for key in columns}
|
||||
for row in value[:200]
|
||||
if isinstance(row, dict)
|
||||
]
|
||||
elif isinstance(value, dict):
|
||||
result[section] = {
|
||||
key: TPlusRelationshipAudit._safe_value(value.get(key)) for key in columns
|
||||
}
|
||||
else:
|
||||
result[section] = None
|
||||
result["pending_confirmations"] = [
|
||||
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
class TPlusRelationshipAudit:
|
||||
"""执行固定核验查询;结果只包含系统字典和数量,不包含客户记录。"""
|
||||
|
||||
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
|
||||
self.session = session
|
||||
root = query_dir or TPlusReader._default_query_dir()
|
||||
source = (root / "audit" / "queries.sql").read_text(encoding="utf-8")
|
||||
sections = re.split(r"(?m)^-- @query (\w+)\s*$", source)
|
||||
self.queries = dict(zip(sections[1::2], sections[2::2], strict=True))
|
||||
if set(self.queries) != set(_COLUMNS):
|
||||
raise ValueError("关联核验查询清单不完整")
|
||||
|
||||
def fetch(self, name: str, **params) -> list[dict]:
|
||||
"""只接受登记的查询名与绑定参数,过滤非合同字段。"""
|
||||
if name not in self.queries:
|
||||
raise ValueError("不支持的核验查询")
|
||||
result = self.session.execute(text(self.queries[name]), params)
|
||||
try:
|
||||
return [
|
||||
{key: self._safe_value(row.get(key)) for key in _COLUMNS[name]}
|
||||
for row in result.mappings().all()
|
||||
]
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
@staticmethod
|
||||
def _safe_value(value):
|
||||
if value is None or isinstance(value, (int, bool)):
|
||||
return value
|
||||
return scrub(re.sub(r"[\x00-\x1f]", " ", str(value)))[:200]
|
||||
|
||||
def run(self) -> dict:
|
||||
"""生成事实报告;类型缺失或有歧义时不执行该关联链路。"""
|
||||
version = self.fetch("source_version")[0]["source_version"]
|
||||
types = self.fetch("voucher_types")
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"captured_at": datetime.now().isoformat(),
|
||||
"source_version": version,
|
||||
"is_target_engine": str(version).startswith("10.50."),
|
||||
"compatibility_status": "PENDING",
|
||||
"state_enums": self.fetch("state_enums"),
|
||||
"receipt_kinds": self.fetch("receipt_kinds"),
|
||||
"receipt_line_kinds": self.fetch("receipt_line_kinds"),
|
||||
"sales_links": None,
|
||||
"order_coverage": None,
|
||||
"relationship_status": "UNRESOLVED",
|
||||
"relationship_basis": "主表按 isReceiveFlag=1(是否收款)筛选;明细按 SM_VoucherType 的 SA04 类型关联。主表 idvouchertype 是来源类型,不能据此筛本单类型。",
|
||||
"pending_confirmations": [
|
||||
{"key": k, "title": t, "reason": r} for k, t, r in PENDING_CONFIRMATIONS
|
||||
],
|
||||
}
|
||||
sale = [r["id"] for r in types if r["code"] == "SA04"]
|
||||
if len(sale) == 1:
|
||||
params = {"sale_delivery_type_id": sale[0], "receive_flag": 1}
|
||||
report["sales_links"] = self.fetch("sales_links", **params)[0]
|
||||
report["order_coverage"] = self.fetch("order_coverage", **params)[0]
|
||||
report["relationship_status"] = "OBSERVED"
|
||||
return report
|
||||
123
backend/app/services/tplus/staging.py
Normal file
123
backend/app/services/tplus/staging.py
Normal file
@ -0,0 +1,123 @@
|
||||
"""T+ 原始行 staging 服务。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import insert, select, tuple_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import TPlusSyncRecord
|
||||
|
||||
|
||||
def _json_default(value: Any) -> str:
|
||||
"""将日期、Decimal 等源字段转换为稳定 JSON 文本。"""
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Decimal):
|
||||
return str(value)
|
||||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||||
return "0x" + bytes(value).hex()
|
||||
raise TypeError("T+ 源行包含无法序列化的数据类型")
|
||||
|
||||
|
||||
def stage_row(
|
||||
session: Session,
|
||||
*,
|
||||
batch_no: str,
|
||||
entity_type: str,
|
||||
source_table: str,
|
||||
row: dict[str, Any],
|
||||
) -> tuple[TPlusSyncRecord, bool]:
|
||||
"""幂等写入单条原始行;返回记录和是否新插入。"""
|
||||
values = prepare_row(batch_no, entity_type, source_table, row)
|
||||
existing = session.execute(
|
||||
select(TPlusSyncRecord).where(
|
||||
TPlusSyncRecord.entity_type == entity_type,
|
||||
TPlusSyncRecord.source_id == values["source_id"],
|
||||
TPlusSyncRecord.row_hash == values["row_hash"],
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
|
||||
record = TPlusSyncRecord(**values)
|
||||
session.add(record)
|
||||
session.flush()
|
||||
return record, True
|
||||
|
||||
|
||||
def prepare_row(batch_no: str, entity_type: str, source_table: str, row: dict) -> dict:
|
||||
"""规范化原始值,生成与 JSON 列序列化一致的哈希。"""
|
||||
source_id = str(row["id"]) if row.get("id") is not None else ""
|
||||
if not source_id:
|
||||
raise ValueError("T+ 源行缺少 id,无法建立幂等键")
|
||||
source_no = row.get("code")
|
||||
payload = json.dumps(
|
||||
row, ensure_ascii=False, sort_keys=True, default=_json_default, allow_nan=False
|
||||
)
|
||||
return {
|
||||
"batch_no": batch_no,
|
||||
"entity_type": entity_type,
|
||||
"source_table": source_table,
|
||||
"source_id": source_id,
|
||||
"source_no": str(source_no).strip() if source_no is not None else None,
|
||||
"source_line_no": str(row.get("line_no")) if row.get("line_no") is not None else None,
|
||||
"source_updated_at": _source_datetime(row),
|
||||
"row_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
|
||||
"payload_json": json.loads(payload),
|
||||
}
|
||||
|
||||
|
||||
def stage_rows(session: Session, batch_no: str, mapping, rows: list[dict]) -> tuple[int, list]:
|
||||
"""批量去重和插入;数据库拒绝整批时逐条隔离,保留有效行。"""
|
||||
records, errors = {}, []
|
||||
for row in rows:
|
||||
try:
|
||||
values = prepare_row(batch_no, mapping.entity_type, mapping.source_table, row)
|
||||
records[(values["source_id"], values["row_hash"])] = values
|
||||
except (ValueError, TypeError) as exc:
|
||||
errors.append((row, type(exc).__name__))
|
||||
if not records:
|
||||
return 0, errors
|
||||
existing = set(
|
||||
session.execute(
|
||||
select(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).where(
|
||||
TPlusSyncRecord.entity_type == mapping.entity_type,
|
||||
tuple_(TPlusSyncRecord.source_id, TPlusSyncRecord.row_hash).in_(list(records)),
|
||||
)
|
||||
).all()
|
||||
)
|
||||
pending = [v for k, v in records.items() if k not in existing]
|
||||
if not pending:
|
||||
return 0, errors
|
||||
try:
|
||||
with session.begin_nested():
|
||||
session.execute(insert(TPlusSyncRecord), pending)
|
||||
return len(pending), errors
|
||||
except Exception:
|
||||
# 唯一约束竞争或坏行触发回滚后,再逐条处理,不能丢掉同批有效行。
|
||||
inserted = 0
|
||||
for row in rows:
|
||||
try:
|
||||
with session.begin_nested():
|
||||
_, is_new = stage_row(
|
||||
session,
|
||||
batch_no=batch_no,
|
||||
entity_type=mapping.entity_type,
|
||||
source_table=mapping.source_table,
|
||||
row=row,
|
||||
)
|
||||
inserted += int(is_new)
|
||||
except Exception as exc:
|
||||
if not any(old is row for old, _ in errors):
|
||||
errors.append((row, type(exc).__name__))
|
||||
return inserted, errors
|
||||
|
||||
|
||||
def _source_datetime(row: dict[str, Any]) -> datetime | None:
|
||||
"""只接收真实时间值;SQL Server rowversion 不能当作日期。"""
|
||||
value = row.get("source_updated_at") or row.get("updated")
|
||||
return value if isinstance(value, datetime) else None
|
||||
131
backend/app/services/tplus/sync_service.py
Normal file
131
backend/app/services/tplus/sync_service.py
Normal file
@ -0,0 +1,131 @@
|
||||
"""T+ 只读快照同步:批量 staging、幂等去重和失败隔离。"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime
|
||||
from itertools import batched
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.tplus_lock import sync_lock
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
from app.services.tplus.mapping import TPlusMapping, get_mapping
|
||||
from app.services.tplus.reader import TPlusReader
|
||||
from app.services.tplus.staging import stage_rows
|
||||
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running
|
||||
|
||||
|
||||
class TPlusSyncService:
|
||||
"""按分页提交原始快照,业务表始终不参与本服务。"""
|
||||
|
||||
def __init__(self, source_session: Session, target_session: Session) -> None:
|
||||
self.source_session = source_session
|
||||
self.target_session = target_session
|
||||
configured = get_settings().t_plus_query_dir
|
||||
self.reader = TPlusReader(source_session, Path(configured) if configured else None)
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
entity_types: Iterable[str],
|
||||
watermark: datetime | None = None,
|
||||
batch_no: str | None = None,
|
||||
batch_size: int = 500,
|
||||
) -> TPlusSyncBatch:
|
||||
"""执行可重复的全量读取,提交每页进度;字段语义未定时拒绝时间增量。"""
|
||||
entities = tuple(dict.fromkeys(entity_types))
|
||||
if not entities or not 1 <= batch_size <= 5000:
|
||||
raise ValueError("至少指定一个实体,batch_size 必须在 1 到 5000 之间")
|
||||
if watermark is not None:
|
||||
raise ValueError("增量水位语义尚未确认,当前只支持全量读取和哈希去重")
|
||||
for entity in entities:
|
||||
get_mapping(entity)
|
||||
with sync_lock(self.target_session):
|
||||
recover_running(self.target_session)
|
||||
batch = self._start_direct(entities, batch_no)
|
||||
try:
|
||||
return self.execute_claimed(batch, batch_size)
|
||||
except BaseException:
|
||||
self.target_session.rollback()
|
||||
fail_batch(self.target_session, batch, "INTERRUPTED")
|
||||
raise
|
||||
|
||||
def _start_direct(self, entities, batch_no):
|
||||
batch = TPlusSyncBatch(
|
||||
batch_no=batch_no or new_batch_no(),
|
||||
sync_mode="FULL",
|
||||
status="RUNNING",
|
||||
source_database=get_settings().t_plus_database or None,
|
||||
started_at=datetime.now(),
|
||||
source_record_count=0,
|
||||
staged_record_count=0,
|
||||
failed_record_count=0,
|
||||
skipped_record_count=0,
|
||||
entity_types_json=list(entities),
|
||||
)
|
||||
self.target_session.add(batch)
|
||||
self.target_session.commit()
|
||||
return batch
|
||||
|
||||
def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
|
||||
"""处理已持锁且标记 RUNNING 的任务;仅由直接入口或 worker 调用。"""
|
||||
if batch.status != "RUNNING" or not batch.entity_types_json:
|
||||
raise ValueError("任务尚未领取或读取范围不完整")
|
||||
mappings = [get_mapping(entity) for entity in batch.entity_types_json]
|
||||
for mapping in mappings:
|
||||
try:
|
||||
self._read_entity(batch, mapping, batch_size)
|
||||
except Exception as exc:
|
||||
self.target_session.rollback()
|
||||
self._record_error(batch, mapping, {}, "READ_ERROR", type(exc).__name__)
|
||||
self.target_session.commit()
|
||||
batch.status = "SUCCEEDED"
|
||||
if batch.failed_record_count:
|
||||
failed_rows = (
|
||||
self.target_session.scalar(
|
||||
select(func.count())
|
||||
.select_from(TPlusSyncError)
|
||||
.where(
|
||||
TPlusSyncError.batch_no == batch.batch_no,
|
||||
TPlusSyncError.error_code == "STAGE_ERROR",
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
batch.status = "PARTIAL" if batch.source_record_count > failed_rows else "FAILED"
|
||||
batch.error_message = "部分数据未完成,请查询同步错误记录并重跑全量快照"
|
||||
batch.finished_at = datetime.now()
|
||||
batch.watermark_to = None
|
||||
self.target_session.commit()
|
||||
return batch
|
||||
|
||||
def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
|
||||
"""批量写入一类数据;每页提交后即使进程中断也可幂等重跑。"""
|
||||
source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
|
||||
for chunk in batched(source, size):
|
||||
inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
|
||||
batch.source_record_count += len(chunk)
|
||||
batch.staged_record_count += inserted
|
||||
batch.skipped_record_count += len(chunk) - inserted - len(errors)
|
||||
for row, reason in errors:
|
||||
self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
|
||||
self.target_session.commit()
|
||||
|
||||
def _record_error(
|
||||
self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
|
||||
) -> None:
|
||||
"""只记录错误类别和源 ID,避免数据库异常携带个人信息或凭据。"""
|
||||
self.target_session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=batch.batch_no,
|
||||
entity_type=mapping.entity_type,
|
||||
source_table=mapping.source_table,
|
||||
source_id=str(row["id"]) if row.get("id") is not None else None,
|
||||
source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
|
||||
error_code=code,
|
||||
error_message=f"源数据处理失败({reason}),请核对连接和字段",
|
||||
)
|
||||
)
|
||||
batch.failed_record_count += 1
|
||||
184
backend/app/services/tplus/task_service.py
Normal file
184
backend/app/services/tplus/task_service.py
Normal file
@ -0,0 +1,184 @@
|
||||
"""持久化任务创建和重试,不在 HTTP 事务中读取源库。"""
|
||||
|
||||
import secrets
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.exceptions import BadRequestError, NotFoundError
|
||||
from app.core.tplus_db import resolve_tplus_settings
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError, User
|
||||
from app.repositories import tplus_repository as repository
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
|
||||
from app.services import audit_service
|
||||
from app.services.audit_service import AuditRecord, RequestMeta
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskActor:
|
||||
"""HTTP 操作者与请求元信息,CLI 调用不传。"""
|
||||
|
||||
user: User
|
||||
meta: RequestMeta | None = None
|
||||
|
||||
|
||||
def new_batch_no() -> str:
|
||||
"""生成最长 24 字符的可追溯批次号。"""
|
||||
return f"TP{datetime.now():%Y%m%d%H%M%S}{secrets.token_hex(4)}"
|
||||
|
||||
|
||||
def _audit(session, actor, action, data):
|
||||
if actor is None:
|
||||
return nullcontext()
|
||||
return audit_service.audit(
|
||||
session,
|
||||
operator=actor.user,
|
||||
meta=actor.meta,
|
||||
record=AuditRecord("T+数据同步", action, "T+ 数据任务" + action, data),
|
||||
)
|
||||
|
||||
|
||||
def _source_database() -> str:
|
||||
try:
|
||||
return resolve_tplus_settings(get_settings()).t_plus_database
|
||||
except RuntimeError as exc:
|
||||
raise BadRequestError(str(exc)) from None
|
||||
|
||||
|
||||
def _same_intent(batch, mode, entities, parent):
|
||||
if (
|
||||
batch.sync_mode != mode
|
||||
or sorted(batch.entity_types_json or []) != sorted(entities)
|
||||
or batch.parent_batch_no != parent
|
||||
):
|
||||
raise BadRequestError("幂等键已用于不同的任务请求")
|
||||
return batch
|
||||
|
||||
|
||||
def _insert(session, values):
|
||||
try:
|
||||
with session.begin_nested():
|
||||
batch = TPlusSyncBatch(**values)
|
||||
session.add(batch)
|
||||
session.flush()
|
||||
return batch, True
|
||||
except IntegrityError:
|
||||
batch = repository.get_request(session, values["request_key"], for_update=True)
|
||||
if batch is None:
|
||||
raise
|
||||
return _same_intent(
|
||||
batch, values["sync_mode"], values["entity_types_json"], values["parent_batch_no"]
|
||||
), False
|
||||
|
||||
|
||||
def _values(payload, database, actor, parent=None):
|
||||
return {
|
||||
"batch_no": new_batch_no(),
|
||||
"request_key": str(payload.request_key),
|
||||
"status": "QUEUED",
|
||||
"sync_mode": "AUDIT" if payload.task_type == "AUDIT" else "FULL",
|
||||
"entity_types_json": sorted(payload.entity_types),
|
||||
"source_database": database,
|
||||
"requested_by": actor.user.id if actor else None,
|
||||
"parent_batch_no": parent,
|
||||
"started_at": None,
|
||||
"source_record_count": 0,
|
||||
"staged_record_count": 0,
|
||||
"failed_record_count": 0,
|
||||
"skipped_record_count": 0,
|
||||
}
|
||||
|
||||
|
||||
def create_task(session: Session, payload: TPlusCreateIn, actor: TaskActor | None = None):
|
||||
"""原子创建任务;重放已存在的意图不依赖此刻的源连接状态。"""
|
||||
with _audit(session, actor, "创建读取任务", payload.model_dump(mode="json")):
|
||||
existing = repository.get_request(session, str(payload.request_key))
|
||||
mode = "AUDIT" if payload.task_type == "AUDIT" else "FULL"
|
||||
if existing:
|
||||
batch = _same_intent(existing, mode, payload.entity_types, None)
|
||||
else:
|
||||
batch, _ = _insert(session, _values(payload, _source_database(), actor))
|
||||
session.commit()
|
||||
return batch
|
||||
|
||||
|
||||
def retry_task(
|
||||
session: Session, parent_no: str, payload: TPlusRetryIn, actor: TaskActor | None = None
|
||||
):
|
||||
"""锁父批次后创建一个新任务;不覆盖原批次及异常历史。"""
|
||||
with _audit(
|
||||
session,
|
||||
actor,
|
||||
"重试读取任务",
|
||||
{"parent_batch_no": parent_no, "request_key": str(payload.request_key)},
|
||||
):
|
||||
existing = repository.get_request(session, str(payload.request_key))
|
||||
if existing:
|
||||
if existing.parent_batch_no != parent_no:
|
||||
raise BadRequestError("幂等键已用于不同的重试请求")
|
||||
session.commit()
|
||||
batch = existing
|
||||
else:
|
||||
parent = repository.get_batch(session, parent_no, for_update=True)
|
||||
if parent is None:
|
||||
raise NotFoundError("同步批次不存在")
|
||||
request = _retry_request(parent, payload)
|
||||
active = repository.active_retry(session, parent_no)
|
||||
if active:
|
||||
if active.request_key != str(payload.request_key):
|
||||
raise BadRequestError("该批次已有未完成的重试任务,请刷新列表查看")
|
||||
batch = active
|
||||
else:
|
||||
batch, inserted = _insert(
|
||||
session, _values(request, _source_database(), actor, parent_no)
|
||||
)
|
||||
if inserted:
|
||||
repository.increment_retry_errors(session, parent_no)
|
||||
session.commit()
|
||||
return batch
|
||||
|
||||
|
||||
def _retry_request(parent, payload):
|
||||
if parent.status not in ("FAILED", "PARTIAL"):
|
||||
raise BadRequestError("只能重试失败或部分失败的批次")
|
||||
if parent.sync_mode not in ("FULL", "AUDIT"):
|
||||
raise BadRequestError("该同步方式尚未启用,请重新选择读取任务")
|
||||
if parent.sync_mode != "AUDIT" and parent.entity_types_json is None:
|
||||
raise BadRequestError("历史批次未记录范围,请重新选择读取类型")
|
||||
return TPlusCreateIn(
|
||||
task_type="AUDIT" if parent.sync_mode == "AUDIT" else "READ",
|
||||
entity_types=parent.entity_types_json or [],
|
||||
request_key=payload.request_key,
|
||||
)
|
||||
|
||||
|
||||
def fail_batch(session: Session, batch: TPlusSyncBatch, code: str) -> None:
|
||||
"""只保存固定错误摘要;即使原异常含连接密码也不直接落库。"""
|
||||
messages = {
|
||||
"INTERRUPTED": "执行进程中断,请重新读取;已暂存数据会自动去重",
|
||||
"EXECUTION_ERROR": "任务执行失败,请检查只读连接和字段映射后重试",
|
||||
"TYPE_UNRESOLVED": "核验所需单据类型缺失或不唯一,请核对源字典",
|
||||
}
|
||||
message = messages.get(code, "任务未完成,请核对运行配置")
|
||||
batch.status, batch.finished_at, batch.error_message = "FAILED", datetime.now(), message
|
||||
batch.failed_record_count = (batch.failed_record_count or 0) + 1
|
||||
session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=batch.batch_no,
|
||||
entity_type="system",
|
||||
source_table="system",
|
||||
error_code=code,
|
||||
error_message=message,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def recover_running(session: Session) -> None:
|
||||
"""仅在持有执行锁时调用;锁证明之前的执行者已经退出。"""
|
||||
for batch in repository.running_batches(session):
|
||||
fail_batch(session, batch, "INTERRUPTED")
|
||||
61
backend/app/services/tplus/worker.py
Normal file
61
backend/app/services/tplus/worker.py
Normal file
@ -0,0 +1,61 @@
|
||||
"""持久化任务执行者;获取连接级锁后才恢复或领取任务。"""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.tplus_db import get_tplus_session_factory, resolve_tplus_settings
|
||||
from app.core.tplus_lock import TPlusBusyError, sync_lock
|
||||
from app.repositories import tplus_repository as repository
|
||||
from app.services.tplus import task_service
|
||||
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
|
||||
from app.services.tplus.sync_service import TPlusSyncService
|
||||
|
||||
|
||||
def run_once(session, source_factory=None):
|
||||
"""返回本轮处理的批次号;忙锁或队列为空时不假称已执行。"""
|
||||
try:
|
||||
with sync_lock(session):
|
||||
task_service.recover_running(session)
|
||||
batch = repository.next_queued(session)
|
||||
if batch is None:
|
||||
session.rollback()
|
||||
return None
|
||||
batch.status, batch.started_at = "RUNNING", datetime.now()
|
||||
session.commit()
|
||||
_execute(session, batch, source_factory)
|
||||
return batch.batch_no
|
||||
except TPlusBusyError:
|
||||
session.rollback()
|
||||
return None
|
||||
|
||||
|
||||
def _execute(session, batch, source_factory):
|
||||
try:
|
||||
settings = resolve_tplus_settings(task_service.get_settings())
|
||||
if batch.source_database and batch.source_database != settings.t_plus_database:
|
||||
raise RuntimeError("任务排队后的源账套配置已改变")
|
||||
factory = source_factory or get_tplus_session_factory()
|
||||
with factory() as source:
|
||||
if batch.sync_mode == "AUDIT":
|
||||
root = Path(settings.t_plus_query_dir) if settings.t_plus_query_dir else None
|
||||
report = TPlusRelationshipAudit(source, root).run()
|
||||
batch.audit_result_json = report
|
||||
batch.source_record_count = sum(row["row_count"] for row in report["receipt_kinds"])
|
||||
batch.finished_at, batch.status = datetime.now(), "SUCCEEDED"
|
||||
if report["relationship_status"] != "OBSERVED":
|
||||
task_service.fail_batch(session, batch, "TYPE_UNRESOLVED")
|
||||
else:
|
||||
session.commit()
|
||||
else:
|
||||
TPlusSyncService(source, session).execute_claimed(batch)
|
||||
except BaseException as exc:
|
||||
session.rollback()
|
||||
task_service.fail_batch(
|
||||
session,
|
||||
batch,
|
||||
"INTERRUPTED"
|
||||
if isinstance(exc, (KeyboardInterrupt, SystemExit))
|
||||
else "EXECUTION_ERROR",
|
||||
)
|
||||
if isinstance(exc, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
@ -38,6 +38,11 @@ dependencies = [
|
||||
"python-multipart>=0.0.32",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
tplus = [
|
||||
"pyodbc>=5.2,<6",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.3",
|
||||
|
||||
@ -1,6 +1,21 @@
|
||||
# 后端测试
|
||||
|
||||
> AI摘要:本项目后端的测试用例清单与运行记录。按「基础设施层随实现即时测、业务模块整模块完成时统一测」分层组织。当前 697 个用例全绿(2026-10-16,M3 增量批次后)。
|
||||
> AI 摘要:本轮 T+ 管理已完成真实 MySQL、只读源库与浏览器验证。最终单元测试 602 项通过;管理/并发/结构定点 18 项通过。真正的 2008 R2 验收仍未执行。
|
||||
|
||||
## T+ 本轮核验(2026-10-27)
|
||||
|
||||
- 完整回归在独立 MySQL `payroll_tplus_validation_20261027`、Redis 15 库运行:984 passed、2 failed。两项失败均为配置单测预期默认 Redis 0 库,而进程继承了集成用的 `REDIS_DB=15`;没有修改默认配置或削弱断言。
|
||||
- 后续将单元测试放在不设置上述集成覆盖值的独立进程中运行:602 passed。最终管理 API、真实并发和结构一致性定点测试:18 passed,包含新增加的配置关闭及创建/重试审计。
|
||||
- 真实并发:相同请求只生成一个批次;不同键同时重试同父批次只有一个成功,无死锁;执行锁跨 ORM commit 仍有效,退出后可再获取。
|
||||
- V15 在隔离库 downgrade 到 V14,再 upgrade 到 head 通过;当前 head `e5f607a8b9c0`。模型、Alembic、留存 SQL 和初始化脚本的结构检查通过;此前 V1–V14 留存 SQL 已从空库执行通过。
|
||||
- 人员重读 847 条,新增 0、重复跳过 847、失败 0;关联报告覆盖 103,318 条销货类收款明细,50,044 条有明确订单关联,六类业务表行数不变。独立证据位于 D 盘 reports/management_verification_2026-10-27.json。
|
||||
- Chrome 真实接口检查通过:排队、受控失败、异常查看、页面重试、人员去重、关联检查、筛选、普通用户拒绝访问,以及 320/768/1024/1440 宽度。无页面运行错误;临时账号和批次已清理。
|
||||
- 加载、空态、配置关闭、概况失败另外使用浏览器响应模拟验证,不能与真实源链路混记。证据位于 .ai/archive/ai-aux/preview/tplus/。
|
||||
- 320 像素主内容可操作,现有全局顶栏仍有文字挤压;未扩大本轮范围修改共享布局。宽表保留横向滚动,截图和限制见同步管理验收报告。
|
||||
- 前端类型检查、生产构建和 Ruff 通过;构建保留既有的大分块提醒,测试有现存 Starlette TestClient 弃用提醒。
|
||||
- 源查询保持参数化与旧版语法,时间增量和业务表写入均关闭。合法 Windows/SQL Server 2008 R2 安装介质仍未取得,不能把 2022 的结果标记为客户版本兼容。
|
||||
|
||||
单元测试与集成测试建议分独立终端运行:单元测试不设置集成覆盖值;集成测试使用隔离 MySQL 和 Redis。通用集成夹具会清理数据,不能指向开发或客户业务库。Windows 可设置 `PYTEST_DEBUG_TEMPROOT` 到项目 D 盘的 ai-aux/pytest-tmp,避免不同权限令牌共享系统临时目录。
|
||||
|
||||
## 一、测试约定
|
||||
|
||||
|
||||
@ -4,6 +4,36 @@ import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tplus_session(monkeypatch):
|
||||
"""T+ 管理逻辑的真实内存事务;MySQL 并发锁另做集成验证。"""
|
||||
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
|
||||
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
metadata = MetaData()
|
||||
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
|
||||
table = model.__table__.to_metadata(metadata)
|
||||
table.c.id.type = Integer()
|
||||
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
|
||||
metadata.create_all(engine)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_database="unit_tplus",
|
||||
t_plus_user="reader",
|
||||
t_plus_odbc_driver="unit",
|
||||
)
|
||||
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
|
||||
with Session(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client() -> TestClient:
|
||||
"""构造应用测试客户端(会话级复用,避免重复构建应用)。
|
||||
|
||||
@ -324,3 +324,36 @@ def normal(make_user, auth_headers) -> dict:
|
||||
user = make_user([ROLE_USER])
|
||||
user["headers"] = auth_headers(user)
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tplus_tasks(monkeypatch):
|
||||
"""隔离测试任务,仅清理此夹具创建的批次和异常。"""
|
||||
from app.core.config import Settings
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_database="unit_tplus",
|
||||
t_plus_user="reader",
|
||||
t_plus_odbc_driver="unit",
|
||||
)
|
||||
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
|
||||
keys = []
|
||||
|
||||
def payload():
|
||||
key = str(uuid.uuid4())
|
||||
keys.append(key)
|
||||
return {"task_type": "READ", "entity_types": ["personnel"], "request_key": key}
|
||||
|
||||
yield payload
|
||||
with get_session_factory()() as db:
|
||||
numbers = list(
|
||||
db.scalars(select(TPlusSyncBatch.batch_no).where(TPlusSyncBatch.request_key.in_(keys)))
|
||||
)
|
||||
if numbers:
|
||||
db.execute(delete(TPlusSyncError).where(TPlusSyncError.batch_no.in_(numbers)))
|
||||
db.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.batch_no.in_(numbers)))
|
||||
db.commit()
|
||||
|
||||
@ -487,8 +487,18 @@ class TestManagerAggregate:
|
||||
assert len([i for i in out["items"] if i["calc_type"] == 2]) == 1
|
||||
assert len([i for i in out["items"] if i["calc_type"] == 1]) == 1
|
||||
|
||||
def test_月末取参_预约变更影响本月聚合(self, client: TestClient, admin: dict) -> None:
|
||||
def test_月末取参_预约变更影响本月聚合(
|
||||
self, client: TestClient, admin: dict, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""取参锚点=所属月月末:10-25 生效的预约变更应影响 2026-10 月结。"""
|
||||
|
||||
class ScenarioDate(date):
|
||||
@classmethod
|
||||
def today(cls):
|
||||
return cls(2026, 10, 20)
|
||||
|
||||
# 固定预约场景的申请日,避免现实日期超过 10-25 后误测成“禁止追溯修改”。
|
||||
monkeypatch.setattr("app.services.param_service.date", ScenarioDate)
|
||||
resp = client.post(
|
||||
"/api/v1/system/params/changes",
|
||||
json={
|
||||
|
||||
90
backend/tests/integration/test_tplus_concurrency.py
Normal file
90
backend/tests/integration/test_tplus_concurrency.py
Normal file
@ -0,0 +1,90 @@
|
||||
"""在隔离 MySQL 中验证幂等竞争、父批次串行化和连接级锁。"""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Barrier
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, func, select, text
|
||||
|
||||
from app.core.db import get_session_factory
|
||||
from app.core.exceptions import BadRequestError
|
||||
from app.core.tplus_lock import TPlusBusyError, sync_lock
|
||||
from app.models import TPlusSyncBatch
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
|
||||
from app.services.tplus import task_service
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _parallel(action):
|
||||
barrier = Barrier(2)
|
||||
|
||||
def run():
|
||||
with get_session_factory()() as session:
|
||||
barrier.wait(timeout=10)
|
||||
return action(session)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(run) for _ in range(2)]
|
||||
return [future.result(timeout=20) for future in futures]
|
||||
|
||||
|
||||
def test_并发相同请求只创建一个批次(tplus_tasks):
|
||||
payload = TPlusCreateIn(**tplus_tasks())
|
||||
numbers = _parallel(lambda session: task_service.create_task(session, payload).batch_no)
|
||||
assert numbers[0] == numbers[1]
|
||||
with get_session_factory()() as session:
|
||||
assert (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(TPlusSyncBatch)
|
||||
.where(TPlusSyncBatch.request_key == str(payload.request_key))
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_并发不同请求重试同父批次只允许一个(tplus_tasks):
|
||||
with get_session_factory()() as session:
|
||||
parent = task_service.create_task(session, TPlusCreateIn(**tplus_tasks()))
|
||||
parent.status = "FAILED"
|
||||
session.commit()
|
||||
number = parent.batch_no
|
||||
|
||||
def retry(session):
|
||||
try:
|
||||
return task_service.retry_task(
|
||||
session, number, TPlusRetryIn(request_key=uuid4())
|
||||
).batch_no
|
||||
except BadRequestError:
|
||||
session.rollback()
|
||||
return "REJECTED"
|
||||
|
||||
try:
|
||||
numbers = _parallel(retry)
|
||||
assert numbers.count("REJECTED") == 1
|
||||
with get_session_factory()() as session:
|
||||
assert (
|
||||
session.scalar(
|
||||
select(func.count())
|
||||
.select_from(TPlusSyncBatch)
|
||||
.where(TPlusSyncBatch.parent_batch_no == number)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
finally:
|
||||
with get_session_factory()() as session:
|
||||
session.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.parent_batch_no == number))
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_执行锁跨提交保持且释放后可获取():
|
||||
with get_session_factory()() as owner, get_session_factory()() as other:
|
||||
with sync_lock(owner):
|
||||
owner.execute(text("SELECT 1"))
|
||||
owner.commit()
|
||||
with pytest.raises(TPlusBusyError), sync_lock(other):
|
||||
pytest.fail("另一执行者不应取得锁")
|
||||
with sync_lock(other):
|
||||
assert other.scalar(text("SELECT 1")) == 1
|
||||
129
backend/tests/integration/test_tplus_management.py
Normal file
129
backend/tests/integration/test_tplus_management.py
Normal file
@ -0,0 +1,129 @@
|
||||
"""管理员可管理原始读取任务;权限、幂等和异常脱敏由真实接口验证。"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import get_session_factory
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
BASE = "/api/v1/system/tplus"
|
||||
|
||||
|
||||
def test_未登录和普通用户不能访问管理(client, admin, normal, tplus_tasks):
|
||||
assert client.get(BASE + "/overview").status_code == 401
|
||||
assert client.get(BASE + "/overview", headers=normal["headers"]).status_code == 403
|
||||
assert (
|
||||
client.post(BASE + "/batches", json=tplus_tasks(), headers=normal["headers"]).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_创建重复请求返回同一批次并可分页查看(client, admin, tplus_tasks):
|
||||
payload = tplus_tasks()
|
||||
first = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
|
||||
assert first.status_code == 200
|
||||
second = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
|
||||
assert second.status_code == 200
|
||||
number = first.json()["data"]["batch_no"]
|
||||
assert second.json()["data"]["batch_no"] == number
|
||||
page = client.get(
|
||||
BASE + "/batches", params={"pageSize": 1, "status": "QUEUED"}, headers=admin["headers"]
|
||||
)
|
||||
assert page.status_code == 200 and page.json()["data"]["pageSize"] == 1
|
||||
detail = client.get(BASE + "/batches/" + number, headers=admin["headers"])
|
||||
assert detail.status_code == 200
|
||||
assert detail.json()["data"]["status"] == "QUEUED"
|
||||
|
||||
|
||||
def test_异常返回固定说明不泄漏数据库参数(client, admin, tplus_tasks):
|
||||
response = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"])
|
||||
assert response.status_code == 200
|
||||
number = response.json()["data"]["batch_no"]
|
||||
with get_session_factory()() as db:
|
||||
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
|
||||
batch.status, batch.error_message = "FAILED", "password=private-value phone=13800000000"
|
||||
db.add(
|
||||
TPlusSyncError(
|
||||
batch_no=number,
|
||||
entity_type="personnel",
|
||||
source_table="AA_Person",
|
||||
source_id="12",
|
||||
error_code="READ_ERROR",
|
||||
error_message=batch.error_message,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
for path in (f"/batches/{number}", f"/batches/{number}/errors"):
|
||||
result = client.get(BASE + path, headers=admin["headers"])
|
||||
assert result.status_code == 200
|
||||
assert "private-value" not in result.text and "13800000000" not in result.text
|
||||
|
||||
|
||||
def test_不支持的实体和不存在的批次有明确错误(client, admin, tplus_tasks):
|
||||
payload = tplus_tasks()
|
||||
payload["entity_types"] = ["users;drop"]
|
||||
assert client.post(BASE + "/batches", json=payload, headers=admin["headers"]).status_code == 400
|
||||
assert client.get(BASE + "/batches/TPmissing", headers=admin["headers"]).status_code == 404
|
||||
|
||||
|
||||
def test_配置关闭时可查询历史但不能排队(client, admin, tplus_tasks, monkeypatch):
|
||||
from app.services.tplus import task_service
|
||||
|
||||
settings = task_service.get_settings().model_copy(update={"t_plus_enabled": False})
|
||||
monkeypatch.setattr(task_service, "get_settings", lambda: settings)
|
||||
response = client.get(BASE + "/overview", headers=admin["headers"])
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["is_configured"] is False
|
||||
assert response.json()["data"]["business_import_enabled"] is False
|
||||
assert response.json()["data"]["compatibility_status"] == "PENDING"
|
||||
assert (
|
||||
client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).status_code
|
||||
== 400
|
||||
)
|
||||
|
||||
|
||||
def test_失败批次重试保留异常并返回新批次(client, admin, tplus_tasks):
|
||||
from app.models import OperationLog
|
||||
|
||||
parent = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).json()[
|
||||
"data"
|
||||
]
|
||||
number = parent["batch_no"]
|
||||
with get_session_factory()() as db:
|
||||
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
|
||||
batch.status, batch.failed_record_count = "FAILED", 1
|
||||
db.add(
|
||||
TPlusSyncError(
|
||||
batch_no=number,
|
||||
entity_type="personnel",
|
||||
source_table="AA_Person",
|
||||
source_id="1",
|
||||
error_code="READ_ERROR",
|
||||
error_message="读取失败",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
request = {"request_key": tplus_tasks()["request_key"]}
|
||||
first = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
|
||||
replay = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
|
||||
assert first.status_code == replay.status_code == 200
|
||||
child = first.json()["data"]
|
||||
assert child["parent_batch_no"] == number and child["batch_no"] != number
|
||||
assert replay.json()["data"]["batch_no"] == child["batch_no"]
|
||||
detail = client.get(BASE + f"/batches/{number}", headers=admin["headers"]).json()["data"]
|
||||
assert detail["can_retry"] is False
|
||||
with get_session_factory()() as db:
|
||||
assert (
|
||||
db.scalar(select(TPlusSyncError.retry_count).where(TPlusSyncError.batch_no == number))
|
||||
== 1
|
||||
)
|
||||
assert db.scalar(
|
||||
select(OperationLog.id)
|
||||
.where(
|
||||
OperationLog.user_id == admin["id"],
|
||||
OperationLog.module == "T+数据同步",
|
||||
OperationLog.action == "重试读取任务",
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
@ -13,7 +13,12 @@ VALID_SECRET = "x" * 32
|
||||
|
||||
def _settings(**overrides: object) -> Settings:
|
||||
"""构造一个不读取 .env 的配置实例。"""
|
||||
params: dict[str, object] = {"_env_file": None, "app_secret_key": VALID_SECRET}
|
||||
# _env_file=None 只禁用文件读取;显式值避免隔离测试库的环境变量影响默认值断言。
|
||||
params: dict[str, object] = {
|
||||
"_env_file": None,
|
||||
"app_secret_key": VALID_SECRET,
|
||||
"mysql_database": "payroll",
|
||||
}
|
||||
params.update(overrides)
|
||||
return Settings(**params) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@ -44,6 +44,10 @@ EXPECTED_TABLES = {
|
||||
"monthly_settlements",
|
||||
"settlement_items",
|
||||
"adjustment_orders",
|
||||
"tplus_sync_batches",
|
||||
"tplus_sync_errors",
|
||||
"tplus_sync_records",
|
||||
"tplus_sync_watermarks",
|
||||
}
|
||||
|
||||
# 单复数同形的集合名词:本身即复数语义,不受「以 s 结尾」检查约束
|
||||
|
||||
58
backend/tests/unit/test_tplus_config.py
Normal file
58
backend/tests/unit/test_tplus_config.py
Normal file
@ -0,0 +1,58 @@
|
||||
"""T+ 配置和读取适配层的纯单元测试。"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.tplus.mapping import MappingStatus, get_mapping
|
||||
from app.services.tplus.reader import TPlusReader
|
||||
|
||||
|
||||
def _settings(**overrides: object) -> Settings:
|
||||
params: dict[str, object] = {"_env_file": None, "app_secret_key": "x" * 32}
|
||||
params.update(overrides)
|
||||
return Settings(**params) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_T加数据库URL使用mssql_pyodbc并隐藏密码() -> None:
|
||||
settings = _settings(
|
||||
t_plus_enabled=True,
|
||||
t_plus_database="UFTData472115_000001",
|
||||
t_plus_user="reader",
|
||||
t_plus_password="p@ss:w/ord",
|
||||
)
|
||||
url = settings.t_plus_database_url
|
||||
assert url.drivername == "mssql+pyodbc"
|
||||
assert url.database == "UFTData472115_000001"
|
||||
assert "p@ss:w/ord" not in url.render_as_string()
|
||||
|
||||
|
||||
def test_T加未启用时读取器不建立连接() -> None:
|
||||
settings = _settings(t_plus_enabled=False)
|
||||
assert settings.t_plus_enabled is False
|
||||
|
||||
|
||||
def test_T加查询文件覆盖六类实体() -> None:
|
||||
reader = TPlusReader.__new__(TPlusReader)
|
||||
reader.query_dir = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
|
||||
assert reader.inventory() == (
|
||||
"personnel",
|
||||
"products",
|
||||
"customers",
|
||||
"businesses",
|
||||
"shipments",
|
||||
"receipts",
|
||||
)
|
||||
for entity in reader.inventory():
|
||||
assert (reader.query_dir / f"{entity}.sql").is_file()
|
||||
|
||||
|
||||
def test_T加高风险映射保持待确认() -> None:
|
||||
assert get_mapping("businesses").status is MappingStatus.PENDING
|
||||
assert "成本口径" in get_mapping("businesses").unresolved
|
||||
|
||||
|
||||
def test_未知T加实体被拒绝() -> None:
|
||||
with pytest.raises(KeyError):
|
||||
get_mapping("unknown")
|
||||
72
backend/tests/unit/test_tplus_connection_file.py
Normal file
72
backend/tests/unit/test_tplus_connection_file.py
Normal file
@ -0,0 +1,72 @@
|
||||
"""显式连接文件不应污染应用配置,也不能启用未授权的数据源。"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.tplus_db import resolve_tplus_settings
|
||||
|
||||
|
||||
def test_未开启连接时不读取配置文件(tmp_path):
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_connection_file=str(tmp_path / "missing.json"),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="未启用"):
|
||||
resolve_tplus_settings(settings)
|
||||
|
||||
|
||||
def test_文件连接只覆盖源字段且原Settings不变(tmp_path):
|
||||
path = tmp_path / "reader.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 1433,
|
||||
"database": "UFTData",
|
||||
"user": "reader",
|
||||
"password": "test-only-value",
|
||||
"driver": "SQL Server",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_connection_file=str(path),
|
||||
)
|
||||
resolved = resolve_tplus_settings(settings)
|
||||
assert resolved.t_plus_database == "UFTData"
|
||||
assert resolved.mysql_database == settings.mysql_database
|
||||
assert settings.t_plus_password == ""
|
||||
assert "test-only-value" not in str(resolved.t_plus_database_url)
|
||||
|
||||
|
||||
def test_连接文件不接受sa或应用配置覆盖(tmp_path):
|
||||
path = tmp_path / "bad.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 1433,
|
||||
"database": "UFTData",
|
||||
"user": "sa",
|
||||
"password": "test-only-value",
|
||||
"driver": "SQL Server",
|
||||
"app_secret_key": "bad",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_connection_file=str(path),
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
resolve_tplus_settings(settings)
|
||||
43
backend/tests/unit/test_tplus_query_service.py
Normal file
43
backend/tests/unit/test_tplus_query_service.py
Normal file
@ -0,0 +1,43 @@
|
||||
"""对外元数据必须过滤历史原异常和未知报告字段。"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusErrorQuery
|
||||
from app.services.tplus import query_service, task_service
|
||||
from app.services.tplus.relationship_audit import public_report
|
||||
|
||||
|
||||
def test_历史报告只开放白名单并保持兼容待验():
|
||||
result = public_report(
|
||||
{
|
||||
"source_version": "16.0",
|
||||
"compatibility_status": "PASSED",
|
||||
"password": "sensitive",
|
||||
"sales_links": {"row_count": 1, "payload": "secret"},
|
||||
"state_enums": [{"name": "password=hidden", "extra": "secret"}],
|
||||
}
|
||||
)
|
||||
assert result["compatibility_status"] == "PENDING"
|
||||
assert "password" not in result and "payload" not in result["sales_links"]
|
||||
assert "hidden" not in str(result) and "secret" not in str(result)
|
||||
|
||||
|
||||
def test_异常不返回手机号或原始错误文本(tplus_session):
|
||||
batch = task_service.create_task(
|
||||
tplus_session, TPlusCreateIn(entity_types=["personnel"], request_key=uuid4())
|
||||
)
|
||||
tplus_session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=batch.batch_no,
|
||||
entity_type="personnel",
|
||||
source_table="AA_Person",
|
||||
source_id="13800000000",
|
||||
error_code="READ_ERROR",
|
||||
error_message="password=private",
|
||||
)
|
||||
)
|
||||
tplus_session.commit()
|
||||
page = query_service.list_errors(tplus_session, batch.batch_no, TPlusErrorQuery())
|
||||
assert page.records[0].source_id is None
|
||||
assert "private" not in page.model_dump_json()
|
||||
97
backend/tests/unit/test_tplus_relationship_audit.py
Normal file
97
backend/tests/unit/test_tplus_relationship_audit.py
Normal file
@ -0,0 +1,97 @@
|
||||
"""关联核验必须使用真实类型字典,不能依靠恰好相同的跨表 ID。"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
|
||||
|
||||
|
||||
def _audit(types=None):
|
||||
source = Mock()
|
||||
rows = {
|
||||
"source_version": [{"source_version": "16.0.4165.4"}],
|
||||
"voucher_types": types
|
||||
if types is not None
|
||||
else [
|
||||
{"id": 9104, "code": "SA04", "name": "销货单"},
|
||||
{"id": 9092, "code": "ARAP01", "name": "收款单"},
|
||||
],
|
||||
"state_enums": [{"id": 189, "code": "01", "name": "生效", "enum_name": "VoucherState"}],
|
||||
"receipt_kinds": [
|
||||
{
|
||||
"voucher_type_code": "ARAP01",
|
||||
"voucher_type_name": "收款单",
|
||||
"receive_flag": 1,
|
||||
"row_count": 10,
|
||||
}
|
||||
],
|
||||
"receipt_line_kinds": [
|
||||
{"voucher_type_id": 9104, "voucher_type_name": "销货单", "row_count": 12}
|
||||
],
|
||||
"sales_links": [
|
||||
{
|
||||
"row_count": 12,
|
||||
"delivery_matches": 12,
|
||||
"detail_matches": 7,
|
||||
"direct_order_matches": 4,
|
||||
"invalid_order_ids": 0,
|
||||
}
|
||||
],
|
||||
"order_coverage": [
|
||||
{"row_count": 12, "without_order": 2, "single_order": 8, "multiple_orders": 2}
|
||||
],
|
||||
}
|
||||
audit = TPlusRelationshipAudit(source)
|
||||
audit.fetch = Mock(side_effect=lambda name, **kw: rows[name])
|
||||
return audit
|
||||
|
||||
|
||||
def test_类型标识从字典读取而不是硬编码104():
|
||||
audit = _audit()
|
||||
result = audit.run()
|
||||
args = next(call.kwargs for call in audit.fetch.call_args_list if call.args[0] == "sales_links")
|
||||
assert args == {"sale_delivery_type_id": 9104, "receive_flag": 1}
|
||||
assert result["sales_links"]["delivery_matches"] == 12
|
||||
|
||||
|
||||
def test_枚举名称按源库保留且2022不通过2008验收():
|
||||
result = _audit().run()
|
||||
assert result["state_enums"][0]["name"] == "生效"
|
||||
assert result["is_target_engine"] is False
|
||||
assert result["compatibility_status"] == "PENDING"
|
||||
|
||||
|
||||
def test_同类型字典有歧义就停止该链路统计不猜ID():
|
||||
audit = _audit(
|
||||
[
|
||||
{"id": 1, "code": "SA04", "name": "A"},
|
||||
{"id": 2, "code": "SA04", "name": "B"},
|
||||
{"id": 3, "code": "ARAP01", "name": "收款单"},
|
||||
]
|
||||
)
|
||||
result = audit.run()
|
||||
assert result["relationship_status"] == "UNRESOLVED"
|
||||
assert result["sales_links"] is None
|
||||
assert not any(call.args[0] == "sales_links" for call in audit.fetch.call_args_list)
|
||||
|
||||
|
||||
def test_多订单和成本日期仍需业务确认不自动选择():
|
||||
result = _audit().run()
|
||||
assert result["order_coverage"]["multiple_orders"] == 2
|
||||
keys = {item["key"] for item in result["pending_confirmations"]}
|
||||
assert {"cost_basis", "receipt_date", "ship_date", "allocation"} <= keys
|
||||
|
||||
|
||||
def test_SQL查询有固定白名单且不接受自定义表名():
|
||||
audit = TPlusRelationshipAudit(Mock())
|
||||
with pytest.raises(ValueError, match="核验查询"):
|
||||
audit.fetch("SELECT * FROM customers")
|
||||
|
||||
|
||||
def test_主表来源单据类型为空不妨碍按是否收款核验():
|
||||
# 字典明确 idvouchertype 是“来源单据类型”,不是当前收款单的类型。
|
||||
audit = _audit([{"id": 9104, "code": "SA04", "name": "销货单"}])
|
||||
assert audit.run()["relationship_status"] == "OBSERVED"
|
||||
assert "p.isReceiveFlag = :receive_flag" in audit.queries["sales_links"]
|
||||
assert "p.idvouchertype =" not in audit.queries["sales_links"]
|
||||
40
backend/tests/unit/test_tplus_sql_compatibility.py
Normal file
40
backend/tests/unit/test_tplus_sql_compatibility.py
Normal file
@ -0,0 +1,40 @@
|
||||
"""限制 T+ 源查询语法;真实 2008 R2 的驱动/引擎实测仍是最终验收条件。"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
QUERY_ROOT = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(QUERY_ROOT.glob("*.sql")), ids=lambda p: p.stem)
|
||||
def test_源查询只使用旧版本分页并保持只读(path):
|
||||
sql = re.sub(r"--[^\n]*", "", path.read_text(encoding="utf-8"))
|
||||
assert not re.search(
|
||||
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
|
||||
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
|
||||
sql,
|
||||
re.I,
|
||||
)
|
||||
assert "TOP (:batch_size)" in sql
|
||||
assert ":after_id" in sql
|
||||
assert not re.search(r"SELECT\s+\*", sql, re.I)
|
||||
|
||||
|
||||
def test_收付款主表没有更新时间时不虚构水位():
|
||||
sql = (QUERY_ROOT / "receipts.sql").read_text(encoding="utf-8")
|
||||
assert "CAST(NULL AS datetime) AS source_updated_at" in sql
|
||||
assert not re.search(r"h\.updated\b", sql, re.I)
|
||||
|
||||
|
||||
def test_关联核验保持只读且不使用新版函数():
|
||||
sql = re.sub(r"--[^\n]*", "", (QUERY_ROOT / "audit/queries.sql").read_text(encoding="utf-8"))
|
||||
assert not re.search(
|
||||
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
|
||||
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
|
||||
sql,
|
||||
re.I,
|
||||
)
|
||||
assert not re.search(r"SELECT\s+\*", sql, re.I)
|
||||
assert ":sale_delivery_type_id" in sql and ":receive_flag" in sql
|
||||
126
backend/tests/unit/test_tplus_staging.py
Normal file
126
backend/tests/unit/test_tplus_staging.py
Normal file
@ -0,0 +1,126 @@
|
||||
"""T+ staging 的幂等和原始行保护测试。"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import TPlusSyncRecord
|
||||
from app.services.tplus.staging import stage_row
|
||||
|
||||
|
||||
class _Result:
|
||||
def __init__(self, value: TPlusSyncRecord | None) -> None:
|
||||
self.value = value
|
||||
|
||||
def scalar_one_or_none(self) -> TPlusSyncRecord | None:
|
||||
return self.value
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, existing: TPlusSyncRecord | None = None) -> None:
|
||||
self.existing = existing
|
||||
self.added: list[object] = []
|
||||
self.flushed = False
|
||||
|
||||
def execute(self, _statement: object) -> _Result:
|
||||
return _Result(self.existing)
|
||||
|
||||
def add(self, value: object) -> None:
|
||||
self.added.append(value)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushed = True
|
||||
|
||||
|
||||
def _row() -> dict[str, object]:
|
||||
return {
|
||||
"id": 1001,
|
||||
"code": "SO-1001",
|
||||
"updated": datetime(2026, 10, 27, 10, 30),
|
||||
"amount": 12.5,
|
||||
}
|
||||
|
||||
|
||||
def test_stage_row_首次写入保留源编号更新时间和哈希() -> None:
|
||||
session = _Session()
|
||||
record, inserted = stage_row(
|
||||
session,
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row=_row(),
|
||||
)
|
||||
|
||||
assert inserted is True
|
||||
assert isinstance(session.added[0], TPlusSyncRecord)
|
||||
assert record.source_id == "1001"
|
||||
assert record.source_no == "SO-1001"
|
||||
assert record.source_updated_at == datetime(2026, 10, 27, 10, 30)
|
||||
assert len(record.row_hash) == 64
|
||||
assert session.flushed is True
|
||||
|
||||
|
||||
def test_stage_row_同一行重复执行返回已有记录() -> None:
|
||||
existing = TPlusSyncRecord(
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
source_id="1001",
|
||||
row_hash="x" * 64,
|
||||
payload_json={},
|
||||
)
|
||||
session = _Session(existing)
|
||||
record, inserted = stage_row(
|
||||
session,
|
||||
batch_no="TP20261027103000EFGH",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row=_row(),
|
||||
)
|
||||
|
||||
assert inserted is False
|
||||
assert record is existing
|
||||
assert session.added == []
|
||||
|
||||
|
||||
def test_stage_row_缺少源主键时拒绝写入() -> None:
|
||||
with pytest.raises(ValueError, match="缺少 id"):
|
||||
stage_row(
|
||||
_Session(),
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row={"code": "SO-1001"},
|
||||
)
|
||||
|
||||
|
||||
def test_stage_row_日期金额和二进制行版本能够实际序列化() -> None:
|
||||
row = {"id": 0, "code": "A", "updated": b"\x00\x01", "amount": Decimal("12.50")}
|
||||
record, _ = stage_row(
|
||||
_Session(), batch_no="TP1", entity_type="products", source_table="AA_Inventory", row=row
|
||||
)
|
||||
assert record.source_id == "0"
|
||||
assert record.source_updated_at is None
|
||||
assert record.payload_json["updated"] == "0x0001"
|
||||
assert record.payload_json["amount"] == "12.50"
|
||||
assert json.loads(json.dumps(record.payload_json)) == record.payload_json
|
||||
|
||||
|
||||
def test_stage_row_键顺序不影响原始行哈希() -> None:
|
||||
first, _ = stage_row(
|
||||
_Session(),
|
||||
batch_no="TP1",
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
row={"id": 1, "code": "A"},
|
||||
)
|
||||
second, _ = stage_row(
|
||||
_Session(),
|
||||
batch_no="TP2",
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
row={"code": "A", "id": 1},
|
||||
)
|
||||
assert first.row_hash == second.row_hash
|
||||
110
backend/tests/unit/test_tplus_sync.py
Normal file
110
backend/tests/unit/test_tplus_sync.py
Normal file
@ -0,0 +1,110 @@
|
||||
"""用真实事务验证 T+ staging 重跑、失败隔离和敏感信息保护。"""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, func, select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
|
||||
from app.services.tplus.sync_service import TPlusSyncService
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target(monkeypatch):
|
||||
"""以 SQLite 执行 staging 事务,MySQL 专属 DDL 仍由迁移测试覆盖。"""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
metadata = MetaData()
|
||||
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
|
||||
table = model.__table__.to_metadata(metadata)
|
||||
table.c.id.type = Integer()
|
||||
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
|
||||
metadata.create_all(engine)
|
||||
settings = Settings(_env_file=None, app_secret_key="x" * 32)
|
||||
monkeypatch.setattr("app.services.tplus.sync_service.get_settings", lambda: settings)
|
||||
monkeypatch.setattr("app.services.tplus.sync_service.sync_lock", lambda session: nullcontext())
|
||||
with Session(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _service(target, rows):
|
||||
service = TPlusSyncService(Mock(), target)
|
||||
service.reader = Mock()
|
||||
service.reader.iter_entity.side_effect = lambda *a, **kw: iter(rows)
|
||||
return service
|
||||
|
||||
|
||||
def test_重复全量读取不增加记录且业务表未被引用(target):
|
||||
rows = [{"id": 1, "code": "A"}, {"id": 2, "code": "B"}]
|
||||
service = _service(target, rows)
|
||||
first = service.run(entity_types=["products"])
|
||||
second = service.run(entity_types=["products"])
|
||||
assert first.status == second.status == "SUCCEEDED"
|
||||
assert first.staged_record_count == 2
|
||||
assert second.staged_record_count == 0
|
||||
assert second.source_record_count == 2
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
|
||||
assert second.watermark_to is None
|
||||
|
||||
|
||||
def test_单条缺失主键不会回滚其他有效行(target):
|
||||
service = _service(target, [{"id": 1, "code": "A"}, {"code": "bad"}, {"id": 2}])
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.staged_record_count == 2
|
||||
assert batch.failed_record_count == 1
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
|
||||
|
||||
|
||||
def test_数据库约束失败后仍可写入下一条(target):
|
||||
service = _service(target, [{"id": 1, "code": "A"}])
|
||||
# 用数据库真实触发器拒绝坏行,验证 executemany 失败后的回滚和逐条隔离。
|
||||
service.reader.iter_entity.side_effect = lambda *a, **kw: iter([{"id": 1}, {"id": 2}])
|
||||
target.execute(
|
||||
text(
|
||||
"CREATE TRIGGER reject_first BEFORE INSERT ON tplus_sync_records "
|
||||
"WHEN NEW.source_id='1' BEGIN SELECT RAISE(ABORT, 'invalid row'); END"
|
||||
)
|
||||
)
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.staged_record_count == 1
|
||||
assert target.scalar(select(TPlusSyncRecord.source_id)) == "2"
|
||||
|
||||
|
||||
def test_读取失败有审计且异常中的敏感参数不落日志(target):
|
||||
service = _service(target, [])
|
||||
service.reader.iter_entity.side_effect = RuntimeError("password=secret phone=13800000000")
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "FAILED"
|
||||
error = target.scalar(select(TPlusSyncError))
|
||||
assert error.error_code == "READ_ERROR"
|
||||
assert "secret" not in error.error_message
|
||||
assert "13800000000" not in error.error_message
|
||||
|
||||
|
||||
def test_成功一类后另一类读取失败应标记部分成功(target):
|
||||
service = _service(target, [])
|
||||
|
||||
def source(entity, **kwargs):
|
||||
if entity == "products":
|
||||
return iter([{"id": 1}])
|
||||
raise RuntimeError("读取中断")
|
||||
|
||||
service.reader.iter_entity.side_effect = source
|
||||
batch = service.run(entity_types=["products", "customers"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.source_record_count == batch.staged_record_count == 1
|
||||
assert batch.failed_record_count == 1
|
||||
|
||||
|
||||
def test_未确认时间增量不能创建误导性批次(target):
|
||||
from datetime import datetime
|
||||
|
||||
service = _service(target, [])
|
||||
with pytest.raises(ValueError, match="增量水位语义尚未确认"):
|
||||
service.run(entity_types=["receipts"], watermark=datetime(2026, 10, 27))
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 0
|
||||
21
backend/tests/unit/test_tplus_task_models.py
Normal file
21
backend/tests/unit/test_tplus_task_models.py
Normal file
@ -0,0 +1,21 @@
|
||||
"""持久化任务需要完整的幂等和重试证据,不能只有运行中的内存对象。"""
|
||||
|
||||
from app.models import TPlusSyncBatch
|
||||
|
||||
|
||||
def test_批次记录范围重试幂等与报告():
|
||||
table = TPlusSyncBatch.__table__
|
||||
assert {
|
||||
"entity_types_json",
|
||||
"skipped_record_count",
|
||||
"parent_batch_no",
|
||||
"request_key",
|
||||
"requested_by",
|
||||
"audit_result_json",
|
||||
} <= set(table.c.keys())
|
||||
assert table.c.started_at.nullable is True
|
||||
|
||||
|
||||
def test_请求键由数据库唯一约束保护():
|
||||
constraints = TPlusSyncBatch.__table__.constraints
|
||||
assert any(c.name == "uk_tplus_sync_batches_request_key" for c in constraints)
|
||||
93
backend/tests/unit/test_tplus_task_service.py
Normal file
93
backend/tests/unit/test_tplus_task_service.py
Normal file
@ -0,0 +1,93 @@
|
||||
"""同一意图不可重复排队,重试必须保留原始批次和范围。"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.exceptions import BadRequestError
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
|
||||
from app.services.tplus import task_service
|
||||
|
||||
|
||||
def _create(**kwargs):
|
||||
return TPlusCreateIn(request_key=uuid4(), entity_types=["products"], **kwargs)
|
||||
|
||||
|
||||
def test_同键同意图复用同一批次(tplus_session):
|
||||
request = _create()
|
||||
first = task_service.create_task(tplus_session, request)
|
||||
second = task_service.create_task(tplus_session, request)
|
||||
assert first.id == second.id
|
||||
assert first.status == "QUEUED"
|
||||
assert first.started_at is None
|
||||
assert first.entity_types_json == ["products"]
|
||||
assert tplus_session.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 1
|
||||
|
||||
|
||||
def test_同键不同范围拒绝(tplus_session):
|
||||
request = _create()
|
||||
task_service.create_task(tplus_session, request)
|
||||
changed = TPlusCreateIn(request_key=request.request_key, entity_types=["personnel"])
|
||||
with pytest.raises(BadRequestError, match="幂等"):
|
||||
task_service.create_task(tplus_session, changed)
|
||||
|
||||
|
||||
def test_失败重试创建新批次并保留原错误(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "FAILED"
|
||||
tplus_session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=parent.batch_no,
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
error_code="READ_ERROR",
|
||||
error_message="读取失败",
|
||||
)
|
||||
)
|
||||
tplus_session.commit()
|
||||
retry = TPlusRetryIn(request_key=uuid4())
|
||||
child = task_service.retry_task(tplus_session, parent.batch_no, retry)
|
||||
replay = task_service.retry_task(tplus_session, parent.batch_no, retry)
|
||||
assert child.id == replay.id and child.id != parent.id
|
||||
assert child.parent_batch_no == parent.batch_no
|
||||
assert parent.status == "FAILED"
|
||||
assert tplus_session.scalar(select(TPlusSyncError.retry_count)) == 1
|
||||
|
||||
|
||||
def test_已有未完成重试时不重复排队(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "FAILED"
|
||||
tplus_session.commit()
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
with pytest.raises(BadRequestError, match="重试任务"):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
|
||||
|
||||
def test_成功或未知历史范围不能自动重试(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "SUCCEEDED"
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
parent.status, parent.entity_types_json = "FAILED", None
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError, match="范围"):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
|
||||
|
||||
def test_关联核验不需要读取实体范围(tplus_session):
|
||||
batch = task_service.create_task(
|
||||
tplus_session, TPlusCreateIn(task_type="AUDIT", request_key=uuid4())
|
||||
)
|
||||
assert batch.sync_mode == "AUDIT"
|
||||
assert batch.entity_types_json == []
|
||||
|
||||
|
||||
def test_预留增量批次不得静默改为全量重试(tplus_session):
|
||||
batch = task_service.create_task(tplus_session, _create())
|
||||
batch.status, batch.sync_mode = "FAILED", "INCREMENTAL"
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError, match="方式"):
|
||||
task_service.retry_task(tplus_session, batch.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
100
backend/tests/unit/test_tplus_worker.py
Normal file
100
backend/tests/unit/test_tplus_worker.py
Normal file
@ -0,0 +1,100 @@
|
||||
"""排队任务要有实际执行者;忙锁、断连和进程中断不能被误记成成功。"""
|
||||
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from unittest.mock import Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.tplus_lock import TPlusBusyError
|
||||
from app.models import TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn
|
||||
from app.services.tplus import task_service, worker
|
||||
|
||||
|
||||
def _queued(session, audit=False):
|
||||
return task_service.create_task(
|
||||
session,
|
||||
TPlusCreateIn(
|
||||
request_key=uuid4(),
|
||||
task_type="AUDIT" if audit else "READ",
|
||||
entity_types=[] if audit else ["products"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _worker_setup(monkeypatch):
|
||||
monkeypatch.setattr(worker, "sync_lock", lambda session: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
"app.services.tplus.reader.TPlusReader.iter_entity",
|
||||
lambda *a, **kw: iter([{"id": 1, "code": "A"}]),
|
||||
)
|
||||
return lambda: nullcontext(Mock())
|
||||
|
||||
|
||||
def test_读取任务由worker处理并可观察重复数(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
first = _queued(tplus_session)
|
||||
assert worker.run_once(tplus_session, source_factory=source) == first.batch_no
|
||||
assert first.status == "SUCCEEDED" and first.started_at is not None
|
||||
second = _queued(tplus_session)
|
||||
worker.run_once(tplus_session, source_factory=source)
|
||||
assert second.staged_record_count == 0
|
||||
assert second.skipped_record_count == 1
|
||||
|
||||
|
||||
def test_忙锁时不领取任务也不恢复其他运行记录(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
|
||||
@contextmanager
|
||||
def busy(_session):
|
||||
raise TPlusBusyError("已有任务")
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(worker, "sync_lock", busy)
|
||||
assert worker.run_once(tplus_session, source_factory=source) is None
|
||||
assert batch.status == "QUEUED"
|
||||
|
||||
|
||||
def test_源连接失败写入固定错误不泄漏凭据(tplus_session, monkeypatch):
|
||||
_worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
|
||||
def unavailable():
|
||||
raise RuntimeError("password=private-secret")
|
||||
|
||||
worker.run_once(tplus_session, source_factory=unavailable)
|
||||
assert batch.status == "FAILED"
|
||||
error = tplus_session.scalar(select(TPlusSyncError))
|
||||
assert "private-secret" not in error.error_message
|
||||
assert error.error_code == "EXECUTION_ERROR"
|
||||
|
||||
|
||||
def test_取得锁后把遗留运行中任务记为中断失败(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
batch.status = "RUNNING"
|
||||
tplus_session.commit()
|
||||
assert worker.run_once(tplus_session, source_factory=source) is None
|
||||
assert batch.status == "FAILED"
|
||||
assert tplus_session.scalar(select(TPlusSyncError.error_code)) == "INTERRUPTED"
|
||||
|
||||
|
||||
def test_核验任务持久化报告但不修改业务数据(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session, audit=True)
|
||||
monkeypatch.setattr(
|
||||
worker.TPlusRelationshipAudit,
|
||||
"run",
|
||||
lambda self: {
|
||||
"relationship_status": "OBSERVED",
|
||||
"receipt_kinds": [{"row_count": 10}],
|
||||
"compatibility_status": "PENDING",
|
||||
"is_target_engine": False,
|
||||
},
|
||||
)
|
||||
worker.run_once(tplus_session, source_factory=source)
|
||||
assert batch.status == "SUCCEEDED"
|
||||
assert batch.audit_result_json["compatibility_status"] == "PENDING"
|
||||
assert batch.staged_record_count == 0
|
||||
792
backend/uv.lock
generated
792
backend/uv.lock
generated
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user