feat(tplus): 新增 T+ 只读同步服务与同步管理

This commit is contained in:
liuwenxin committed 2026-10-28 11:42:15 +08:00
1 parent 78bef43c06
commit 4fe5916c2c
43 files changed
+3309 -388

No files matched your search

+16 -1
View File
@@ -1,6 +1,21 @@
# 后端测试
> AI摘要:本项目后端的测试用例清单与运行记录。按「基础设施层随实现即时测、业务模块整模块完成时统一测」分层组织。当前 697 个用例全绿(2026-10-16,M3 增量批次后)。
> AI 摘要:本轮 T+ 管理已完成真实 MySQL、只读源库与浏览器验证。最终单元测试 602 项通过;管理/并发/结构定点 18 项通过。真正的 2008 R2 验收仍未执行。
## T+ 本轮核验(2026-10-27)
- 完整回归在独立 MySQL `payroll_tplus_validation_20261027`、Redis 15 库运行:984 passed、2 failed。两项失败均为配置单测预期默认 Redis 0 库,而进程继承了集成用的 `REDIS_DB=15`;没有修改默认配置或削弱断言。
- 后续将单元测试放在不设置上述集成覆盖值的独立进程中运行:602 passed。最终管理 API、真实并发和结构一致性定点测试:18 passed,包含新增加的配置关闭及创建/重试审计。
- 真实并发:相同请求只生成一个批次;不同键同时重试同父批次只有一个成功,无死锁;执行锁跨 ORM commit 仍有效,退出后可再获取。
- V15 在隔离库 downgrade 到 V14,再 upgrade 到 head 通过;当前 head `e5f607a8b9c0`。模型、Alembic、留存 SQL 和初始化脚本的结构检查通过;此前 V1–V14 留存 SQL 已从空库执行通过。
- 人员重读 847 条,新增 0、重复跳过 847、失败 0;关联报告覆盖 103,318 条销货类收款明细,50,044 条有明确订单关联,六类业务表行数不变。独立证据位于 D 盘 reports/management_verification_2026-10-27.json。
- Chrome 真实接口检查通过:排队、受控失败、异常查看、页面重试、人员去重、关联检查、筛选、普通用户拒绝访问,以及 320/768/1024/1440 宽度。无页面运行错误;临时账号和批次已清理。
- 加载、空态、配置关闭、概况失败另外使用浏览器响应模拟验证,不能与真实源链路混记。证据位于 .ai/archive/ai-aux/preview/tplus/。
- 320 像素主内容可操作,现有全局顶栏仍有文字挤压;未扩大本轮范围修改共享布局。宽表保留横向滚动,截图和限制见同步管理验收报告。
- 前端类型检查、生产构建和 Ruff 通过;构建保留既有的大分块提醒,测试有现存 Starlette TestClient 弃用提醒。
- 源查询保持参数化与旧版语法,时间增量和业务表写入均关闭。合法 Windows/SQL Server 2008 R2 安装介质仍未取得,不能把 2022 的结果标记为客户版本兼容。
单元测试与集成测试建议分独立终端运行:单元测试不设置集成覆盖值;集成测试使用隔离 MySQL 和 Redis。通用集成夹具会清理数据,不能指向开发或客户业务库。Windows 可设置 `PYTEST_DEBUG_TEMPROOT` 到项目 D 盘的 ai-aux/pytest-tmp,避免不同权限令牌共享系统临时目录。
## 一、测试约定
+30
View File
@@ -4,6 +4,36 @@ import pytest
from fastapi.testclient import TestClient
@pytest.fixture
def tplus_session(monkeypatch):
"""T+ 管理逻辑的真实内存事务;MySQL 并发锁另做集成验证。"""
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, text
from sqlalchemy.orm import Session
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
engine = create_engine("sqlite:///:memory:")
metadata = MetaData()
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
table = model.__table__.to_metadata(metadata)
table.c.id.type = Integer()
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
metadata.create_all(engine)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_database="unit_tplus",
t_plus_user="reader",
t_plus_odbc_driver="unit",
)
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
with Session(engine, expire_on_commit=False) as session:
yield session
engine.dispose()
@pytest.fixture(scope="session")
def client() -> TestClient:
"""构造应用测试客户端(会话级复用,避免重复构建应用)。
+33
View File
@@ -324,3 +324,36 @@ def normal(make_user, auth_headers) -> dict:
user = make_user([ROLE_USER])
user["headers"] = auth_headers(user)
return user
@pytest.fixture
def tplus_tasks(monkeypatch):
"""隔离测试任务,仅清理此夹具创建的批次和异常。"""
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_database="unit_tplus",
t_plus_user="reader",
t_plus_odbc_driver="unit",
)
monkeypatch.setattr("app.services.tplus.task_service.get_settings", lambda: settings)
keys = []
def payload():
key = str(uuid.uuid4())
keys.append(key)
return {"task_type": "READ", "entity_types": ["personnel"], "request_key": key}
yield payload
with get_session_factory()() as db:
numbers = list(
db.scalars(select(TPlusSyncBatch.batch_no).where(TPlusSyncBatch.request_key.in_(keys)))
)
if numbers:
db.execute(delete(TPlusSyncError).where(TPlusSyncError.batch_no.in_(numbers)))
db.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.batch_no.in_(numbers)))
db.commit()
+11 -1
View File
@@ -487,8 +487,18 @@ class TestManagerAggregate:
assert len([i for i in out["items"] if i["calc_type"] == 2]) == 1
assert len([i for i in out["items"] if i["calc_type"] == 1]) == 1
def test_月末取参_预约变更影响本月聚合(self, client: TestClient, admin: dict) -> None:
def test_月末取参_预约变更影响本月聚合(
self, client: TestClient, admin: dict, monkeypatch: pytest.MonkeyPatch
) -> None:
"""取参锚点=所属月月末:10-25 生效的预约变更应影响 2026-10 月结。"""
class ScenarioDate(date):
@classmethod
def today(cls):
return cls(2026, 10, 20)
# 固定预约场景的申请日,避免现实日期超过 10-25 后误测成“禁止追溯修改”。
monkeypatch.setattr("app.services.param_service.date", ScenarioDate)
resp = client.post(
"/api/v1/system/params/changes",
json={
@@ -0,0 +1,90 @@
"""在隔离 MySQL 中验证幂等竞争、父批次串行化和连接级锁。"""
from concurrent.futures import ThreadPoolExecutor
from threading import Barrier
from uuid import uuid4
import pytest
from sqlalchemy import delete, func, select, text
from app.core.db import get_session_factory
from app.core.exceptions import BadRequestError
from app.core.tplus_lock import TPlusBusyError, sync_lock
from app.models import TPlusSyncBatch
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services.tplus import task_service
pytestmark = pytest.mark.integration
def _parallel(action):
barrier = Barrier(2)
def run():
with get_session_factory()() as session:
barrier.wait(timeout=10)
return action(session)
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(run) for _ in range(2)]
return [future.result(timeout=20) for future in futures]
def test_并发相同请求只创建一个批次(tplus_tasks):
payload = TPlusCreateIn(**tplus_tasks())
numbers = _parallel(lambda session: task_service.create_task(session, payload).batch_no)
assert numbers[0] == numbers[1]
with get_session_factory()() as session:
assert (
session.scalar(
select(func.count())
.select_from(TPlusSyncBatch)
.where(TPlusSyncBatch.request_key == str(payload.request_key))
)
== 1
)
def test_并发不同请求重试同父批次只允许一个(tplus_tasks):
with get_session_factory()() as session:
parent = task_service.create_task(session, TPlusCreateIn(**tplus_tasks()))
parent.status = "FAILED"
session.commit()
number = parent.batch_no
def retry(session):
try:
return task_service.retry_task(
session, number, TPlusRetryIn(request_key=uuid4())
).batch_no
except BadRequestError:
session.rollback()
return "REJECTED"
try:
numbers = _parallel(retry)
assert numbers.count("REJECTED") == 1
with get_session_factory()() as session:
assert (
session.scalar(
select(func.count())
.select_from(TPlusSyncBatch)
.where(TPlusSyncBatch.parent_batch_no == number)
)
== 1
)
finally:
with get_session_factory()() as session:
session.execute(delete(TPlusSyncBatch).where(TPlusSyncBatch.parent_batch_no == number))
session.commit()
def test_执行锁跨提交保持且释放后可获取():
with get_session_factory()() as owner, get_session_factory()() as other:
with sync_lock(owner):
owner.execute(text("SELECT 1"))
owner.commit()
with pytest.raises(TPlusBusyError), sync_lock(other):
pytest.fail("另一执行者不应取得锁")
with sync_lock(other):
assert other.scalar(text("SELECT 1")) == 1
@@ -0,0 +1,129 @@
"""管理员可管理原始读取任务;权限、幂等和异常脱敏由真实接口验证。"""
import pytest
from sqlalchemy import select
from app.core.db import get_session_factory
from app.models import TPlusSyncBatch, TPlusSyncError
pytestmark = pytest.mark.integration
BASE = "/api/v1/system/tplus"
def test_未登录和普通用户不能访问管理(client, admin, normal, tplus_tasks):
assert client.get(BASE + "/overview").status_code == 401
assert client.get(BASE + "/overview", headers=normal["headers"]).status_code == 403
assert (
client.post(BASE + "/batches", json=tplus_tasks(), headers=normal["headers"]).status_code
== 403
)
def test_创建重复请求返回同一批次并可分页查看(client, admin, tplus_tasks):
payload = tplus_tasks()
first = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
assert first.status_code == 200
second = client.post(BASE + "/batches", json=payload, headers=admin["headers"])
assert second.status_code == 200
number = first.json()["data"]["batch_no"]
assert second.json()["data"]["batch_no"] == number
page = client.get(
BASE + "/batches", params={"pageSize": 1, "status": "QUEUED"}, headers=admin["headers"]
)
assert page.status_code == 200 and page.json()["data"]["pageSize"] == 1
detail = client.get(BASE + "/batches/" + number, headers=admin["headers"])
assert detail.status_code == 200
assert detail.json()["data"]["status"] == "QUEUED"
def test_异常返回固定说明不泄漏数据库参数(client, admin, tplus_tasks):
response = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"])
assert response.status_code == 200
number = response.json()["data"]["batch_no"]
with get_session_factory()() as db:
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
batch.status, batch.error_message = "FAILED", "password=private-value phone=13800000000"
db.add(
TPlusSyncError(
batch_no=number,
entity_type="personnel",
source_table="AA_Person",
source_id="12",
error_code="READ_ERROR",
error_message=batch.error_message,
)
)
db.commit()
for path in (f"/batches/{number}", f"/batches/{number}/errors"):
result = client.get(BASE + path, headers=admin["headers"])
assert result.status_code == 200
assert "private-value" not in result.text and "13800000000" not in result.text
def test_不支持的实体和不存在的批次有明确错误(client, admin, tplus_tasks):
payload = tplus_tasks()
payload["entity_types"] = ["users;drop"]
assert client.post(BASE + "/batches", json=payload, headers=admin["headers"]).status_code == 400
assert client.get(BASE + "/batches/TPmissing", headers=admin["headers"]).status_code == 404
def test_配置关闭时可查询历史但不能排队(client, admin, tplus_tasks, monkeypatch):
from app.services.tplus import task_service
settings = task_service.get_settings().model_copy(update={"t_plus_enabled": False})
monkeypatch.setattr(task_service, "get_settings", lambda: settings)
response = client.get(BASE + "/overview", headers=admin["headers"])
assert response.status_code == 200
assert response.json()["data"]["is_configured"] is False
assert response.json()["data"]["business_import_enabled"] is False
assert response.json()["data"]["compatibility_status"] == "PENDING"
assert (
client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).status_code
== 400
)
def test_失败批次重试保留异常并返回新批次(client, admin, tplus_tasks):
from app.models import OperationLog
parent = client.post(BASE + "/batches", json=tplus_tasks(), headers=admin["headers"]).json()[
"data"
]
number = parent["batch_no"]
with get_session_factory()() as db:
batch = db.scalar(select(TPlusSyncBatch).where(TPlusSyncBatch.batch_no == number))
batch.status, batch.failed_record_count = "FAILED", 1
db.add(
TPlusSyncError(
batch_no=number,
entity_type="personnel",
source_table="AA_Person",
source_id="1",
error_code="READ_ERROR",
error_message="读取失败",
)
)
db.commit()
request = {"request_key": tplus_tasks()["request_key"]}
first = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
replay = client.post(BASE + f"/batches/{number}/retry", json=request, headers=admin["headers"])
assert first.status_code == replay.status_code == 200
child = first.json()["data"]
assert child["parent_batch_no"] == number and child["batch_no"] != number
assert replay.json()["data"]["batch_no"] == child["batch_no"]
detail = client.get(BASE + f"/batches/{number}", headers=admin["headers"]).json()["data"]
assert detail["can_retry"] is False
with get_session_factory()() as db:
assert (
db.scalar(select(TPlusSyncError.retry_count).where(TPlusSyncError.batch_no == number))
== 1
)
assert db.scalar(
select(OperationLog.id)
.where(
OperationLog.user_id == admin["id"],
OperationLog.module == "T+数据同步",
OperationLog.action == "重试读取任务",
)
.limit(1)
)
+6 -1
View File
@@ -13,7 +13,12 @@ VALID_SECRET = "x" * 32
def _settings(**overrides: object) -> Settings:
"""构造一个不读取 .env 的配置实例。"""
params: dict[str, object] = {"_env_file": None, "app_secret_key": VALID_SECRET}
# _env_file=None 只禁用文件读取;显式值避免隔离测试库的环境变量影响默认值断言。
params: dict[str, object] = {
"_env_file": None,
"app_secret_key": VALID_SECRET,
"mysql_database": "payroll",
}
params.update(overrides)
return Settings(**params) # type: ignore[arg-type]
+4
View File
@@ -44,6 +44,10 @@ EXPECTED_TABLES = {
"monthly_settlements",
"settlement_items",
"adjustment_orders",
"tplus_sync_batches",
"tplus_sync_errors",
"tplus_sync_records",
"tplus_sync_watermarks",
}
# 单复数同形的集合名词:本身即复数语义,不受「以 s 结尾」检查约束
+58
View File
@@ -0,0 +1,58 @@
"""T+ 配置和读取适配层的纯单元测试。"""
from pathlib import Path
import pytest
from app.core.config import Settings
from app.services.tplus.mapping import MappingStatus, get_mapping
from app.services.tplus.reader import TPlusReader
def _settings(**overrides: object) -> Settings:
params: dict[str, object] = {"_env_file": None, "app_secret_key": "x" * 32}
params.update(overrides)
return Settings(**params) # type: ignore[arg-type]
def test_T加数据库URL使用mssql_pyodbc并隐藏密码() -> None:
settings = _settings(
t_plus_enabled=True,
t_plus_database="UFTData472115_000001",
t_plus_user="reader",
t_plus_password="p@ss:w/ord",
)
url = settings.t_plus_database_url
assert url.drivername == "mssql+pyodbc"
assert url.database == "UFTData472115_000001"
assert "p@ss:w/ord" not in url.render_as_string()
def test_T加未启用时读取器不建立连接() -> None:
settings = _settings(t_plus_enabled=False)
assert settings.t_plus_enabled is False
def test_T加查询文件覆盖六类实体() -> None:
reader = TPlusReader.__new__(TPlusReader)
reader.query_dir = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
assert reader.inventory() == (
"personnel",
"products",
"customers",
"businesses",
"shipments",
"receipts",
)
for entity in reader.inventory():
assert (reader.query_dir / f"{entity}.sql").is_file()
def test_T加高风险映射保持待确认() -> None:
assert get_mapping("businesses").status is MappingStatus.PENDING
assert "成本口径" in get_mapping("businesses").unresolved
def test_未知T加实体被拒绝() -> None:
with pytest.raises(KeyError):
get_mapping("unknown")
@@ -0,0 +1,72 @@
"""显式连接文件不应污染应用配置,也不能启用未授权的数据源。"""
import json
import pytest
from app.core.config import Settings
from app.core.tplus_db import resolve_tplus_settings
def test_未开启连接时不读取配置文件(tmp_path):
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_connection_file=str(tmp_path / "missing.json"),
)
with pytest.raises(RuntimeError, match="未启用"):
resolve_tplus_settings(settings)
def test_文件连接只覆盖源字段且原Settings不变(tmp_path):
path = tmp_path / "reader.json"
path.write_text(
json.dumps(
{
"host": "127.0.0.1",
"port": 1433,
"database": "UFTData",
"user": "reader",
"password": "test-only-value",
"driver": "SQL Server",
}
),
encoding="utf-8",
)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_connection_file=str(path),
)
resolved = resolve_tplus_settings(settings)
assert resolved.t_plus_database == "UFTData"
assert resolved.mysql_database == settings.mysql_database
assert settings.t_plus_password == ""
assert "test-only-value" not in str(resolved.t_plus_database_url)
def test_连接文件不接受sa或应用配置覆盖(tmp_path):
path = tmp_path / "bad.json"
path.write_text(
json.dumps(
{
"host": "127.0.0.1",
"port": 1433,
"database": "UFTData",
"user": "sa",
"password": "test-only-value",
"driver": "SQL Server",
"app_secret_key": "bad",
}
),
encoding="utf-8",
)
settings = Settings(
_env_file=None,
app_secret_key="x" * 32,
t_plus_enabled=True,
t_plus_connection_file=str(path),
)
with pytest.raises(RuntimeError):
resolve_tplus_settings(settings)
@@ -0,0 +1,43 @@
"""对外元数据必须过滤历史原异常和未知报告字段。"""
from uuid import uuid4
from app.models import TPlusSyncError
from app.schemas.tplus import TPlusCreateIn, TPlusErrorQuery
from app.services.tplus import query_service, task_service
from app.services.tplus.relationship_audit import public_report
def test_历史报告只开放白名单并保持兼容待验():
result = public_report(
{
"source_version": "16.0",
"compatibility_status": "PASSED",
"password": "sensitive",
"sales_links": {"row_count": 1, "payload": "secret"},
"state_enums": [{"name": "password=hidden", "extra": "secret"}],
}
)
assert result["compatibility_status"] == "PENDING"
assert "password" not in result and "payload" not in result["sales_links"]
assert "hidden" not in str(result) and "secret" not in str(result)
def test_异常不返回手机号或原始错误文本(tplus_session):
batch = task_service.create_task(
tplus_session, TPlusCreateIn(entity_types=["personnel"], request_key=uuid4())
)
tplus_session.add(
TPlusSyncError(
batch_no=batch.batch_no,
entity_type="personnel",
source_table="AA_Person",
source_id="13800000000",
error_code="READ_ERROR",
error_message="password=private",
)
)
tplus_session.commit()
page = query_service.list_errors(tplus_session, batch.batch_no, TPlusErrorQuery())
assert page.records[0].source_id is None
assert "private" not in page.model_dump_json()
@@ -0,0 +1,97 @@
"""关联核验必须使用真实类型字典,不能依靠恰好相同的跨表 ID。"""
from unittest.mock import Mock
import pytest
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
def _audit(types=None):
source = Mock()
rows = {
"source_version": [{"source_version": "16.0.4165.4"}],
"voucher_types": types
if types is not None
else [
{"id": 9104, "code": "SA04", "name": "销货单"},
{"id": 9092, "code": "ARAP01", "name": "收款单"},
],
"state_enums": [{"id": 189, "code": "01", "name": "生效", "enum_name": "VoucherState"}],
"receipt_kinds": [
{
"voucher_type_code": "ARAP01",
"voucher_type_name": "收款单",
"receive_flag": 1,
"row_count": 10,
}
],
"receipt_line_kinds": [
{"voucher_type_id": 9104, "voucher_type_name": "销货单", "row_count": 12}
],
"sales_links": [
{
"row_count": 12,
"delivery_matches": 12,
"detail_matches": 7,
"direct_order_matches": 4,
"invalid_order_ids": 0,
}
],
"order_coverage": [
{"row_count": 12, "without_order": 2, "single_order": 8, "multiple_orders": 2}
],
}
audit = TPlusRelationshipAudit(source)
audit.fetch = Mock(side_effect=lambda name, **kw: rows[name])
return audit
def test_类型标识从字典读取而不是硬编码104():
audit = _audit()
result = audit.run()
args = next(call.kwargs for call in audit.fetch.call_args_list if call.args[0] == "sales_links")
assert args == {"sale_delivery_type_id": 9104, "receive_flag": 1}
assert result["sales_links"]["delivery_matches"] == 12
def test_枚举名称按源库保留且2022不通过2008验收():
result = _audit().run()
assert result["state_enums"][0]["name"] == "生效"
assert result["is_target_engine"] is False
assert result["compatibility_status"] == "PENDING"
def test_同类型字典有歧义就停止该链路统计不猜ID():
audit = _audit(
[
{"id": 1, "code": "SA04", "name": "A"},
{"id": 2, "code": "SA04", "name": "B"},
{"id": 3, "code": "ARAP01", "name": "收款单"},
]
)
result = audit.run()
assert result["relationship_status"] == "UNRESOLVED"
assert result["sales_links"] is None
assert not any(call.args[0] == "sales_links" for call in audit.fetch.call_args_list)
def test_多订单和成本日期仍需业务确认不自动选择():
result = _audit().run()
assert result["order_coverage"]["multiple_orders"] == 2
keys = {item["key"] for item in result["pending_confirmations"]}
assert {"cost_basis", "receipt_date", "ship_date", "allocation"} <= keys
def test_SQL查询有固定白名单且不接受自定义表名():
audit = TPlusRelationshipAudit(Mock())
with pytest.raises(ValueError, match="核验查询"):
audit.fetch("SELECT * FROM customers")
def test_主表来源单据类型为空不妨碍按是否收款核验():
# 字典明确 idvouchertype 是“来源单据类型”,不是当前收款单的类型。
audit = _audit([{"id": 9104, "code": "SA04", "name": "销货单"}])
assert audit.run()["relationship_status"] == "OBSERVED"
assert "p.isReceiveFlag = :receive_flag" in audit.queries["sales_links"]
assert "p.idvouchertype =" not in audit.queries["sales_links"]
@@ -0,0 +1,40 @@
"""限制 T+ 源查询语法;真实 2008 R2 的驱动/引擎实测仍是最终验收条件。"""
import re
from pathlib import Path
import pytest
QUERY_ROOT = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
@pytest.mark.parametrize("path", sorted(QUERY_ROOT.glob("*.sql")), ids=lambda p: p.stem)
def test_源查询只使用旧版本分页并保持只读(path):
sql = re.sub(r"--[^\n]*", "", path.read_text(encoding="utf-8"))
assert not re.search(
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
sql,
re.I,
)
assert "TOP (:batch_size)" in sql
assert ":after_id" in sql
assert not re.search(r"SELECT\s+\*", sql, re.I)
def test_收付款主表没有更新时间时不虚构水位():
sql = (QUERY_ROOT / "receipts.sql").read_text(encoding="utf-8")
assert "CAST(NULL AS datetime) AS source_updated_at" in sql
assert not re.search(r"h\.updated\b", sql, re.I)
def test_关联核验保持只读且不使用新版函数():
sql = re.sub(r"--[^\n]*", "", (QUERY_ROOT / "audit/queries.sql").read_text(encoding="utf-8"))
assert not re.search(
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
sql,
re.I,
)
assert not re.search(r"SELECT\s+\*", sql, re.I)
assert ":sale_delivery_type_id" in sql and ":receive_flag" in sql
+126
View File
@@ -0,0 +1,126 @@
"""T+ staging 的幂等和原始行保护测试。"""
import json
from datetime import datetime
from decimal import Decimal
import pytest
from app.models import TPlusSyncRecord
from app.services.tplus.staging import stage_row
class _Result:
def __init__(self, value: TPlusSyncRecord | None) -> None:
self.value = value
def scalar_one_or_none(self) -> TPlusSyncRecord | None:
return self.value
class _Session:
def __init__(self, existing: TPlusSyncRecord | None = None) -> None:
self.existing = existing
self.added: list[object] = []
self.flushed = False
def execute(self, _statement: object) -> _Result:
return _Result(self.existing)
def add(self, value: object) -> None:
self.added.append(value)
def flush(self) -> None:
self.flushed = True
def _row() -> dict[str, object]:
return {
"id": 1001,
"code": "SO-1001",
"updated": datetime(2026, 10, 27, 10, 30),
"amount": 12.5,
}
def test_stage_row_首次写入保留源编号更新时间和哈希() -> None:
session = _Session()
record, inserted = stage_row(
session,
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
row=_row(),
)
assert inserted is True
assert isinstance(session.added[0], TPlusSyncRecord)
assert record.source_id == "1001"
assert record.source_no == "SO-1001"
assert record.source_updated_at == datetime(2026, 10, 27, 10, 30)
assert len(record.row_hash) == 64
assert session.flushed is True
def test_stage_row_同一行重复执行返回已有记录() -> None:
existing = TPlusSyncRecord(
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
source_id="1001",
row_hash="x" * 64,
payload_json={},
)
session = _Session(existing)
record, inserted = stage_row(
session,
batch_no="TP20261027103000EFGH",
entity_type="businesses",
source_table="SA_SaleOrder",
row=_row(),
)
assert inserted is False
assert record is existing
assert session.added == []
def test_stage_row_缺少源主键时拒绝写入() -> None:
with pytest.raises(ValueError, match="缺少 id"):
stage_row(
_Session(),
batch_no="TP20261027103000ABCD",
entity_type="businesses",
source_table="SA_SaleOrder",
row={"code": "SO-1001"},
)
def test_stage_row_日期金额和二进制行版本能够实际序列化() -> None:
row = {"id": 0, "code": "A", "updated": b"\x00\x01", "amount": Decimal("12.50")}
record, _ = stage_row(
_Session(), batch_no="TP1", entity_type="products", source_table="AA_Inventory", row=row
)
assert record.source_id == "0"
assert record.source_updated_at is None
assert record.payload_json["updated"] == "0x0001"
assert record.payload_json["amount"] == "12.50"
assert json.loads(json.dumps(record.payload_json)) == record.payload_json
def test_stage_row_键顺序不影响原始行哈希() -> None:
first, _ = stage_row(
_Session(),
batch_no="TP1",
entity_type="products",
source_table="AA_Inventory",
row={"id": 1, "code": "A"},
)
second, _ = stage_row(
_Session(),
batch_no="TP2",
entity_type="products",
source_table="AA_Inventory",
row={"code": "A", "id": 1},
)
assert first.row_hash == second.row_hash
+110
View File
@@ -0,0 +1,110 @@
"""用真实事务验证 T+ staging 重跑、失败隔离和敏感信息保护。"""
from contextlib import nullcontext
from unittest.mock import Mock
import pytest
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, func, select, text
from sqlalchemy.orm import Session
from app.core.config import Settings
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
from app.services.tplus.sync_service import TPlusSyncService
@pytest.fixture
def target(monkeypatch):
"""以 SQLite 执行 staging 事务,MySQL 专属 DDL 仍由迁移测试覆盖。"""
engine = create_engine("sqlite:///:memory:")
metadata = MetaData()
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
table = model.__table__.to_metadata(metadata)
table.c.id.type = Integer()
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
metadata.create_all(engine)
settings = Settings(_env_file=None, app_secret_key="x" * 32)
monkeypatch.setattr("app.services.tplus.sync_service.get_settings", lambda: settings)
monkeypatch.setattr("app.services.tplus.sync_service.sync_lock", lambda session: nullcontext())
with Session(engine, expire_on_commit=False) as session:
yield session
engine.dispose()
def _service(target, rows):
service = TPlusSyncService(Mock(), target)
service.reader = Mock()
service.reader.iter_entity.side_effect = lambda *a, **kw: iter(rows)
return service
def test_重复全量读取不增加记录且业务表未被引用(target):
rows = [{"id": 1, "code": "A"}, {"id": 2, "code": "B"}]
service = _service(target, rows)
first = service.run(entity_types=["products"])
second = service.run(entity_types=["products"])
assert first.status == second.status == "SUCCEEDED"
assert first.staged_record_count == 2
assert second.staged_record_count == 0
assert second.source_record_count == 2
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
assert second.watermark_to is None
def test_单条缺失主键不会回滚其他有效行(target):
service = _service(target, [{"id": 1, "code": "A"}, {"code": "bad"}, {"id": 2}])
batch = service.run(entity_types=["products"])
assert batch.status == "PARTIAL"
assert batch.staged_record_count == 2
assert batch.failed_record_count == 1
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
def test_数据库约束失败后仍可写入下一条(target):
service = _service(target, [{"id": 1, "code": "A"}])
# 用数据库真实触发器拒绝坏行,验证 executemany 失败后的回滚和逐条隔离。
service.reader.iter_entity.side_effect = lambda *a, **kw: iter([{"id": 1}, {"id": 2}])
target.execute(
text(
"CREATE TRIGGER reject_first BEFORE INSERT ON tplus_sync_records "
"WHEN NEW.source_id='1' BEGIN SELECT RAISE(ABORT, 'invalid row'); END"
)
)
batch = service.run(entity_types=["products"])
assert batch.status == "PARTIAL"
assert batch.staged_record_count == 1
assert target.scalar(select(TPlusSyncRecord.source_id)) == "2"
def test_读取失败有审计且异常中的敏感参数不落日志(target):
service = _service(target, [])
service.reader.iter_entity.side_effect = RuntimeError("password=secret phone=13800000000")
batch = service.run(entity_types=["products"])
assert batch.status == "FAILED"
error = target.scalar(select(TPlusSyncError))
assert error.error_code == "READ_ERROR"
assert "secret" not in error.error_message
assert "13800000000" not in error.error_message
def test_成功一类后另一类读取失败应标记部分成功(target):
service = _service(target, [])
def source(entity, **kwargs):
if entity == "products":
return iter([{"id": 1}])
raise RuntimeError("读取中断")
service.reader.iter_entity.side_effect = source
batch = service.run(entity_types=["products", "customers"])
assert batch.status == "PARTIAL"
assert batch.source_record_count == batch.staged_record_count == 1
assert batch.failed_record_count == 1
def test_未确认时间增量不能创建误导性批次(target):
from datetime import datetime
service = _service(target, [])
with pytest.raises(ValueError, match="增量水位语义尚未确认"):
service.run(entity_types=["receipts"], watermark=datetime(2026, 10, 27))
assert target.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 0
@@ -0,0 +1,21 @@
"""持久化任务需要完整的幂等和重试证据,不能只有运行中的内存对象。"""
from app.models import TPlusSyncBatch
def test_批次记录范围重试幂等与报告():
table = TPlusSyncBatch.__table__
assert {
"entity_types_json",
"skipped_record_count",
"parent_batch_no",
"request_key",
"requested_by",
"audit_result_json",
} <= set(table.c.keys())
assert table.c.started_at.nullable is True
def test_请求键由数据库唯一约束保护():
constraints = TPlusSyncBatch.__table__.constraints
assert any(c.name == "uk_tplus_sync_batches_request_key" for c in constraints)
@@ -0,0 +1,93 @@
"""同一意图不可重复排队,重试必须保留原始批次和范围。"""
from uuid import uuid4
import pytest
from sqlalchemy import func, select
from app.core.exceptions import BadRequestError
from app.models import TPlusSyncBatch, TPlusSyncError
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
from app.services.tplus import task_service
def _create(**kwargs):
return TPlusCreateIn(request_key=uuid4(), entity_types=["products"], **kwargs)
def test_同键同意图复用同一批次(tplus_session):
request = _create()
first = task_service.create_task(tplus_session, request)
second = task_service.create_task(tplus_session, request)
assert first.id == second.id
assert first.status == "QUEUED"
assert first.started_at is None
assert first.entity_types_json == ["products"]
assert tplus_session.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 1
def test_同键不同范围拒绝(tplus_session):
request = _create()
task_service.create_task(tplus_session, request)
changed = TPlusCreateIn(request_key=request.request_key, entity_types=["personnel"])
with pytest.raises(BadRequestError, match="幂等"):
task_service.create_task(tplus_session, changed)
def test_失败重试创建新批次并保留原错误(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "FAILED"
tplus_session.add(
TPlusSyncError(
batch_no=parent.batch_no,
entity_type="products",
source_table="AA_Inventory",
error_code="READ_ERROR",
error_message="读取失败",
)
)
tplus_session.commit()
retry = TPlusRetryIn(request_key=uuid4())
child = task_service.retry_task(tplus_session, parent.batch_no, retry)
replay = task_service.retry_task(tplus_session, parent.batch_no, retry)
assert child.id == replay.id and child.id != parent.id
assert child.parent_batch_no == parent.batch_no
assert parent.status == "FAILED"
assert tplus_session.scalar(select(TPlusSyncError.retry_count)) == 1
def test_已有未完成重试时不重复排队(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "FAILED"
tplus_session.commit()
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
with pytest.raises(BadRequestError, match="重试任务"):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
def test_成功或未知历史范围不能自动重试(tplus_session):
parent = task_service.create_task(tplus_session, _create())
parent.status = "SUCCEEDED"
tplus_session.commit()
with pytest.raises(BadRequestError):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
parent.status, parent.entity_types_json = "FAILED", None
tplus_session.commit()
with pytest.raises(BadRequestError, match="范围"):
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
def test_关联核验不需要读取实体范围(tplus_session):
batch = task_service.create_task(
tplus_session, TPlusCreateIn(task_type="AUDIT", request_key=uuid4())
)
assert batch.sync_mode == "AUDIT"
assert batch.entity_types_json == []
def test_预留增量批次不得静默改为全量重试(tplus_session):
batch = task_service.create_task(tplus_session, _create())
batch.status, batch.sync_mode = "FAILED", "INCREMENTAL"
tplus_session.commit()
with pytest.raises(BadRequestError, match="方式"):
task_service.retry_task(tplus_session, batch.batch_no, TPlusRetryIn(request_key=uuid4()))
+100
View File
@@ -0,0 +1,100 @@
"""排队任务要有实际执行者;忙锁、断连和进程中断不能被误记成成功。"""
from contextlib import contextmanager, nullcontext
from unittest.mock import Mock
from uuid import uuid4
from sqlalchemy import select
from app.core.tplus_lock import TPlusBusyError
from app.models import TPlusSyncError
from app.schemas.tplus import TPlusCreateIn
from app.services.tplus import task_service, worker
def _queued(session, audit=False):
return task_service.create_task(
session,
TPlusCreateIn(
request_key=uuid4(),
task_type="AUDIT" if audit else "READ",
entity_types=[] if audit else ["products"],
),
)
def _worker_setup(monkeypatch):
monkeypatch.setattr(worker, "sync_lock", lambda session: nullcontext())
monkeypatch.setattr(
"app.services.tplus.reader.TPlusReader.iter_entity",
lambda *a, **kw: iter([{"id": 1, "code": "A"}]),
)
return lambda: nullcontext(Mock())
def test_读取任务由worker处理并可观察重复数(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
first = _queued(tplus_session)
assert worker.run_once(tplus_session, source_factory=source) == first.batch_no
assert first.status == "SUCCEEDED" and first.started_at is not None
second = _queued(tplus_session)
worker.run_once(tplus_session, source_factory=source)
assert second.staged_record_count == 0
assert second.skipped_record_count == 1
def test_忙锁时不领取任务也不恢复其他运行记录(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session)
@contextmanager
def busy(_session):
raise TPlusBusyError("已有任务")
yield
monkeypatch.setattr(worker, "sync_lock", busy)
assert worker.run_once(tplus_session, source_factory=source) is None
assert batch.status == "QUEUED"
def test_源连接失败写入固定错误不泄漏凭据(tplus_session, monkeypatch):
_worker_setup(monkeypatch)
batch = _queued(tplus_session)
def unavailable():
raise RuntimeError("password=private-secret")
worker.run_once(tplus_session, source_factory=unavailable)
assert batch.status == "FAILED"
error = tplus_session.scalar(select(TPlusSyncError))
assert "private-secret" not in error.error_message
assert error.error_code == "EXECUTION_ERROR"
def test_取得锁后把遗留运行中任务记为中断失败(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session)
batch.status = "RUNNING"
tplus_session.commit()
assert worker.run_once(tplus_session, source_factory=source) is None
assert batch.status == "FAILED"
assert tplus_session.scalar(select(TPlusSyncError.error_code)) == "INTERRUPTED"
def test_核验任务持久化报告但不修改业务数据(tplus_session, monkeypatch):
source = _worker_setup(monkeypatch)
batch = _queued(tplus_session, audit=True)
monkeypatch.setattr(
worker.TPlusRelationshipAudit,
"run",
lambda self: {
"relationship_status": "OBSERVED",
"receipt_kinds": [{"row_count": 10}],
"compatibility_status": "PENDING",
"is_target_engine": False,
},
)
worker.run_once(tplus_session, source_factory=source)
assert batch.status == "SUCCEEDED"
assert batch.audit_result_json["compatibility_status"] == "PENDING"
assert batch.staged_record_count == 0