feat(tplus): 新增 T+ 只读同步服务与同步管理
This commit is contained in:
1 parent
78bef43c06
commit
4fe5916c2c
43 files changed
+3309
-388
No files matched your search
@@ -13,7 +13,12 @@ VALID_SECRET = "x" * 32
|
||||
|
||||
def _settings(**overrides: object) -> Settings:
|
||||
"""构造一个不读取 .env 的配置实例。"""
|
||||
params: dict[str, object] = {"_env_file": None, "app_secret_key": VALID_SECRET}
|
||||
# _env_file=None 只禁用文件读取;显式值避免隔离测试库的环境变量影响默认值断言。
|
||||
params: dict[str, object] = {
|
||||
"_env_file": None,
|
||||
"app_secret_key": VALID_SECRET,
|
||||
"mysql_database": "payroll",
|
||||
}
|
||||
params.update(overrides)
|
||||
return Settings(**params) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ EXPECTED_TABLES = {
|
||||
"monthly_settlements",
|
||||
"settlement_items",
|
||||
"adjustment_orders",
|
||||
"tplus_sync_batches",
|
||||
"tplus_sync_errors",
|
||||
"tplus_sync_records",
|
||||
"tplus_sync_watermarks",
|
||||
}
|
||||
|
||||
# 单复数同形的集合名词:本身即复数语义,不受「以 s 结尾」检查约束
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""T+ 配置和读取适配层的纯单元测试。"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.tplus.mapping import MappingStatus, get_mapping
|
||||
from app.services.tplus.reader import TPlusReader
|
||||
|
||||
|
||||
def _settings(**overrides: object) -> Settings:
|
||||
params: dict[str, object] = {"_env_file": None, "app_secret_key": "x" * 32}
|
||||
params.update(overrides)
|
||||
return Settings(**params) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_T加数据库URL使用mssql_pyodbc并隐藏密码() -> None:
|
||||
settings = _settings(
|
||||
t_plus_enabled=True,
|
||||
t_plus_database="UFTData472115_000001",
|
||||
t_plus_user="reader",
|
||||
t_plus_password="p@ss:w/ord",
|
||||
)
|
||||
url = settings.t_plus_database_url
|
||||
assert url.drivername == "mssql+pyodbc"
|
||||
assert url.database == "UFTData472115_000001"
|
||||
assert "p@ss:w/ord" not in url.render_as_string()
|
||||
|
||||
|
||||
def test_T加未启用时读取器不建立连接() -> None:
|
||||
settings = _settings(t_plus_enabled=False)
|
||||
assert settings.t_plus_enabled is False
|
||||
|
||||
|
||||
def test_T加查询文件覆盖六类实体() -> None:
|
||||
reader = TPlusReader.__new__(TPlusReader)
|
||||
reader.query_dir = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
|
||||
assert reader.inventory() == (
|
||||
"personnel",
|
||||
"products",
|
||||
"customers",
|
||||
"businesses",
|
||||
"shipments",
|
||||
"receipts",
|
||||
)
|
||||
for entity in reader.inventory():
|
||||
assert (reader.query_dir / f"{entity}.sql").is_file()
|
||||
|
||||
|
||||
def test_T加高风险映射保持待确认() -> None:
|
||||
assert get_mapping("businesses").status is MappingStatus.PENDING
|
||||
assert "成本口径" in get_mapping("businesses").unresolved
|
||||
|
||||
|
||||
def test_未知T加实体被拒绝() -> None:
|
||||
with pytest.raises(KeyError):
|
||||
get_mapping("unknown")
|
||||
@@ -0,0 +1,72 @@
|
||||
"""显式连接文件不应污染应用配置,也不能启用未授权的数据源。"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.tplus_db import resolve_tplus_settings
|
||||
|
||||
|
||||
def test_未开启连接时不读取配置文件(tmp_path):
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_connection_file=str(tmp_path / "missing.json"),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="未启用"):
|
||||
resolve_tplus_settings(settings)
|
||||
|
||||
|
||||
def test_文件连接只覆盖源字段且原Settings不变(tmp_path):
|
||||
path = tmp_path / "reader.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 1433,
|
||||
"database": "UFTData",
|
||||
"user": "reader",
|
||||
"password": "test-only-value",
|
||||
"driver": "SQL Server",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_connection_file=str(path),
|
||||
)
|
||||
resolved = resolve_tplus_settings(settings)
|
||||
assert resolved.t_plus_database == "UFTData"
|
||||
assert resolved.mysql_database == settings.mysql_database
|
||||
assert settings.t_plus_password == ""
|
||||
assert "test-only-value" not in str(resolved.t_plus_database_url)
|
||||
|
||||
|
||||
def test_连接文件不接受sa或应用配置覆盖(tmp_path):
|
||||
path = tmp_path / "bad.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 1433,
|
||||
"database": "UFTData",
|
||||
"user": "sa",
|
||||
"password": "test-only-value",
|
||||
"driver": "SQL Server",
|
||||
"app_secret_key": "bad",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
app_secret_key="x" * 32,
|
||||
t_plus_enabled=True,
|
||||
t_plus_connection_file=str(path),
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
resolve_tplus_settings(settings)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""对外元数据必须过滤历史原异常和未知报告字段。"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusErrorQuery
|
||||
from app.services.tplus import query_service, task_service
|
||||
from app.services.tplus.relationship_audit import public_report
|
||||
|
||||
|
||||
def test_历史报告只开放白名单并保持兼容待验():
|
||||
result = public_report(
|
||||
{
|
||||
"source_version": "16.0",
|
||||
"compatibility_status": "PASSED",
|
||||
"password": "sensitive",
|
||||
"sales_links": {"row_count": 1, "payload": "secret"},
|
||||
"state_enums": [{"name": "password=hidden", "extra": "secret"}],
|
||||
}
|
||||
)
|
||||
assert result["compatibility_status"] == "PENDING"
|
||||
assert "password" not in result and "payload" not in result["sales_links"]
|
||||
assert "hidden" not in str(result) and "secret" not in str(result)
|
||||
|
||||
|
||||
def test_异常不返回手机号或原始错误文本(tplus_session):
|
||||
batch = task_service.create_task(
|
||||
tplus_session, TPlusCreateIn(entity_types=["personnel"], request_key=uuid4())
|
||||
)
|
||||
tplus_session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=batch.batch_no,
|
||||
entity_type="personnel",
|
||||
source_table="AA_Person",
|
||||
source_id="13800000000",
|
||||
error_code="READ_ERROR",
|
||||
error_message="password=private",
|
||||
)
|
||||
)
|
||||
tplus_session.commit()
|
||||
page = query_service.list_errors(tplus_session, batch.batch_no, TPlusErrorQuery())
|
||||
assert page.records[0].source_id is None
|
||||
assert "private" not in page.model_dump_json()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""关联核验必须使用真实类型字典,不能依靠恰好相同的跨表 ID。"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.tplus.relationship_audit import TPlusRelationshipAudit
|
||||
|
||||
|
||||
def _audit(types=None):
|
||||
source = Mock()
|
||||
rows = {
|
||||
"source_version": [{"source_version": "16.0.4165.4"}],
|
||||
"voucher_types": types
|
||||
if types is not None
|
||||
else [
|
||||
{"id": 9104, "code": "SA04", "name": "销货单"},
|
||||
{"id": 9092, "code": "ARAP01", "name": "收款单"},
|
||||
],
|
||||
"state_enums": [{"id": 189, "code": "01", "name": "生效", "enum_name": "VoucherState"}],
|
||||
"receipt_kinds": [
|
||||
{
|
||||
"voucher_type_code": "ARAP01",
|
||||
"voucher_type_name": "收款单",
|
||||
"receive_flag": 1,
|
||||
"row_count": 10,
|
||||
}
|
||||
],
|
||||
"receipt_line_kinds": [
|
||||
{"voucher_type_id": 9104, "voucher_type_name": "销货单", "row_count": 12}
|
||||
],
|
||||
"sales_links": [
|
||||
{
|
||||
"row_count": 12,
|
||||
"delivery_matches": 12,
|
||||
"detail_matches": 7,
|
||||
"direct_order_matches": 4,
|
||||
"invalid_order_ids": 0,
|
||||
}
|
||||
],
|
||||
"order_coverage": [
|
||||
{"row_count": 12, "without_order": 2, "single_order": 8, "multiple_orders": 2}
|
||||
],
|
||||
}
|
||||
audit = TPlusRelationshipAudit(source)
|
||||
audit.fetch = Mock(side_effect=lambda name, **kw: rows[name])
|
||||
return audit
|
||||
|
||||
|
||||
def test_类型标识从字典读取而不是硬编码104():
|
||||
audit = _audit()
|
||||
result = audit.run()
|
||||
args = next(call.kwargs for call in audit.fetch.call_args_list if call.args[0] == "sales_links")
|
||||
assert args == {"sale_delivery_type_id": 9104, "receive_flag": 1}
|
||||
assert result["sales_links"]["delivery_matches"] == 12
|
||||
|
||||
|
||||
def test_枚举名称按源库保留且2022不通过2008验收():
|
||||
result = _audit().run()
|
||||
assert result["state_enums"][0]["name"] == "生效"
|
||||
assert result["is_target_engine"] is False
|
||||
assert result["compatibility_status"] == "PENDING"
|
||||
|
||||
|
||||
def test_同类型字典有歧义就停止该链路统计不猜ID():
|
||||
audit = _audit(
|
||||
[
|
||||
{"id": 1, "code": "SA04", "name": "A"},
|
||||
{"id": 2, "code": "SA04", "name": "B"},
|
||||
{"id": 3, "code": "ARAP01", "name": "收款单"},
|
||||
]
|
||||
)
|
||||
result = audit.run()
|
||||
assert result["relationship_status"] == "UNRESOLVED"
|
||||
assert result["sales_links"] is None
|
||||
assert not any(call.args[0] == "sales_links" for call in audit.fetch.call_args_list)
|
||||
|
||||
|
||||
def test_多订单和成本日期仍需业务确认不自动选择():
|
||||
result = _audit().run()
|
||||
assert result["order_coverage"]["multiple_orders"] == 2
|
||||
keys = {item["key"] for item in result["pending_confirmations"]}
|
||||
assert {"cost_basis", "receipt_date", "ship_date", "allocation"} <= keys
|
||||
|
||||
|
||||
def test_SQL查询有固定白名单且不接受自定义表名():
|
||||
audit = TPlusRelationshipAudit(Mock())
|
||||
with pytest.raises(ValueError, match="核验查询"):
|
||||
audit.fetch("SELECT * FROM customers")
|
||||
|
||||
|
||||
def test_主表来源单据类型为空不妨碍按是否收款核验():
|
||||
# 字典明确 idvouchertype 是“来源单据类型”,不是当前收款单的类型。
|
||||
audit = _audit([{"id": 9104, "code": "SA04", "name": "销货单"}])
|
||||
assert audit.run()["relationship_status"] == "OBSERVED"
|
||||
assert "p.isReceiveFlag = :receive_flag" in audit.queries["sales_links"]
|
||||
assert "p.idvouchertype =" not in audit.queries["sales_links"]
|
||||
@@ -0,0 +1,40 @@
|
||||
"""限制 T+ 源查询语法;真实 2008 R2 的驱动/引擎实测仍是最终验收条件。"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
QUERY_ROOT = Path(__file__).resolve().parents[3] / "db" / "queries" / "tplus"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(QUERY_ROOT.glob("*.sql")), ids=lambda p: p.stem)
|
||||
def test_源查询只使用旧版本分页并保持只读(path):
|
||||
sql = re.sub(r"--[^\n]*", "", path.read_text(encoding="utf-8"))
|
||||
assert not re.search(
|
||||
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
|
||||
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
|
||||
sql,
|
||||
re.I,
|
||||
)
|
||||
assert "TOP (:batch_size)" in sql
|
||||
assert ":after_id" in sql
|
||||
assert not re.search(r"SELECT\s+\*", sql, re.I)
|
||||
|
||||
|
||||
def test_收付款主表没有更新时间时不虚构水位():
|
||||
sql = (QUERY_ROOT / "receipts.sql").read_text(encoding="utf-8")
|
||||
assert "CAST(NULL AS datetime) AS source_updated_at" in sql
|
||||
assert not re.search(r"h\.updated\b", sql, re.I)
|
||||
|
||||
|
||||
def test_关联核验保持只读且不使用新版函数():
|
||||
sql = re.sub(r"--[^\n]*", "", (QUERY_ROOT / "audit/queries.sql").read_text(encoding="utf-8"))
|
||||
assert not re.search(
|
||||
r"\b(OFFSET|FETCH|OPENJSON|JSON_VALUE|STRING_AGG|TRY_CONVERT|TRY_CAST|"
|
||||
r"INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b",
|
||||
sql,
|
||||
re.I,
|
||||
)
|
||||
assert not re.search(r"SELECT\s+\*", sql, re.I)
|
||||
assert ":sale_delivery_type_id" in sql and ":receive_flag" in sql
|
||||
@@ -0,0 +1,126 @@
|
||||
"""T+ staging 的幂等和原始行保护测试。"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import TPlusSyncRecord
|
||||
from app.services.tplus.staging import stage_row
|
||||
|
||||
|
||||
class _Result:
|
||||
def __init__(self, value: TPlusSyncRecord | None) -> None:
|
||||
self.value = value
|
||||
|
||||
def scalar_one_or_none(self) -> TPlusSyncRecord | None:
|
||||
return self.value
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, existing: TPlusSyncRecord | None = None) -> None:
|
||||
self.existing = existing
|
||||
self.added: list[object] = []
|
||||
self.flushed = False
|
||||
|
||||
def execute(self, _statement: object) -> _Result:
|
||||
return _Result(self.existing)
|
||||
|
||||
def add(self, value: object) -> None:
|
||||
self.added.append(value)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushed = True
|
||||
|
||||
|
||||
def _row() -> dict[str, object]:
|
||||
return {
|
||||
"id": 1001,
|
||||
"code": "SO-1001",
|
||||
"updated": datetime(2026, 10, 27, 10, 30),
|
||||
"amount": 12.5,
|
||||
}
|
||||
|
||||
|
||||
def test_stage_row_首次写入保留源编号更新时间和哈希() -> None:
|
||||
session = _Session()
|
||||
record, inserted = stage_row(
|
||||
session,
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row=_row(),
|
||||
)
|
||||
|
||||
assert inserted is True
|
||||
assert isinstance(session.added[0], TPlusSyncRecord)
|
||||
assert record.source_id == "1001"
|
||||
assert record.source_no == "SO-1001"
|
||||
assert record.source_updated_at == datetime(2026, 10, 27, 10, 30)
|
||||
assert len(record.row_hash) == 64
|
||||
assert session.flushed is True
|
||||
|
||||
|
||||
def test_stage_row_同一行重复执行返回已有记录() -> None:
|
||||
existing = TPlusSyncRecord(
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
source_id="1001",
|
||||
row_hash="x" * 64,
|
||||
payload_json={},
|
||||
)
|
||||
session = _Session(existing)
|
||||
record, inserted = stage_row(
|
||||
session,
|
||||
batch_no="TP20261027103000EFGH",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row=_row(),
|
||||
)
|
||||
|
||||
assert inserted is False
|
||||
assert record is existing
|
||||
assert session.added == []
|
||||
|
||||
|
||||
def test_stage_row_缺少源主键时拒绝写入() -> None:
|
||||
with pytest.raises(ValueError, match="缺少 id"):
|
||||
stage_row(
|
||||
_Session(),
|
||||
batch_no="TP20261027103000ABCD",
|
||||
entity_type="businesses",
|
||||
source_table="SA_SaleOrder",
|
||||
row={"code": "SO-1001"},
|
||||
)
|
||||
|
||||
|
||||
def test_stage_row_日期金额和二进制行版本能够实际序列化() -> None:
|
||||
row = {"id": 0, "code": "A", "updated": b"\x00\x01", "amount": Decimal("12.50")}
|
||||
record, _ = stage_row(
|
||||
_Session(), batch_no="TP1", entity_type="products", source_table="AA_Inventory", row=row
|
||||
)
|
||||
assert record.source_id == "0"
|
||||
assert record.source_updated_at is None
|
||||
assert record.payload_json["updated"] == "0x0001"
|
||||
assert record.payload_json["amount"] == "12.50"
|
||||
assert json.loads(json.dumps(record.payload_json)) == record.payload_json
|
||||
|
||||
|
||||
def test_stage_row_键顺序不影响原始行哈希() -> None:
|
||||
first, _ = stage_row(
|
||||
_Session(),
|
||||
batch_no="TP1",
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
row={"id": 1, "code": "A"},
|
||||
)
|
||||
second, _ = stage_row(
|
||||
_Session(),
|
||||
batch_no="TP2",
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
row={"code": "A", "id": 1},
|
||||
)
|
||||
assert first.row_hash == second.row_hash
|
||||
@@ -0,0 +1,110 @@
|
||||
"""用真实事务验证 T+ staging 重跑、失败隔离和敏感信息保护。"""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import DefaultClause, Integer, MetaData, create_engine, func, select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord
|
||||
from app.services.tplus.sync_service import TPlusSyncService
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target(monkeypatch):
|
||||
"""以 SQLite 执行 staging 事务,MySQL 专属 DDL 仍由迁移测试覆盖。"""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
metadata = MetaData()
|
||||
for model in (TPlusSyncBatch, TPlusSyncError, TPlusSyncRecord):
|
||||
table = model.__table__.to_metadata(metadata)
|
||||
table.c.id.type = Integer()
|
||||
table.c.update_time.server_default = DefaultClause(text("CURRENT_TIMESTAMP"))
|
||||
metadata.create_all(engine)
|
||||
settings = Settings(_env_file=None, app_secret_key="x" * 32)
|
||||
monkeypatch.setattr("app.services.tplus.sync_service.get_settings", lambda: settings)
|
||||
monkeypatch.setattr("app.services.tplus.sync_service.sync_lock", lambda session: nullcontext())
|
||||
with Session(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _service(target, rows):
|
||||
service = TPlusSyncService(Mock(), target)
|
||||
service.reader = Mock()
|
||||
service.reader.iter_entity.side_effect = lambda *a, **kw: iter(rows)
|
||||
return service
|
||||
|
||||
|
||||
def test_重复全量读取不增加记录且业务表未被引用(target):
|
||||
rows = [{"id": 1, "code": "A"}, {"id": 2, "code": "B"}]
|
||||
service = _service(target, rows)
|
||||
first = service.run(entity_types=["products"])
|
||||
second = service.run(entity_types=["products"])
|
||||
assert first.status == second.status == "SUCCEEDED"
|
||||
assert first.staged_record_count == 2
|
||||
assert second.staged_record_count == 0
|
||||
assert second.source_record_count == 2
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
|
||||
assert second.watermark_to is None
|
||||
|
||||
|
||||
def test_单条缺失主键不会回滚其他有效行(target):
|
||||
service = _service(target, [{"id": 1, "code": "A"}, {"code": "bad"}, {"id": 2}])
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.staged_record_count == 2
|
||||
assert batch.failed_record_count == 1
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncRecord)) == 2
|
||||
|
||||
|
||||
def test_数据库约束失败后仍可写入下一条(target):
|
||||
service = _service(target, [{"id": 1, "code": "A"}])
|
||||
# 用数据库真实触发器拒绝坏行,验证 executemany 失败后的回滚和逐条隔离。
|
||||
service.reader.iter_entity.side_effect = lambda *a, **kw: iter([{"id": 1}, {"id": 2}])
|
||||
target.execute(
|
||||
text(
|
||||
"CREATE TRIGGER reject_first BEFORE INSERT ON tplus_sync_records "
|
||||
"WHEN NEW.source_id='1' BEGIN SELECT RAISE(ABORT, 'invalid row'); END"
|
||||
)
|
||||
)
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.staged_record_count == 1
|
||||
assert target.scalar(select(TPlusSyncRecord.source_id)) == "2"
|
||||
|
||||
|
||||
def test_读取失败有审计且异常中的敏感参数不落日志(target):
|
||||
service = _service(target, [])
|
||||
service.reader.iter_entity.side_effect = RuntimeError("password=secret phone=13800000000")
|
||||
batch = service.run(entity_types=["products"])
|
||||
assert batch.status == "FAILED"
|
||||
error = target.scalar(select(TPlusSyncError))
|
||||
assert error.error_code == "READ_ERROR"
|
||||
assert "secret" not in error.error_message
|
||||
assert "13800000000" not in error.error_message
|
||||
|
||||
|
||||
def test_成功一类后另一类读取失败应标记部分成功(target):
|
||||
service = _service(target, [])
|
||||
|
||||
def source(entity, **kwargs):
|
||||
if entity == "products":
|
||||
return iter([{"id": 1}])
|
||||
raise RuntimeError("读取中断")
|
||||
|
||||
service.reader.iter_entity.side_effect = source
|
||||
batch = service.run(entity_types=["products", "customers"])
|
||||
assert batch.status == "PARTIAL"
|
||||
assert batch.source_record_count == batch.staged_record_count == 1
|
||||
assert batch.failed_record_count == 1
|
||||
|
||||
|
||||
def test_未确认时间增量不能创建误导性批次(target):
|
||||
from datetime import datetime
|
||||
|
||||
service = _service(target, [])
|
||||
with pytest.raises(ValueError, match="增量水位语义尚未确认"):
|
||||
service.run(entity_types=["receipts"], watermark=datetime(2026, 10, 27))
|
||||
assert target.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 0
|
||||
@@ -0,0 +1,21 @@
|
||||
"""持久化任务需要完整的幂等和重试证据,不能只有运行中的内存对象。"""
|
||||
|
||||
from app.models import TPlusSyncBatch
|
||||
|
||||
|
||||
def test_批次记录范围重试幂等与报告():
|
||||
table = TPlusSyncBatch.__table__
|
||||
assert {
|
||||
"entity_types_json",
|
||||
"skipped_record_count",
|
||||
"parent_batch_no",
|
||||
"request_key",
|
||||
"requested_by",
|
||||
"audit_result_json",
|
||||
} <= set(table.c.keys())
|
||||
assert table.c.started_at.nullable is True
|
||||
|
||||
|
||||
def test_请求键由数据库唯一约束保护():
|
||||
constraints = TPlusSyncBatch.__table__.constraints
|
||||
assert any(c.name == "uk_tplus_sync_batches_request_key" for c in constraints)
|
||||
@@ -0,0 +1,93 @@
|
||||
"""同一意图不可重复排队,重试必须保留原始批次和范围。"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.exceptions import BadRequestError
|
||||
from app.models import TPlusSyncBatch, TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn, TPlusRetryIn
|
||||
from app.services.tplus import task_service
|
||||
|
||||
|
||||
def _create(**kwargs):
|
||||
return TPlusCreateIn(request_key=uuid4(), entity_types=["products"], **kwargs)
|
||||
|
||||
|
||||
def test_同键同意图复用同一批次(tplus_session):
|
||||
request = _create()
|
||||
first = task_service.create_task(tplus_session, request)
|
||||
second = task_service.create_task(tplus_session, request)
|
||||
assert first.id == second.id
|
||||
assert first.status == "QUEUED"
|
||||
assert first.started_at is None
|
||||
assert first.entity_types_json == ["products"]
|
||||
assert tplus_session.scalar(select(func.count()).select_from(TPlusSyncBatch)) == 1
|
||||
|
||||
|
||||
def test_同键不同范围拒绝(tplus_session):
|
||||
request = _create()
|
||||
task_service.create_task(tplus_session, request)
|
||||
changed = TPlusCreateIn(request_key=request.request_key, entity_types=["personnel"])
|
||||
with pytest.raises(BadRequestError, match="幂等"):
|
||||
task_service.create_task(tplus_session, changed)
|
||||
|
||||
|
||||
def test_失败重试创建新批次并保留原错误(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "FAILED"
|
||||
tplus_session.add(
|
||||
TPlusSyncError(
|
||||
batch_no=parent.batch_no,
|
||||
entity_type="products",
|
||||
source_table="AA_Inventory",
|
||||
error_code="READ_ERROR",
|
||||
error_message="读取失败",
|
||||
)
|
||||
)
|
||||
tplus_session.commit()
|
||||
retry = TPlusRetryIn(request_key=uuid4())
|
||||
child = task_service.retry_task(tplus_session, parent.batch_no, retry)
|
||||
replay = task_service.retry_task(tplus_session, parent.batch_no, retry)
|
||||
assert child.id == replay.id and child.id != parent.id
|
||||
assert child.parent_batch_no == parent.batch_no
|
||||
assert parent.status == "FAILED"
|
||||
assert tplus_session.scalar(select(TPlusSyncError.retry_count)) == 1
|
||||
|
||||
|
||||
def test_已有未完成重试时不重复排队(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "FAILED"
|
||||
tplus_session.commit()
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
with pytest.raises(BadRequestError, match="重试任务"):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
|
||||
|
||||
def test_成功或未知历史范围不能自动重试(tplus_session):
|
||||
parent = task_service.create_task(tplus_session, _create())
|
||||
parent.status = "SUCCEEDED"
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
parent.status, parent.entity_types_json = "FAILED", None
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError, match="范围"):
|
||||
task_service.retry_task(tplus_session, parent.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
|
||||
|
||||
def test_关联核验不需要读取实体范围(tplus_session):
|
||||
batch = task_service.create_task(
|
||||
tplus_session, TPlusCreateIn(task_type="AUDIT", request_key=uuid4())
|
||||
)
|
||||
assert batch.sync_mode == "AUDIT"
|
||||
assert batch.entity_types_json == []
|
||||
|
||||
|
||||
def test_预留增量批次不得静默改为全量重试(tplus_session):
|
||||
batch = task_service.create_task(tplus_session, _create())
|
||||
batch.status, batch.sync_mode = "FAILED", "INCREMENTAL"
|
||||
tplus_session.commit()
|
||||
with pytest.raises(BadRequestError, match="方式"):
|
||||
task_service.retry_task(tplus_session, batch.batch_no, TPlusRetryIn(request_key=uuid4()))
|
||||
@@ -0,0 +1,100 @@
|
||||
"""排队任务要有实际执行者;忙锁、断连和进程中断不能被误记成成功。"""
|
||||
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from unittest.mock import Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.tplus_lock import TPlusBusyError
|
||||
from app.models import TPlusSyncError
|
||||
from app.schemas.tplus import TPlusCreateIn
|
||||
from app.services.tplus import task_service, worker
|
||||
|
||||
|
||||
def _queued(session, audit=False):
|
||||
return task_service.create_task(
|
||||
session,
|
||||
TPlusCreateIn(
|
||||
request_key=uuid4(),
|
||||
task_type="AUDIT" if audit else "READ",
|
||||
entity_types=[] if audit else ["products"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _worker_setup(monkeypatch):
|
||||
monkeypatch.setattr(worker, "sync_lock", lambda session: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
"app.services.tplus.reader.TPlusReader.iter_entity",
|
||||
lambda *a, **kw: iter([{"id": 1, "code": "A"}]),
|
||||
)
|
||||
return lambda: nullcontext(Mock())
|
||||
|
||||
|
||||
def test_读取任务由worker处理并可观察重复数(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
first = _queued(tplus_session)
|
||||
assert worker.run_once(tplus_session, source_factory=source) == first.batch_no
|
||||
assert first.status == "SUCCEEDED" and first.started_at is not None
|
||||
second = _queued(tplus_session)
|
||||
worker.run_once(tplus_session, source_factory=source)
|
||||
assert second.staged_record_count == 0
|
||||
assert second.skipped_record_count == 1
|
||||
|
||||
|
||||
def test_忙锁时不领取任务也不恢复其他运行记录(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
|
||||
@contextmanager
|
||||
def busy(_session):
|
||||
raise TPlusBusyError("已有任务")
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(worker, "sync_lock", busy)
|
||||
assert worker.run_once(tplus_session, source_factory=source) is None
|
||||
assert batch.status == "QUEUED"
|
||||
|
||||
|
||||
def test_源连接失败写入固定错误不泄漏凭据(tplus_session, monkeypatch):
|
||||
_worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
|
||||
def unavailable():
|
||||
raise RuntimeError("password=private-secret")
|
||||
|
||||
worker.run_once(tplus_session, source_factory=unavailable)
|
||||
assert batch.status == "FAILED"
|
||||
error = tplus_session.scalar(select(TPlusSyncError))
|
||||
assert "private-secret" not in error.error_message
|
||||
assert error.error_code == "EXECUTION_ERROR"
|
||||
|
||||
|
||||
def test_取得锁后把遗留运行中任务记为中断失败(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session)
|
||||
batch.status = "RUNNING"
|
||||
tplus_session.commit()
|
||||
assert worker.run_once(tplus_session, source_factory=source) is None
|
||||
assert batch.status == "FAILED"
|
||||
assert tplus_session.scalar(select(TPlusSyncError.error_code)) == "INTERRUPTED"
|
||||
|
||||
|
||||
def test_核验任务持久化报告但不修改业务数据(tplus_session, monkeypatch):
|
||||
source = _worker_setup(monkeypatch)
|
||||
batch = _queued(tplus_session, audit=True)
|
||||
monkeypatch.setattr(
|
||||
worker.TPlusRelationshipAudit,
|
||||
"run",
|
||||
lambda self: {
|
||||
"relationship_status": "OBSERVED",
|
||||
"receipt_kinds": [{"row_count": 10}],
|
||||
"compatibility_status": "PENDING",
|
||||
"is_target_engine": False,
|
||||
},
|
||||
)
|
||||
worker.run_once(tplus_session, source_factory=source)
|
||||
assert batch.status == "SUCCEEDED"
|
||||
assert batch.audit_result_json["compatibility_status"] == "PENDING"
|
||||
assert batch.staged_record_count == 0
|
||||
Reference in new issue
Block a user