feat(tplus): 新增同步暂存与任务管理迁移和环境配置
This commit is contained in:
liuwenxin committed
1 parent
823d9f02ec
commit
78bef43c06
29 files changed
+1326
-5
No files matched your search
@@ -0,0 +1,122 @@
|
||||
"""T+ 只读同步批次、错误、staging 和增量水位。
|
||||
|
||||
Revision ID: d4e5f6a7b8c9
|
||||
Revises: c3a9e1d7b2f4
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.mysql import BIGINT
|
||||
|
||||
revision: str = "d4e5f6a7b8c9"
|
||||
down_revision: str | Sequence[str] | None = "c3a9e1d7b2f4"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""增加 T+ 源更新时间和同步审计/staging 表。"""
|
||||
for table in ("personnel", "products", "customers", "businesses", "shipments", "receipts"):
|
||||
op.execute(
|
||||
f"ALTER TABLE {table} ADD COLUMN source_updated_at DATETIME "
|
||||
"COMMENT 'T+ 源数据更新时间（未确认时可为空）' AFTER synced_at"
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"tplus_sync_batches",
|
||||
sa.Column("id", BIGINT(unsigned=True), autoincrement=True, nullable=False, comment="主键ID"),
|
||||
sa.Column("batch_no", sa.String(length=32), nullable=False, comment="同步批次号"),
|
||||
sa.Column("sync_mode", sa.String(length=16), nullable=False, comment="同步模式：FULL全量 INCREMENTAL增量"),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, comment="状态：RUNNING/SUCCEEDED/PARTIAL/FAILED"),
|
||||
sa.Column("source_database", sa.String(length=128), nullable=True, comment="T+ 源数据库名"),
|
||||
sa.Column("started_at", sa.DateTime(), nullable=False, comment="开始读取时间"),
|
||||
sa.Column("finished_at", sa.DateTime(), nullable=True, comment="结束时间"),
|
||||
sa.Column("watermark_from", sa.DateTime(), nullable=True, comment="增量起始水位"),
|
||||
sa.Column("watermark_to", sa.DateTime(), nullable=True, comment="本批次结束水位"),
|
||||
sa.Column("source_record_count", sa.Integer(), nullable=False, server_default="0", comment="源记录数"),
|
||||
sa.Column("staged_record_count", sa.Integer(), nullable=False, server_default="0", comment="写入 staging 的记录数"),
|
||||
sa.Column("failed_record_count", sa.Integer(), nullable=False, server_default="0", comment="失败记录数"),
|
||||
sa.Column("error_message", sa.Text(), nullable=True, comment="批次错误摘要"),
|
||||
sa.Column("create_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP"), comment="创建时间"),
|
||||
sa.Column("update_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"), comment="更新时间"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tplus_sync_batches"),
|
||||
sa.UniqueConstraint("batch_no", name="uk_tplus_sync_batches_batch_no"),
|
||||
mysql_engine="InnoDB", mysql_charset="utf8mb4", mysql_collate="utf8mb4_general_ci", comment="T+ 同步批次表（记录读取范围、结果和对账统计）",
|
||||
)
|
||||
op.create_index("idx_tplus_sync_batches_status", "tplus_sync_batches", ["status"])
|
||||
op.create_index("idx_tplus_sync_batches_started_at", "tplus_sync_batches", ["started_at"])
|
||||
|
||||
op.create_table(
|
||||
"tplus_sync_errors",
|
||||
sa.Column("id", BIGINT(unsigned=True), autoincrement=True, nullable=False, comment="主键ID"),
|
||||
sa.Column("batch_no", sa.String(length=32), nullable=False, comment="同步批次号"),
|
||||
sa.Column("entity_type", sa.String(length=32), nullable=False, comment="业务实体类型"),
|
||||
sa.Column("source_table", sa.String(length=128), nullable=False, comment="T+ 源表"),
|
||||
sa.Column("source_id", sa.String(length=64), nullable=True, comment="T+ 源主键"),
|
||||
sa.Column("source_line_no", sa.String(length=64), nullable=True, comment="T+ 明细行号或明细主键"),
|
||||
sa.Column("error_code", sa.String(length=32), nullable=False, comment="错误编码"),
|
||||
sa.Column("error_message", sa.Text(), nullable=False, comment="错误说明"),
|
||||
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0", comment="已重试次数"),
|
||||
sa.Column("create_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP"), comment="创建时间"),
|
||||
sa.Column("update_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"), comment="更新时间"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tplus_sync_errors"),
|
||||
mysql_engine="InnoDB", mysql_charset="utf8mb4", mysql_collate="utf8mb4_general_ci", comment="T+ 同步错误表（记录失败行和重试信息）",
|
||||
)
|
||||
op.create_index("idx_tplus_sync_errors_batch_no", "tplus_sync_errors", ["batch_no"])
|
||||
op.create_index("idx_tplus_sync_errors_entity_type", "tplus_sync_errors", ["entity_type"])
|
||||
|
||||
op.create_table(
|
||||
"tplus_sync_records",
|
||||
sa.Column("id", BIGINT(unsigned=True), autoincrement=True, nullable=False, comment="主键ID"),
|
||||
sa.Column("batch_no", sa.String(length=32), nullable=False, comment="同步批次号"),
|
||||
sa.Column("entity_type", sa.String(length=32), nullable=False, comment="业务实体类型"),
|
||||
sa.Column("source_table", sa.String(length=128), nullable=False, comment="T+ 源表"),
|
||||
sa.Column("source_id", sa.String(length=64), nullable=False, comment="T+ 源主键"),
|
||||
sa.Column("source_no", sa.String(length=64), nullable=True, comment="T+ 原始编号"),
|
||||
sa.Column("source_line_no", sa.String(length=64), nullable=True, comment="T+ 明细行号或明细主键"),
|
||||
sa.Column("source_updated_at", sa.DateTime(), nullable=True, comment="T+ 源数据更新时间"),
|
||||
sa.Column("row_hash", sa.String(length=64), nullable=False, comment="原始行 SHA-256"),
|
||||
sa.Column("state", sa.String(length=16), nullable=False, server_default="PENDING", comment="状态：PENDING/APPLIED/SKIPPED/ERROR"),
|
||||
sa.Column("payload_json", sa.JSON(), nullable=False, comment="原始行 JSON"),
|
||||
sa.Column("create_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP"), comment="创建时间"),
|
||||
sa.Column("update_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"), comment="更新时间"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tplus_sync_records"),
|
||||
sa.UniqueConstraint("entity_type", "source_id", "row_hash", name="uk_tplus_sync_records_entity_source_hash"),
|
||||
mysql_engine="InnoDB", mysql_charset="utf8mb4", mysql_collate="utf8mb4_general_ci", comment="T+ 原始同步 staging 表（映射确认前只保存原始行，不直接改业务表）",
|
||||
)
|
||||
op.create_index("idx_tplus_sync_records_batch_no", "tplus_sync_records", ["batch_no"])
|
||||
op.create_index("idx_tplus_sync_records_source_no", "tplus_sync_records", ["source_no"])
|
||||
op.create_index("idx_tplus_sync_records_state", "tplus_sync_records", ["state"])
|
||||
|
||||
op.create_table(
|
||||
"tplus_sync_watermarks",
|
||||
sa.Column("id", BIGINT(unsigned=True), autoincrement=True, nullable=False, comment="主键ID"),
|
||||
sa.Column("entity_type", sa.String(length=32), nullable=False, comment="业务实体类型"),
|
||||
sa.Column("last_source_updated_at", sa.DateTime(), nullable=True, comment="最近源更新时间"),
|
||||
sa.Column("last_source_id", sa.String(length=64), nullable=True, comment="同一时间点的最后源主键"),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, server_default="READY", comment="状态：READY/BLOCKED"),
|
||||
sa.Column("create_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP"), comment="创建时间"),
|
||||
sa.Column("update_time", sa.DateTime(), nullable=False, server_default=sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"), comment="更新时间"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tplus_sync_watermarks"),
|
||||
sa.UniqueConstraint("entity_type", name="uk_tplus_sync_watermarks_entity_type"),
|
||||
mysql_engine="InnoDB", mysql_charset="utf8mb4", mysql_collate="utf8mb4_general_ci", comment="T+ 增量同步水位表",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""删除 T+ 同步支撑表和源更新时间列。"""
|
||||
op.drop_table("tplus_sync_watermarks")
|
||||
op.drop_index("idx_tplus_sync_records_state", table_name="tplus_sync_records")
|
||||
op.drop_index("idx_tplus_sync_records_source_no", table_name="tplus_sync_records")
|
||||
op.drop_index("idx_tplus_sync_records_batch_no", table_name="tplus_sync_records")
|
||||
op.drop_table("tplus_sync_records")
|
||||
op.drop_index("idx_tplus_sync_errors_entity_type", table_name="tplus_sync_errors")
|
||||
op.drop_index("idx_tplus_sync_errors_batch_no", table_name="tplus_sync_errors")
|
||||
op.drop_table("tplus_sync_errors")
|
||||
op.drop_index("idx_tplus_sync_batches_started_at", table_name="tplus_sync_batches")
|
||||
op.drop_index("idx_tplus_sync_batches_status", table_name="tplus_sync_batches")
|
||||
op.drop_table("tplus_sync_batches")
|
||||
for table in ("receipts", "shipments", "businesses", "customers", "products", "personnel"):
|
||||
op.drop_column(table, "source_updated_at")
|
||||
@@ -0,0 +1,118 @@
|
||||
"""V15 T+ 持久化任务、幂等和重试元数据。"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.mysql import BIGINT
|
||||
|
||||
revision: str = "e5f607a8b9c0"
|
||||
down_revision: str | Sequence[str] | None = "d4e5f6a7b8c9"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""仅扩展管理元数据，不改原始暂存行或工资业务表。"""
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column("entity_types_json", sa.JSON(), comment="请求读取的实体类型；历史批次可为空"),
|
||||
)
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column(
|
||||
"skipped_record_count",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="0",
|
||||
comment="重复快照跳过条数",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column("parent_batch_no", sa.String(32), comment="重试来源批次号，保留原批次记录"),
|
||||
)
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column("request_key", sa.String(36), comment="请求幂等键，空值用于历史和直接CLI任务"),
|
||||
)
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column(
|
||||
"requested_by", BIGINT(unsigned=True), comment="请求用户ID，CLI任务为空，无物理外键"
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"tplus_sync_batches",
|
||||
sa.Column("audit_result_json", sa.JSON(), comment="关联核验结果，只含系统字典和汇总"),
|
||||
)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"started_at",
|
||||
existing_type=sa.DateTime(),
|
||||
nullable=True,
|
||||
comment="开始执行时间（排队时为空）",
|
||||
)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"sync_mode",
|
||||
existing_type=sa.String(16),
|
||||
nullable=False,
|
||||
comment="任务方式：FULL全量 AUDIT关联核验 INCREMENTAL预留",
|
||||
)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"status",
|
||||
existing_type=sa.String(16),
|
||||
nullable=False,
|
||||
comment="状态：QUEUED/RUNNING/SUCCEEDED/PARTIAL/FAILED",
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
"uk_tplus_sync_batches_request_key", "tplus_sync_batches", ["request_key"]
|
||||
)
|
||||
op.create_index(
|
||||
"idx_tplus_sync_batches_parent_status", "tplus_sync_batches", ["parent_batch_no", "status"]
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE tplus_sync_batches SET skipped_record_count=GREATEST(0, source_record_count-staged_record_count-failed_record_count) WHERE status='SUCCEEDED' AND sync_mode='FULL'"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""回滚前关闭 worker；原始暂存数据和历史批次仍保留。"""
|
||||
op.execute(
|
||||
"UPDATE tplus_sync_batches SET started_at=create_time, status='FAILED', finished_at=CURRENT_TIMESTAMP, error_message='任务管理版本已回滚' WHERE status='QUEUED'"
|
||||
)
|
||||
op.execute("UPDATE tplus_sync_batches SET started_at=create_time WHERE started_at IS NULL")
|
||||
op.drop_index("idx_tplus_sync_batches_parent_status", table_name="tplus_sync_batches")
|
||||
op.drop_constraint("uk_tplus_sync_batches_request_key", "tplus_sync_batches", type_="unique")
|
||||
for name in (
|
||||
"audit_result_json",
|
||||
"requested_by",
|
||||
"request_key",
|
||||
"parent_batch_no",
|
||||
"skipped_record_count",
|
||||
"entity_types_json",
|
||||
):
|
||||
op.drop_column("tplus_sync_batches", name)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"started_at",
|
||||
existing_type=sa.DateTime(),
|
||||
nullable=False,
|
||||
comment="开始读取时间",
|
||||
)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"sync_mode",
|
||||
existing_type=sa.String(16),
|
||||
nullable=False,
|
||||
comment="同步模式：FULL全量 INCREMENTAL增量",
|
||||
)
|
||||
op.alter_column(
|
||||
"tplus_sync_batches",
|
||||
"status",
|
||||
existing_type=sa.String(16),
|
||||
nullable=False,
|
||||
comment="状态：RUNNING/SUCCEEDED/PARTIAL/FAILED",
|
||||
)
|
||||
Reference in new issue
Block a user