b8920c0d28
补充业务详情指导成本入口、来源版本和历史封存保护，接入月份锁并完善权限复核。 同步接入读取服务运行状态到页面，补充前端控制、回归测试和服务端只读连接授权校验。 定向隔离测试：89 项通过，1 条既有 Starlette 弃用警告。
710 lines
40 KiB
Python
710 lines
40 KiB
Python
"""将已发布的源单据接入业务；绝不以核算资料齐全作为建档条件。"""
|
||
|
||
from collections import Counter
|
||
from datetime import date, datetime
|
||
from decimal import Decimal, InvalidOperation
|
||
|
||
from sqlalchemy import String, and_, cast, func, null, or_, select
|
||
from sqlalchemy.exc import IntegrityError
|
||
|
||
from app.models import (
|
||
Business,
|
||
BusinessItem,
|
||
BusinessStatusLog,
|
||
Customer,
|
||
Personnel,
|
||
Product,
|
||
Receipt,
|
||
Shipment,
|
||
ShipmentItem,
|
||
TPlusBusinessDocument,
|
||
TPlusSourceHead,
|
||
TPlusSourceLink,
|
||
TPlusSyncRecord,
|
||
TPlusWorkLine,
|
||
)
|
||
from app.services.no_generator import next_no
|
||
from app.services.tplus.snapshots import source_manifest, source_statement
|
||
from app.services.tplus.workbench import WorkbenchError, digest, latest_facts
|
||
|
||
START = date(2026, 3, 1)
|
||
ENTITIES = ("businesses", "shipments", "receipts")
|
||
DETAILS = {"businesses": "business_items", "shipments": "shipment_items"}
|
||
SOURCE_FIELDS = {
|
||
"businesses": ("idcustomer", "voucherdate", "idclerk", "voucherState"),
|
||
"shipments": ("idcustomer", "voucherdate", "idclerk", "voucherState", "isCounteractDelivery"),
|
||
"receipts": ("idpartner", "idperson", "idclerk", "voucherDate", "amount", "isReceiveFlag",
|
||
"voucherstate", "cancelState", "accounting_voucher_count", "valid_accounting_voucher_count",
|
||
"accounting_voucher_id"),
|
||
}
|
||
|
||
|
||
def decimal_value(value, scale=2, *, positive=False):
|
||
"""拒绝精度丢失；源字符串末尾零不算超出精度。"""
|
||
try:
|
||
number = Decimal(str(value))
|
||
if not number.is_finite() or abs(number) >= Decimal(10) ** (14 - scale):
|
||
raise ValueError
|
||
if number != number.quantize(Decimal(1).scaleb(-scale)):
|
||
raise ValueError
|
||
if positive and number <= 0:
|
||
raise ValueError
|
||
return number
|
||
except (ValueError, InvalidOperation):
|
||
raise WorkbenchError("SOURCE_VALUE_INVALID") from None
|
||
|
||
|
||
def _source_personnel_id(raw):
|
||
person_id = raw.get("idclerk")
|
||
if person_id is None:
|
||
person_id = raw.get("idperson")
|
||
return str(person_id) if person_id is not None else None
|
||
|
||
|
||
def statement(session, source, entity, *, start=None):
|
||
query = source_statement(session, source, entity)
|
||
if start:
|
||
field = "voucherDate" if entity == "receipts" else "voucherdate"
|
||
raw_date = func.substr(func.nullif(TPlusSyncRecord.payload_json[field].as_string(), "null"), 1, 10)
|
||
projected = TPlusSourceHead.source_date
|
||
fallback = and_(projected.is_(None), or_(
|
||
raw_date.is_(None), raw_date >= start.isoformat(), ~raw_date.like("____-__-__"),
|
||
))
|
||
applied = select(TPlusBusinessDocument.id).where(
|
||
TPlusBusinessDocument.source_database == source,
|
||
TPlusBusinessDocument.entity_type == entity,
|
||
TPlusBusinessDocument.source_id == TPlusSyncRecord.source_id,
|
||
or_(TPlusBusinessDocument.applied_record_id.is_not(None),
|
||
TPlusBusinessDocument.business_id.is_not(None)),
|
||
).exists()
|
||
query = query.where(or_(projected >= start.isoformat(), ~projected.like("____-__-__"), fallback, applied))
|
||
return query
|
||
|
||
|
||
def preview(session, source, start=START):
|
||
"""只读统计当前完整快照；无需连接 SQL Server 或写业务表。"""
|
||
counts, candidates = {}, {}
|
||
for entity in ENTITIES:
|
||
query = statement(session, source, entity, start=start)
|
||
if entity == "receipts":
|
||
query = query.where(TPlusSyncRecord.payload_json["isReceiveFlag"].as_integer() == 1)
|
||
observed = query.with_only_columns(TPlusSyncRecord.source_id).subquery()
|
||
counts[entity] = session.scalar(select(func.count()).select_from(observed)) or 0
|
||
joined = observed.outerjoin(TPlusBusinessDocument, and_(
|
||
TPlusBusinessDocument.source_database == source,
|
||
TPlusBusinessDocument.entity_type == entity,
|
||
TPlusBusinessDocument.source_id == observed.c.source_id,
|
||
))
|
||
new_count = session.scalar(select(func.count()).select_from(joined).where(
|
||
TPlusBusinessDocument.id.is_(None))) or 0
|
||
candidates[entity] = {"new": new_count, "existing": counts[entity] - new_count}
|
||
conflicts = session.scalar(select(func.count()).select_from(TPlusBusinessDocument).where(
|
||
TPlusBusinessDocument.source_database == source, TPlusBusinessDocument.status == "CONFLICT")) or 0
|
||
customer_scope = preview_customer_scope(session, source, start)
|
||
product_scope = preview_product_scope(session, source, start)
|
||
return {"start_date": start.isoformat(), "documents": counts, "candidates": candidates, "existing_conflicts": conflicts,
|
||
"customer_scope": customer_scope, "product_scope": product_scope,
|
||
"formal_payroll_written": False, "message": "按客户自动建档；缺少人工资料只阻止核算"}
|
||
|
||
|
||
def preview_customer_scope(session, source, start):
|
||
"""只读预检客户身份；单据状态、产品及数据库排序规则仍须执行时核验。"""
|
||
customer_ids = set()
|
||
for entity in ENTITIES:
|
||
query = statement(session, source, entity, start=start)
|
||
if entity == "receipts":
|
||
query = query.where(TPlusSyncRecord.payload_json["isReceiveFlag"].as_integer() == 1)
|
||
field = "idpartner" if entity == "receipts" else "idcustomer"
|
||
for projected, raw in session.execute(query.with_only_columns(
|
||
TPlusSourceHead.source_customer_id, TPlusSyncRecord.payload_json[field].as_string()
|
||
)):
|
||
value = projected if projected is not None else raw
|
||
if value not in (None, "", "null"):
|
||
customer_ids.add(str(value))
|
||
facts = latest_facts(session, source)
|
||
records, customer_links, business_links = {}, {}, set()
|
||
ordered = sorted(customer_ids)
|
||
for offset in range(0, len(ordered), 500):
|
||
ids = ordered[offset:offset + 500]
|
||
records.update({row.source_id: row for row in session.scalars(
|
||
statement(session, source, "customers").where(TPlusSyncRecord.source_id.in_(ids))
|
||
)})
|
||
for link in session.scalars(select(TPlusSourceLink).where(
|
||
TPlusSourceLink.source_database == source,
|
||
or_(and_(TPlusSourceLink.entity_type == "customers", TPlusSourceLink.source_id.in_(ids)),
|
||
and_(TPlusSourceLink.entity_type == "businesses", TPlusSourceLink.source_id.in_(["customer:" + i for i in ids]))),
|
||
)):
|
||
if link.entity_type == "customers":
|
||
customer_links[link.source_id] = link.target_id
|
||
else:
|
||
business_links.add(link.source_id.removeprefix("customer:"))
|
||
grouped, problems = {}, []
|
||
for source_id in ordered:
|
||
if source_id in customer_links or source_id in business_links:
|
||
continue
|
||
record = records.get(source_id)
|
||
if record is None:
|
||
problems.append((source_id, "GROUP_SOURCE_MISSING"))
|
||
continue
|
||
name = str(record.payload_json.get("name") or "").strip()
|
||
if not name or len(name) > 128:
|
||
problems.append((source_id, "SOURCE_VALUE_INVALID"))
|
||
continue
|
||
fact = facts.get(("CUSTOMER", source_id))
|
||
owner = (fact.data_json if fact else {}).get("owner_name", "")
|
||
grouped.setdefault((name.casefold(), str(owner).casefold()), []).append((source_id, name, owner))
|
||
for values in grouped.values():
|
||
if len(values) > 1:
|
||
problems.extend((source_id, "DUPLICATE_SOURCE_IDENTITY") for source_id, _, _ in values)
|
||
# 仅查询候选名称；按数据库排序规则匹配后再以来源身份报告，不按名称合并。
|
||
names = sorted({item[1] for values in grouped.values() for item in values})
|
||
existing = set()
|
||
for offset in range(0, len(names), 500):
|
||
existing.update((str(name).casefold(), str(owner).casefold()) for name, owner in session.execute(
|
||
select(Customer.company_name, Customer.owner_name).where(Customer.company_name.in_(names[offset:offset + 500]))
|
||
))
|
||
for key, values in grouped.items():
|
||
if key in existing:
|
||
problems.extend((source_id, "TARGET_UNIQUE_CONFLICT") for source_id, _, _ in values)
|
||
problems.sort()
|
||
return {"candidate_customers": len(customer_ids), "without_business_link": len(customer_ids - business_links),
|
||
"identity_warnings": len(problems), "warning_counts": dict(Counter(reason for _, reason in problems)),
|
||
"examples": [{"source_id": source_id, "reason": reason} for source_id, reason in problems[:100]],
|
||
"truncated": len(problems) > 100,
|
||
"note": "客户范围为候选上限；同名预检不覆盖所有排序规则、产品冲突及单据有效性，执行时仍逐单核对。"}
|
||
|
||
|
||
def preview_product_scope(session, source, start):
|
||
"""基于候选单据查找涉及的产品，不读取或创建正式档案。"""
|
||
product_ids = set()
|
||
for entity, child_entity in DETAILS.items():
|
||
parents = list(session.scalars(statement(session, source, entity, start=start)
|
||
.with_only_columns(TPlusSyncRecord.source_id)))
|
||
for offset in range(0, len(parents), 500):
|
||
ids = parents[offset:offset + 500]
|
||
raw_parent = cast(TPlusSyncRecord.payload_json["source_parent_id"].as_string(), String)
|
||
query = statement(session, source, child_entity).where(or_(
|
||
TPlusSourceHead.source_parent_id.in_(ids),
|
||
and_(TPlusSourceHead.source_parent_id.is_(None), raw_parent.in_(ids)),
|
||
)).with_only_columns(TPlusSyncRecord.payload_json["idinventory"].as_string())
|
||
product_ids.update(str(value) for value in session.scalars(query) if value not in (None, "", "null"))
|
||
records, linked = {}, set()
|
||
ordered = sorted(product_ids)
|
||
for offset in range(0, len(ordered), 500):
|
||
ids = ordered[offset:offset + 500]
|
||
records.update({row.source_id: row for row in session.scalars(
|
||
statement(session, source, "products").where(TPlusSyncRecord.source_id.in_(ids))
|
||
)})
|
||
linked.update(session.scalars(select(TPlusSourceLink.source_id).where(
|
||
TPlusSourceLink.source_database == source, TPlusSourceLink.entity_type == "products",
|
||
TPlusSourceLink.source_id.in_(ids),
|
||
)))
|
||
grouped, problems = {}, []
|
||
for source_id in ordered:
|
||
if source_id in linked:
|
||
continue
|
||
record = records.get(source_id)
|
||
if record is None:
|
||
problems.append((source_id, "GROUP_SOURCE_MISSING"))
|
||
continue
|
||
raw = record.payload_json
|
||
name, spec, unit = str(raw.get("name") or "").strip(), str(raw.get("specification") or ""), raw.get("unit_name")
|
||
if not name or len(name) > 128 or len(spec) > 128 or (unit and len(unit) > 16):
|
||
problems.append((source_id, "SOURCE_VALUE_INVALID"))
|
||
continue
|
||
grouped.setdefault((name.casefold(), spec.casefold()), []).append((source_id, name, spec))
|
||
for values in grouped.values():
|
||
if len(values) > 1:
|
||
problems.extend((source_id, "DUPLICATE_SOURCE_IDENTITY") for source_id, _, _ in values)
|
||
names = sorted({item[1] for values in grouped.values() for item in values})
|
||
existing = set()
|
||
for offset in range(0, len(names), 500):
|
||
existing.update((str(name).casefold(), str(spec).casefold()) for name, spec in session.execute(
|
||
select(Product.product_name, Product.spec_model).where(Product.product_name.in_(names[offset:offset + 500]))
|
||
))
|
||
for key, values in grouped.items():
|
||
if key in existing:
|
||
problems.extend((source_id, "TARGET_UNIQUE_CONFLICT") for source_id, _, _ in values)
|
||
problems.sort()
|
||
return {"candidate_products": len(product_ids), "without_product_link": len(product_ids - linked),
|
||
"identity_warnings": len(problems), "warning_counts": dict(Counter(reason for _, reason in problems)),
|
||
"examples": [{"source_id": source_id, "reason": reason} for source_id, reason in problems[:100]],
|
||
"truncated": len(problems) > 100,
|
||
"note": "产品范围为候选上限；同名预检不覆盖所有排序规则或单据有效性，执行时仍逐单核对。"}
|
||
|
||
|
||
class Context:
|
||
def __init__(self, session, source, operator=None):
|
||
self.session, self.source = session, source
|
||
self.operator = operator
|
||
self.facts = latest_facts(session, source)
|
||
self.records, self.targets, self.recalculated = {}, {}, set()
|
||
self.people = {r.source_id: r for r in session.scalars(statement(session, source, "personnel"))}
|
||
|
||
def record(self, entity, source_id):
|
||
key = entity, str(source_id)
|
||
if key not in self.records:
|
||
self.records[key] = self.session.scalar(statement(self.session, self.source, entity).where(TPlusSyncRecord.source_id == str(source_id)))
|
||
if self.records[key] is None:
|
||
raise WorkbenchError("GROUP_SOURCE_MISSING")
|
||
return self.records[key]
|
||
|
||
def link(self, entity, source_id):
|
||
return self.session.scalar(select(TPlusSourceLink).where(
|
||
TPlusSourceLink.source_database == self.source, TPlusSourceLink.entity_type == entity,
|
||
TPlusSourceLink.source_id == str(source_id)))
|
||
|
||
def add_link(self, entity, source_id, target):
|
||
self.session.add(TPlusSourceLink(source_database=self.source, entity_type=entity,
|
||
source_id=str(source_id), target_id=target.id))
|
||
|
||
def master(self, entity, source_id):
|
||
key = entity, str(source_id)
|
||
if key in self.targets:
|
||
return self.targets[key]
|
||
model = Customer if entity == "customers" else Product
|
||
link = self.link(entity, source_id)
|
||
if link:
|
||
target = self.session.get(model, link.target_id)
|
||
if target is None:
|
||
raise WorkbenchError("TARGET_MISSING")
|
||
else:
|
||
record = self.record(entity, source_id)
|
||
raw = record.payload_json
|
||
name = str(raw.get("name") or "").strip()
|
||
if not name or len(name) > 128:
|
||
raise WorkbenchError("SOURCE_VALUE_INVALID")
|
||
if entity == "customers":
|
||
fact = self.facts.get(("CUSTOMER", str(source_id)))
|
||
values = fact.data_json if fact else {}
|
||
target = Customer(company_name=name, owner_name=values.get("owner_name", ""),
|
||
customer_type=values.get("customer_type"))
|
||
conflict = self.session.scalar(select(Customer.id).where(Customer.company_name == name, Customer.owner_name == target.owner_name).limit(1))
|
||
else:
|
||
fact = self.facts.get(("PRODUCT", str(source_id)))
|
||
values = fact.data_json if fact else {}
|
||
spec, unit = str(raw.get("specification") or ""), raw.get("unit_name")
|
||
if len(spec) > 128 or (unit and len(unit) > 16):
|
||
raise WorkbenchError("SOURCE_VALUE_INVALID")
|
||
target = Product(product_name=name, spec_model=spec, unit=unit,
|
||
product_category=values.get("category"), status=0 if raw.get("disabled") else 1)
|
||
conflict = self.session.scalar(select(Product.id).where(Product.product_name == name, Product.spec_model == spec).limit(1))
|
||
if conflict:
|
||
raise WorkbenchError("TARGET_UNIQUE_CONFLICT")
|
||
target.t_plus_id, target.source_no = str(source_id), record.source_no
|
||
target.synced_at = datetime.now()
|
||
self.session.add(target)
|
||
self.session.flush()
|
||
self.add_link(entity, source_id, target)
|
||
self.targets[key] = target
|
||
return target
|
||
|
||
def business(self, customer_id, day, run_no):
|
||
key = "customer:" + str(customer_id)
|
||
link = self.link("businesses", key)
|
||
if link:
|
||
business = self.session.get(Business, link.target_id, with_for_update=True)
|
||
if business is None:
|
||
raise WorkbenchError("TARGET_MISSING")
|
||
if business.integration_json is None:
|
||
# 既有完整核算导入不能被新接入器改写。
|
||
raise WorkbenchError("LEGACY_BUSINESS_REVIEW_REQUIRED")
|
||
return business
|
||
customer = self.master("customers", customer_id)
|
||
business = Business(business_no=next_no(self.session, model=Business, column_name="business_no", prefix="YW"),
|
||
customer_id=customer.id, business_date=day, status=1, is_special=0,
|
||
t_plus_id=key, source_no=key, sync_batch_no=run_no, synced_at=datetime.now(),
|
||
integration_json={"source_database": self.source, "customer_source_id": str(customer_id), "personnel_names": [], "personnel_codes": []},
|
||
total_cost_amount=null(), total_freight_amount=null())
|
||
self.session.add(business)
|
||
self.session.flush()
|
||
self.add_link("businesses", key, business)
|
||
self.session.add(BusinessStatusLog(business_id=business.id, old_status=1, new_status=1,
|
||
change_reason="T+业务自动建档", operator_name="T+业务接入"))
|
||
return business
|
||
|
||
def person(self, raw):
|
||
source_id = _source_personnel_id(raw)
|
||
person = self.people.get(source_id)
|
||
if not person:
|
||
return {"source_personnel_id": source_id, "source_personnel_name": None,
|
||
"source_personnel_no": None, "personnel_id": None}
|
||
code = str(person.payload_json.get("code") or "")
|
||
link = self.link("personnel", person.source_id)
|
||
target = self.session.get(Personnel, link.target_id) if link else None
|
||
return {"source_personnel_id": source_id, "source_personnel_name": person.payload_json.get("name"), "source_personnel_no": code,
|
||
"personnel_id": target.id if target else None}
|
||
|
||
|
||
def details(ctx, entity, source_id):
|
||
parent = TPlusSourceHead.source_parent_id
|
||
raw_parent = cast(TPlusSyncRecord.payload_json["source_parent_id"].as_string(), String)
|
||
return list(ctx.session.scalars(statement(ctx.session, ctx.source, DETAILS[entity]).where(or_(
|
||
parent == str(source_id), and_(parent.is_(None), raw_parent == str(source_id)),
|
||
)).order_by(TPlusSyncRecord.id)))
|
||
|
||
|
||
def rebuild_projections(session, source):
|
||
"""按已发布快照补齐旧 worker 未写入的索引，不改源记录。"""
|
||
entities = ("businesses", "business_items", "shipments", "shipment_items", "receipts")
|
||
queries = {entity: source_statement(session, source, entity) for entity in entities}
|
||
for entity, query in queries.items():
|
||
cursor = 0
|
||
while True:
|
||
missing = (TPlusSourceHead.source_parent_id.is_(None) if entity in DETAILS.values()
|
||
else TPlusSourceHead.source_date.is_(None))
|
||
rows = session.execute(query.with_only_columns(TPlusSourceHead, TPlusSyncRecord.payload_json)
|
||
.where(TPlusSourceHead.id > cursor, missing)
|
||
.order_by(TPlusSourceHead.id).limit(500).with_for_update()).all()
|
||
if not rows:
|
||
break
|
||
for head, raw in rows:
|
||
parent = raw.get("source_parent_id")
|
||
if parent is not None and len(str(parent)) <= 64:
|
||
head.source_parent_id = str(parent)
|
||
customer = raw.get("idcustomer") or raw.get("idpartner")
|
||
if head.source_customer_id is None and customer is not None and len(str(customer)) <= 64:
|
||
head.source_customer_id = str(customer)
|
||
day = raw.get("voucherdate") or raw.get("voucherDate")
|
||
if head.source_date is None and day is not None:
|
||
head.source_date = str(day)[:10]
|
||
cursor = rows[-1][0].id
|
||
session.commit()
|
||
|
||
|
||
def ledger(ctx, entity, record):
|
||
row = ctx.session.scalar(select(TPlusBusinessDocument).where(
|
||
TPlusBusinessDocument.source_database == ctx.source, TPlusBusinessDocument.entity_type == entity,
|
||
TPlusBusinessDocument.source_id == record.source_id).with_for_update())
|
||
if row is None:
|
||
row = TPlusBusinessDocument(source_database=ctx.source, entity_type=entity, source_id=record.source_id,
|
||
record_id=record.id, source_no=record.source_no, data_json={})
|
||
ctx.session.add(row)
|
||
row.record_id = record.id
|
||
return row
|
||
|
||
|
||
def _line_values(ctx, record):
|
||
raw = record.payload_json
|
||
product = ctx.master("products", raw.get("idinventory"))
|
||
quantity = decimal_value(raw.get("quantity"), positive=True)
|
||
source_price = raw.get("taxPrice") if raw.get("taxPrice") is not None else raw.get("price")
|
||
price = decimal_value(source_price, 4) if source_price is not None else None
|
||
amount = decimal_value(raw.get("taxAmount"), positive=True)
|
||
return {"product_id": product.id, "product_name": product.product_name, "spec_model": product.spec_model,
|
||
"unit": product.unit, "quantity": quantity, "sale_price": price}, amount, product.product_category
|
||
|
||
|
||
def _shipment_order_line(ctx, business_id, child):
|
||
"""来源明细号、订单号和产品均一致时，才将发货视为订单的一部分。"""
|
||
raw = child.payload_json
|
||
order_detail_id = raw.get("saleOrderDetailId")
|
||
if order_detail_id is None:
|
||
return None, False
|
||
doc = ctx.session.scalar(select(TPlusBusinessDocument).where(
|
||
TPlusBusinessDocument.source_database == ctx.source,
|
||
TPlusBusinessDocument.entity_type == "business_items",
|
||
TPlusBusinessDocument.source_id == str(order_detail_id),
|
||
TPlusBusinessDocument.business_id == business_id,
|
||
))
|
||
if doc is None:
|
||
return None, False
|
||
order_record = ctx.session.get(TPlusSyncRecord, doc.applied_record_id) if doc.applied_record_id else None
|
||
order_raw = order_record.payload_json if order_record else {}
|
||
parent = order_raw.get("source_parent_id")
|
||
order_id = raw.get("saleOrderId")
|
||
matches = (doc.status == "APPLIED" and order_record is not None
|
||
and order_raw.get("idinventory") is not None
|
||
and str(order_raw.get("idinventory")) == str(raw.get("idinventory"))
|
||
and (order_id is None or (parent is not None and str(parent) == str(order_id))))
|
||
return (doc if matches else None), not matches
|
||
|
||
|
||
def _recalculate(ctx, business):
|
||
_refresh_business_totals(ctx.session, business)
|
||
ctx.recalculated.add(business.id)
|
||
|
||
|
||
def _refresh_business_totals(session, business):
|
||
from app.services.tplus.sales_relations import resolve_items
|
||
|
||
items = list(session.scalars(select(BusinessItem).where(BusinessItem.business_id == business.id)))
|
||
order_ids = list(session.scalars(select(TPlusBusinessDocument.target_id).where(
|
||
TPlusBusinessDocument.business_id == business.id, TPlusBusinessDocument.entity_type == "business_items",
|
||
TPlusBusinessDocument.status == "APPLIED")))
|
||
ambiguous, sale_items = resolve_items(session, business, items, order_ids)
|
||
business.total_sale_amount = None if ambiguous else decimal_value(sum((i.sale_amount for i in sale_items), Decimal(0)))
|
||
business.total_cost_amount = decimal_value(sum((i.cost_amount for i in sale_items), Decimal(0))) if not ambiguous and sale_items and all(i.cost_amount is not None for i in sale_items) else None
|
||
business.integration_json = {**(business.integration_json or {}), "sales_relation_pending": ambiguous}
|
||
ships = list(session.scalars(select(Shipment).where(Shipment.business_id == business.id)))
|
||
business.total_ship_amount = decimal_value(sum((i.total_amount for i in ships), Decimal(0)))
|
||
business.total_freight_amount = decimal_value(sum((i.freight_amount for i in ships), Decimal(0))) if ships and all(i.freight_amount is not None for i in ships) else None
|
||
business.total_received_amount = decimal_value(session.scalar(select(func.coalesce(func.sum(Receipt.amount), 0)).where(Receipt.business_id == business.id)))
|
||
|
||
|
||
def _pending_document(ctx, doc, entity, raw, run_no, reason):
|
||
"""保留未生效或更正单，并关联可识别的客户业务供核对。"""
|
||
date_value = raw.get("voucherDate") if entity == "receipts" else raw.get("voucherdate")
|
||
customer_id = raw.get("idpartner") if entity == "receipts" else raw.get("idcustomer")
|
||
try:
|
||
day = date.fromisoformat(str(date_value)[:10])
|
||
except (ValueError, TypeError):
|
||
raise WorkbenchError("SOURCE_VALUE_INVALID") from None
|
||
if customer_id is None:
|
||
raise WorkbenchError("GROUP_CUSTOMER_MISMATCH")
|
||
business = ctx.business(customer_id, day, run_no)
|
||
doc.business_id = business.id
|
||
doc.data_json = {**ctx.person(raw), "source_date": day.isoformat(), "source_customer_id": str(customer_id)}
|
||
doc.status, doc.reason = "PENDING", reason
|
||
if business.id not in ctx.recalculated:
|
||
_recalculate(ctx, business)
|
||
return "PENDING", doc
|
||
|
||
|
||
def _apply_document(ctx, entity, record, children, run_no, start):
|
||
raw = record.payload_json
|
||
doc = ledger(ctx, entity, record)
|
||
customer_id = raw.get("idpartner") if entity == "receipts" else raw.get("idcustomer")
|
||
previous_customer = (doc.data_json or {}).get("source_customer_id")
|
||
if (doc.applied_record_id is None and doc.business_id is not None
|
||
and previous_customer is not None and customer_id is not None
|
||
and str(previous_customer) != str(customer_id)):
|
||
doc.status, doc.reason = "CONFLICT", "PENDING_SOURCE_CUSTOMER_CHANGED"
|
||
return "CONFLICT", doc
|
||
if doc.applied_record_id is None and doc.business_id is not None:
|
||
date_value = raw.get("voucherDate") if entity == "receipts" else raw.get("voucherdate")
|
||
try:
|
||
source_day = date.fromisoformat(str(date_value)[:10])
|
||
except (ValueError, TypeError):
|
||
source_day = None
|
||
if source_day is not None and source_day < start:
|
||
doc.status, doc.reason = "CONFLICT", "PENDING_SOURCE_OUT_OF_RANGE"
|
||
return "CONFLICT", doc
|
||
fingerprint = digest({"header": {k: raw.get(k) for k in SOURCE_FIELDS[entity]},
|
||
"items": [{k: r.payload_json.get(k) for k in ("id", "idinventory", "quantity", "taxPrice", "price", "taxAmount", "saleOrderId", "saleOrderDetailId")} for r in children]})
|
||
if doc.applied_record_id is not None:
|
||
person_changed = False
|
||
header_amount_changed = False
|
||
if entity in ("businesses", "shipments"):
|
||
if doc.applied_record_id != record.id:
|
||
original = ctx.session.get(TPlusSyncRecord, doc.applied_record_id)
|
||
header_amount_changed = original is None or original.payload_json.get("taxAmount") != raw.get("taxAmount")
|
||
if "source_personnel_id" in doc.data_json:
|
||
previous_person = doc.data_json["source_personnel_id"]
|
||
previous_person = str(previous_person) if previous_person is not None else None
|
||
else:
|
||
previous = ctx.session.get(TPlusSyncRecord, doc.applied_record_id)
|
||
previous_person = _source_personnel_id(previous.payload_json) if previous else None
|
||
person_changed = previous is None
|
||
person_changed = person_changed or previous_person != _source_personnel_id(raw)
|
||
if doc.fingerprint != fingerprint or doc.source_no != record.source_no or person_changed or header_amount_changed:
|
||
doc.status, doc.reason = "CONFLICT", "FORMAL_SOURCE_CHANGED"
|
||
return "CONFLICT", doc
|
||
if entity in DETAILS:
|
||
for child in children:
|
||
raw_line = child.payload_json
|
||
if raw_line.get("taxPrice") is not None or raw_line.get("price") is None:
|
||
continue
|
||
line_doc = ctx.session.scalar(select(TPlusBusinessDocument).where(
|
||
TPlusBusinessDocument.source_database == ctx.source,
|
||
TPlusBusinessDocument.entity_type == DETAILS[entity],
|
||
TPlusBusinessDocument.source_id == child.source_id,
|
||
TPlusBusinessDocument.business_id == doc.business_id,
|
||
))
|
||
model = BusinessItem if entity == "businesses" else ShipmentItem
|
||
target_line = ctx.session.get(model, line_doc.target_id) if line_doc and line_doc.target_id else None
|
||
try:
|
||
expected_price = decimal_value(raw_line["price"], 4)
|
||
except WorkbenchError:
|
||
expected_price = None
|
||
if target_line is None or expected_price is None or target_line.sale_price != expected_price:
|
||
doc.status, doc.reason = "CONFLICT", "SOURCE_PRICE_REVIEW_REQUIRED"
|
||
return "CONFLICT", doc
|
||
if entity == "shipments" and doc.business_id is not None:
|
||
for child in children:
|
||
_, mismatch = _shipment_order_line(ctx, doc.business_id, child)
|
||
if mismatch:
|
||
doc.status, doc.reason = "CONFLICT", "ORDER_SHIPMENT_MISMATCH"
|
||
return "CONFLICT", doc
|
||
doc.record_id, doc.status, doc.reason = record.id, "APPLIED", None
|
||
if doc.business_id is not None and doc.business_id not in ctx.recalculated:
|
||
business = ctx.session.get(Business, doc.business_id, with_for_update=True)
|
||
if business is not None:
|
||
_recalculate(ctx, business)
|
||
return "UNCHANGED", doc
|
||
if entity == "receipts" and raw.get("isReceiveFlag") != 1:
|
||
doc.status, doc.reason = "EXCLUDED", "NOT_RECEIPT"
|
||
return "EXCLUDED", doc
|
||
if (entity != "receipts" and raw.get("voucherState") != 189) or (entity == "receipts" and raw.get("voucherstate") != 189):
|
||
return _pending_document(ctx, doc, entity, raw, run_no, "SOURCE_DOCUMENT_INACTIVE")
|
||
if entity == "shipments" and raw.get("isCounteractDelivery"):
|
||
return _pending_document(ctx, doc, entity, raw, run_no, "CORRECTION_PAIR_REQUIRED")
|
||
day_text = raw.get("voucherDate") if entity == "receipts" else raw.get("voucherdate")
|
||
try:
|
||
day = date.fromisoformat(str(day_text)[:10])
|
||
except (ValueError, TypeError):
|
||
raise WorkbenchError("SOURCE_VALUE_INVALID") from None
|
||
if entity != "receipts" and not children:
|
||
raise WorkbenchError("SHIPMENT_DETAILS_MISSING")
|
||
# 先校验整张单据，避免不完整来源建出空壳业务。
|
||
order_mismatch = False
|
||
if entity == "receipts":
|
||
decimal_value(raw.get("amount"), positive=True)
|
||
else:
|
||
for child in children:
|
||
decimal_value(child.payload_json.get("quantity"), positive=True)
|
||
decimal_value(child.payload_json.get("taxAmount"), positive=True)
|
||
customer_id = raw.get("idpartner") if entity == "receipts" else raw.get("idcustomer")
|
||
if customer_id is None:
|
||
raise WorkbenchError("GROUP_CUSTOMER_MISMATCH")
|
||
# 新月份可追加，但已生成工资月份的迟到单据必须另行核对。
|
||
from app.models import MonthlySettlement
|
||
from app.services.payroll_months import lock_month_row
|
||
month = day.strftime("%Y-%m")
|
||
control = lock_month_row(ctx.session, month, ctx.operator)
|
||
locked = ctx.session.scalar(select(MonthlySettlement.id).where(
|
||
MonthlySettlement.settlement_month == month).limit(1).with_for_update())
|
||
if locked or (control and control.generated_at):
|
||
raise WorkbenchError("PAYROLL_MONTH_LOCKED")
|
||
business = ctx.business(customer_id, day, run_no)
|
||
if business.status != 1:
|
||
raise WorkbenchError("BUSINESS_TERMINAL_REVIEW_REQUIRED")
|
||
person = ctx.person(raw)
|
||
summary = dict(business.integration_json)
|
||
for key, value in (("personnel_names", person["source_personnel_name"]), ("personnel_codes", person["source_personnel_no"])):
|
||
summary[key] = sorted(set(summary.get(key, []) + ([value] if value else [])))
|
||
business.integration_json = summary
|
||
doc.business_id = business.id
|
||
doc.data_json = {**person, "source_date": day.isoformat(), "source_customer_id": str(customer_id)}
|
||
if entity == "receipts":
|
||
target = Receipt(business_id=business.id, receipt_batch_no=next_no(ctx.session, model=Receipt, column_name="receipt_batch_no", prefix="HK"),
|
||
receipt_date=day, amount=decimal_value(raw["amount"]), is_barter=0, settlement_status=0,
|
||
t_plus_id=record.source_id, source_no=record.source_no, sync_batch_no=run_no, synced_at=datetime.now())
|
||
ctx.session.add(target)
|
||
ctx.session.flush()
|
||
ctx.add_link(entity, record.source_id, target)
|
||
doc.target_id = target.id
|
||
else:
|
||
target = None
|
||
if entity == "shipments":
|
||
target = Shipment(business_id=business.id, shipment_no=next_no(ctx.session, model=Shipment, column_name="shipment_no", prefix="SH"),
|
||
ship_date=day, freight_amount=None, total_amount=decimal_value(sum((decimal_value(c.payload_json["taxAmount"]) for c in children), Decimal(0))),
|
||
t_plus_id=record.source_id, source_no=record.source_no, sync_batch_no=run_no, synced_at=datetime.now())
|
||
ctx.session.add(target)
|
||
ctx.session.flush()
|
||
ctx.add_link(entity, record.source_id, target)
|
||
doc.target_id = target.id if target else business.id
|
||
for child in children:
|
||
line, amount, category = _line_values(ctx, child)
|
||
child_doc = ledger(ctx, DETAILS[entity], child)
|
||
if entity == "businesses":
|
||
item = BusinessItem(business_id=business.id, **line, sale_amount=amount, product_category=category)
|
||
else:
|
||
item = ShipmentItem(shipment_id=target.id, **line, amount=amount)
|
||
order_line, mismatch = _shipment_order_line(ctx, business.id, child)
|
||
order_mismatch = order_mismatch or mismatch
|
||
if order_line is None:
|
||
ctx.session.add(BusinessItem(business_id=business.id, **line, sale_amount=amount, product_category=category))
|
||
ctx.session.add(item)
|
||
ctx.session.flush()
|
||
child_doc.business_id, child_doc.target_id = business.id, item.id
|
||
child_doc.applied_record_id, child_doc.record_id = child.id, child.id
|
||
child_doc.fingerprint, child_doc.status, child_doc.reason = child.row_hash, "APPLIED", None
|
||
child_doc.data_json = {**person, "source_parent_id": record.source_id, "source_date": day.isoformat(), "source_customer_id": str(customer_id)}
|
||
doc.applied_record_id, doc.fingerprint, doc.status, doc.reason = record.id, fingerprint, "APPLIED", None
|
||
ctx.session.flush()
|
||
_recalculate(ctx, business)
|
||
if order_mismatch:
|
||
doc.status, doc.reason = "CONFLICT", "ORDER_SHIPMENT_MISMATCH"
|
||
return "CONFLICT", doc
|
||
return "CREATE", doc
|
||
|
||
|
||
def execute_sync(session, run):
|
||
"""按单据提交；中断后幂等重跑，不生成 PayrollInput 或工资。"""
|
||
from app.models import User
|
||
from app.services.authorization_service import require_permission
|
||
operator = session.get(User, run.requested_by)
|
||
require_permission(session, operator, "tplus:import")
|
||
start = date.fromisoformat(run.input_json.get("start_date", START.isoformat()))
|
||
manifest = source_manifest(session, run.source_database)
|
||
rebuild_projections(session, run.source_database)
|
||
ctx, counts, processed = Context(session, run.source_database, operator), Counter(), 0
|
||
for entity in ENTITIES:
|
||
# 先固定所有依赖的发布状态；失败的全量采集不得泄露半批数据。
|
||
if entity in DETAILS:
|
||
statement(session, ctx.source, DETAILS[entity])
|
||
cursor = 0
|
||
while True:
|
||
records = list(session.scalars(statement(session, ctx.source, entity, start=start)
|
||
.where(TPlusSourceHead.id > cursor).order_by(TPlusSourceHead.id).limit(200)))
|
||
if not records:
|
||
break
|
||
# record.id 不是 source_head.id（哈希去重可复用旧版本）。
|
||
head_ids = dict(session.execute(select(TPlusSourceHead.record_id, TPlusSourceHead.id).where(
|
||
TPlusSourceHead.source_database == ctx.source, TPlusSourceHead.entity_type == entity,
|
||
TPlusSourceHead.record_id.in_([r.id for r in records]))).all())
|
||
for record in records:
|
||
try:
|
||
with session.begin_nested():
|
||
children = details(ctx, entity, record.source_id) if entity in DETAILS else []
|
||
action, doc = _apply_document(ctx, entity, record, children, run.run_no, start)
|
||
except (WorkbenchError, IntegrityError) as exc:
|
||
ctx.targets.clear()
|
||
ctx.recalculated.clear()
|
||
doc = ledger(ctx, entity, record)
|
||
status = "CONFLICT" if isinstance(exc, IntegrityError) else "PENDING"
|
||
reason = "TARGET_UNIQUE_CONFLICT" if isinstance(exc, IntegrityError) else str(exc)
|
||
if doc.applied_record_id is None:
|
||
try:
|
||
with session.begin_nested():
|
||
_pending_document(ctx, doc, entity, record.payload_json, run.run_no, reason)
|
||
except (WorkbenchError, IntegrityError):
|
||
ctx.targets.clear()
|
||
ctx.recalculated.clear()
|
||
doc = ledger(ctx, entity, record)
|
||
elif doc.business_id not in ctx.recalculated:
|
||
business = session.get(Business, doc.business_id, with_for_update=True)
|
||
if business is not None:
|
||
_recalculate(ctx, business)
|
||
doc.status, doc.reason = status, reason
|
||
action = doc.status
|
||
processed += 1
|
||
counts[action] += 1
|
||
session.add(TPlusWorkLine(run_no=run.run_no, line_no=processed, entity_type=entity,
|
||
source_id=record.source_id, action=action, reason=doc.reason,
|
||
data_json={"source_no": record.source_no, "business_id": doc.business_id, **(doc.data_json or {})}))
|
||
run.summary_json = {"counts": dict(counts), "total": processed, "formal_payroll_written": False,
|
||
"start_date": start.isoformat(), "phase": "RUNNING", "entity": entity}
|
||
session.commit()
|
||
cursor = head_ids[record.id]
|
||
observed = statement(session, ctx.source, entity).with_only_columns(TPlusSyncRecord.source_id).subquery()
|
||
missing = ~select(observed.c.source_id).where(observed.c.source_id == TPlusBusinessDocument.source_id).exists()
|
||
cursor = 0
|
||
while True:
|
||
documents = list(session.scalars(select(TPlusBusinessDocument).where(
|
||
TPlusBusinessDocument.source_database == ctx.source,
|
||
TPlusBusinessDocument.entity_type == entity,
|
||
or_(TPlusBusinessDocument.applied_record_id.is_not(None),
|
||
TPlusBusinessDocument.business_id.is_not(None)),
|
||
TPlusBusinessDocument.id > cursor,
|
||
missing,
|
||
).order_by(TPlusBusinessDocument.id).limit(200)))
|
||
if not documents:
|
||
break
|
||
for doc in documents:
|
||
doc.status, doc.reason = "CONFLICT", "SOURCE_DOCUMENT_MISSING"
|
||
processed += 1
|
||
counts["CONFLICT"] += 1
|
||
session.add(TPlusWorkLine(run_no=run.run_no, line_no=processed, entity_type=entity,
|
||
source_id=doc.source_id, action="CONFLICT", reason=doc.reason,
|
||
data_json={"source_no": doc.source_no, "business_id": doc.business_id, **(doc.data_json or {})}))
|
||
cursor = documents[-1].id
|
||
run.summary_json = {"counts": dict(counts), "total": processed, "formal_payroll_written": False,
|
||
"start_date": start.isoformat(), "phase": "RUNNING", "entity": entity}
|
||
session.commit()
|
||
if source_manifest(session, run.source_database) != manifest:
|
||
raise WorkbenchError("SOURCE_REFRESH_REQUIRED")
|
||
run.status = "PARTIAL" if counts["PENDING"] or counts["CONFLICT"] else "SUCCEEDED"
|
||
run.summary_json = {**(run.summary_json or {}), "phase": "COMPLETED", "source_manifest": manifest}
|