refactor(ingest): unique key (site,handover_no,waybill_no) + business_date purge + shortfall-sum undelivered

- schema: expected_record unique key (site,waybill_no) -> (site,handover_no,waybill_no),
  so the same waybill can belong to multiple handover batches without being overwritten
- store: dedup by (handover_no,waybill_no) instead of waybill_no alone; UPSERT conflict
  key matches the new unique key; handover_no no longer force-overwritten on conflict
- store: add purge_expected, called on force re-download; deletes by business_date
  (download day, same source as task date) rather than batch_out_date, which is offset
  from the download day for sites like yunda where download day != outbound day
- store/runtime: thread force + target_date through ingest_task / _persist_to_db /
  _site_undelivered_handler so force re-download purges then re-ingests in one tx
- db_compare: undelivered_pieces now sums per-waybill shortfall (total - arrived)
  instead of expected_total - arrived_total, unaffected by unrelated/extra scans
This commit is contained in:
Misaka
2026-08-09 20:37:29 +08:00
parent f29a9c6b77
commit 44c84983b4
4 changed files with 62 additions and 13 deletions

View File

@@ -506,7 +506,7 @@ def _site_undelivered_handler(site):
if store.ingest_enabled():
store.ingest_task(
site, "undelivered"
site, "undelivered", force=force, target_date=date
) # 4 站 = ingest expected + actual
print(f">> [入库] {site} 前置入库完成")
except Exception as e:
@@ -693,10 +693,11 @@ def _refresh_ready(site):
_apply_ready(site, flags, dates)
def _persist_to_db(site, kind):
def _persist_to_db(site, kind, force=False, target_date=None):
"""下载成功后把本次数据入库 PostgreSQL尽力而为绝不外抛不影响任务判定
- __compare__ 无源数据,跳过。
- auto_ingest=false 时跳过(无 PG/cpolar 的开发机)。
- force + target_date应到类先按 business_date 清空本次下载日再整批重入(防反复入库归属漂移)。
- 懒导入 store 以回避 import 顺序store↔compare 与 runtime↔compare 共存)。
- 结果写 state_store.ingest_state供 /api/status 反映入库健康。
所有写库/写状态都包 try/except失败仅告警绝不改变 dispatch_task 的 SUCCESS 判定。"""
@@ -711,7 +712,7 @@ def _persist_to_db(site, kind):
if not store.ingest_enabled(): # 移入 tryconfig.yaml 缺失/损坏时也不外抛
print(">> [入库] 已关闭 (auto_ingest=false),跳过")
return
count = store.ingest_task(site, kind)
count = store.ingest_task(site, kind, force=force, target_date=target_date)
# 4 站 undelivered 连带入了 expected+actual按实际入库的类补记 ingest_state
# 否则心跳派生 readyexpected ∧ actual → undelivered会读到陈旧值。
logged = (
@@ -755,7 +756,12 @@ def dispatch_task(ctx, task_spec):
if ret is False:
return (state_store.TASK_FAILED, "任务执行失败(重试耗尽)")
_record_business_date(site, kind, task_spec.get("date"))
_persist_to_db(site, kind)
_persist_to_db(
site,
kind,
force=bool(task_spec.get("force", False)),
target_date=task_spec.get("date"),
)
return (state_store.TASK_SUCCESS, None)
except Exception as e:
return (state_store.TASK_FAILED, str(e))