feat(health): 被动式健康检查 API（FastAPI，覆盖服务健康+数据健康）

在同步进程内内置 FastAPI/uvicorn HTTP endpoint (GET /health)，返回结构化健康快照，便于运维/监控被动探活： - 服务健康：cycle_lag（核心指标，超过 3×poll_interval 判 unhealthy）、队列状态（pending/error/dead/cleaned 实时查）、capture 累计（enqueued/ deferred/aged_out）、最近 cycle 时间/耗时/错误。 - 数据健康：解析最近一次定时 compare_ids 报告（不跑全量 compare 太重）、实时查 SyncLogArchive 异常痕迹（AgedOut/降级）、SyncQueue dead 行样本。 status 三态：healthy(200) / degraded(503, 有dead/error/aged_out/compare 不一致) / unhealthy(503, 主循环停滞)。状态码映射便于按码告警。线程模型：uvicorn 占主线程，capture/apply/cleanup 循环跑后台 daemon 线程， NSSM 停服务时主线程退出、daemon 自动终止。health.enabled=false 时退化为旧行为（同步循环占主线程）。Web 层(routes/)与业务逻辑(HealthChecker) 解耦，后期加运维接口（compare触发/死信管理/metrics）零结构改动。零 SQL/零 schema 改动。新增依赖 fastapi/uvicorn[standard]。
2026-08-05 12:13:14 +08:00
parent 7cef5153e6
commit 03d7bcea39
13 changed files with 1081 additions and 15 deletions
--- a/tests/test_health.py
+++ b/tests/test_health.py
@@ -0,0 +1,183 @@
+"""Tests for the health-check layer: ServiceState, HealthChecker status logic,
+and the /health endpoint via FastAPI TestClient."""
+import datetime as _dt
+from unittest.mock import MagicMock, patch
+
+from fastapi.testclient import TestClient
+
+from sync.config import (
+    AccessConfig, FileMapping, HealthConfig, RuntimeConfig,
+    SqlServerConfig, SyncConfig,
+)
+from sync.health import CYCLE_LAG_MULTIPLIER, HealthChecker, ServiceState
+from sync.web.app import create_app
+
+
+def _cfg(**health_kw):
+    return SyncConfig(
+        sql_server=SqlServerConfig(conn_str="x"),
+        access=AccessConfig(driver="d", roots={"2026": "r"}),
+        runtime=RuntimeConfig(poll_interval_seconds=10),
+        files=[FileMapping(file="f.accdb", root="2026", schema="s", year_suffix="_Y")],
+        health=HealthConfig(**health_kw),
+    )
+
+
+def _state(**kw):
+    s = ServiceState(started_at=_dt.datetime.now(), pid=1)
+    # Force last_cycle_at to "recent" by default so cycle_lag is small.
+    s.last_cycle_at = _dt.datetime.now()
+    for k, v in kw.items():
+        setattr(s, k, v)
+    return s
+
+
+# ---------------------------------------------------------------- ServiceState
+
+def test_state_snapshot_is_independent_copy():
+    s = ServiceState(started_at=_dt.datetime.now(), pid=1)
+    s.update_cycle(active=True, duration_s=1.5)
+    snap = s.snapshot()
+    snap["capture_enqueued"] = 999  # mutate the copy
+    assert s.snapshot()["capture_enqueued"] == 0  # original untouched
+
+
+def test_state_update_cycle_accumulates_capture_totals():
+    from sync.capture import CaptureStats
+    s = ServiceState(started_at=_dt.datetime.now(), pid=1)
+    s.update_cycle(True, 1.0, capture=CaptureStats(enqueued=5, deferred=2, aged_out=1))
+    s.update_cycle(True, 1.0, capture=CaptureStats(enqueued=3, deferred=1, aged_out=0))
+    snap = s.snapshot()
+    assert snap["capture_enqueued"] == 8
+    assert snap["capture_deferred"] == 3
+    assert snap["capture_aged_out"] == 1
+
+
+# --------------------------------------------------------------- status logic
+
+def _checker_with(state, cfg, queue=None, compare_mismatch=False):
+    """Build a checker whose SQL queries are stubbed (no real DB)."""
+    c = HealthChecker(state, cfg)
+    c._queue_status = lambda: queue or {}
+    c._drift_tables = lambda: []
+    c._dead_rows_sample = lambda: []
+    c._last_compare = lambda: {"mismatch": compare_mismatch}
+    return c
+
+
+def test_status_healthy_when_recent_cycle_clean_queue():
+    cfg = _cfg()
+    state = _state()
+    c = _checker_with(state, cfg, queue={"pending": 0, "dead": 0, "error": 0})
+    assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
+                            c._data_health()) == "healthy"
+
+
+def test_status_unhealthy_when_cycle_lag_exceeds_threshold():
+    cfg = _cfg()  # poll_interval=10 -> threshold = 30s
+    # last cycle was 60s ago
+    state = ServiceState(started_at=_dt.datetime.now() - _dt.timedelta(seconds=120), pid=1)
+    state.last_cycle_at = _dt.datetime.now() - _dt.timedelta(seconds=60)
+    c = _checker_with(state, cfg)
+    assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
+                            c._data_health()) == "unhealthy"
+
+
+def test_status_degraded_when_dead_rows():
+    cfg = _cfg()
+    state = _state()
+    c = _checker_with(state, cfg, queue={"dead": 2})
+    assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
+                            c._data_health()) == "degraded"
+
+
+def test_status_degraded_when_error_rows():
+    cfg = _cfg()
+    state = _state()
+    c = _checker_with(state, cfg, queue={"error": 1})
+    assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
+                            c._data_health()) == "degraded"
+
+
+def test_status_degraded_when_aged_out_accumulated():
+    cfg = _cfg()
+    state = _state()
+    state.update_cycle(True, 1.0)  # no capture arg
+    state.capture_aged_out = 3     # simulate prior aged-out incidents
+    c = _checker_with(state, cfg, queue={})
+    snap = c.snapshot()
+    assert snap["status"] == "degraded"
+
+
+def test_status_degraded_when_compare_mismatch():
+    cfg = _cfg()
+    state = _state()
+    c = _checker_with(state, cfg, queue={}, compare_mismatch=True)
+    assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
+                            c._data_health()) == "degraded"
+
+
+# -------------------------------------------------------------- snapshot shape
+
+def test_snapshot_has_expected_top_level_keys():
+    cfg = _cfg()
+    state = _state()
+    c = _checker_with(state, cfg, queue={"pending": 0})
+    snap = c.snapshot()
+    for k in ("service", "status", "checked_at", "pid", "started_at",
+              "uptime_seconds", "service_health", "data_health"):
+        assert k in snap
+    assert "cycle_lag_seconds" in snap["service_health"]
+    assert "last_compare" in snap["data_health"]
+
+
+# ------------------------------------------------------------- HTTP endpoint
+
+def test_health_endpoint_returns_200_when_healthy():
+    cfg = _cfg()
+    state = _state()
+    with patch.object(HealthChecker, "_queue_status", return_value={}), \
+         patch.object(HealthChecker, "_drift_tables", return_value=[]), \
+         patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
+         patch.object(HealthChecker, "_last_compare",
+                      return_value={"available": False}):
+        app = create_app(state, cfg)
+        client = TestClient(app)
+        r = client.get("/health")
+    assert r.status_code == 200
+    body = r.json()
+    assert body["status"] == "healthy"
+    assert body["service"] == "DataMacroSync"
+
+
+def test_health_endpoint_returns_503_when_unhealthy():
+    cfg = _cfg()
+    # stale cycle -> unhealthy
+    state = ServiceState(started_at=_dt.datetime.now() - _dt.timedelta(seconds=120), pid=1)
+    state.last_cycle_at = _dt.datetime.now() - _dt.timedelta(seconds=60)
+    with patch.object(HealthChecker, "_queue_status", return_value={}), \
+         patch.object(HealthChecker, "_drift_tables", return_value=[]), \
+         patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
+         patch.object(HealthChecker, "_last_compare",
+                      return_value={"available": False}):
+        app = create_app(state, cfg)
+        client = TestClient(app)
+        r = client.get("/health")
+    assert r.status_code == 503
+    assert r.json()["status"] == "unhealthy"
+
+
+def test_health_endpoint_returns_503_when_degraded():
+    cfg = _cfg()
+    state = _state()
+    with patch.object(HealthChecker, "_queue_status",
+                      return_value={"dead": 1}), \
+         patch.object(HealthChecker, "_drift_tables", return_value=[]), \
+         patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
+         patch.object(HealthChecker, "_last_compare",
+                      return_value={"available": False}):
+        app = create_app(state, cfg)
+        client = TestClient(app)
+        r = client.get("/health")
+    assert r.status_code == 503
+    assert r.json()["status"] == "degraded"