03d7bcea39
在同步进程内内置 FastAPI/uvicorn HTTP endpoint (GET /health)，返回结构化 健康快照，便于运维/监控被动探活： - 服务健康：cycle_lag（核心指标，超过 3×poll_interval 判 unhealthy）、 队列状态（pending/error/dead/cleaned 实时查）、capture 累计（enqueued/ deferred/aged_out）、最近 cycle 时间/耗时/错误。 - 数据健康：解析最近一次定时 compare_ids 报告（不跑全量 compare 太重）、 实时查 SyncLogArchive 异常痕迹（AgedOut/降级）、SyncQueue dead 行样本。 status 三态：healthy(200) / degraded(503, 有dead/error/aged_out/compare 不一致) / unhealthy(503, 主循环停滞)。状态码映射便于按码告警。 线程模型：uvicorn 占主线程，capture/apply/cleanup 循环跑后台 daemon 线程， NSSM 停服务时主线程退出、daemon 自动终止。health.enabled=false 时退化为 旧行为（同步循环占主线程）。Web 层(routes/)与业务逻辑(HealthChecker) 解耦，后期加运维接口（compare触发/死信管理/metrics）零结构改动。 零 SQL/零 schema 改动。新增依赖 fastapi/uvicorn[standard]。
184 lines
6.9 KiB
Python
184 lines
6.9 KiB
Python
"""Tests for the health-check layer: ServiceState, HealthChecker status logic,
|
|
and the /health endpoint via FastAPI TestClient."""
|
|
import datetime as _dt
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from sync.config import (
|
|
AccessConfig, FileMapping, HealthConfig, RuntimeConfig,
|
|
SqlServerConfig, SyncConfig,
|
|
)
|
|
from sync.health import CYCLE_LAG_MULTIPLIER, HealthChecker, ServiceState
|
|
from sync.web.app import create_app
|
|
|
|
|
|
def _cfg(**health_kw):
|
|
return SyncConfig(
|
|
sql_server=SqlServerConfig(conn_str="x"),
|
|
access=AccessConfig(driver="d", roots={"2026": "r"}),
|
|
runtime=RuntimeConfig(poll_interval_seconds=10),
|
|
files=[FileMapping(file="f.accdb", root="2026", schema="s", year_suffix="_Y")],
|
|
health=HealthConfig(**health_kw),
|
|
)
|
|
|
|
|
|
def _state(**kw):
|
|
s = ServiceState(started_at=_dt.datetime.now(), pid=1)
|
|
# Force last_cycle_at to "recent" by default so cycle_lag is small.
|
|
s.last_cycle_at = _dt.datetime.now()
|
|
for k, v in kw.items():
|
|
setattr(s, k, v)
|
|
return s
|
|
|
|
|
|
# ---------------------------------------------------------------- ServiceState
|
|
|
|
def test_state_snapshot_is_independent_copy():
|
|
s = ServiceState(started_at=_dt.datetime.now(), pid=1)
|
|
s.update_cycle(active=True, duration_s=1.5)
|
|
snap = s.snapshot()
|
|
snap["capture_enqueued"] = 999 # mutate the copy
|
|
assert s.snapshot()["capture_enqueued"] == 0 # original untouched
|
|
|
|
|
|
def test_state_update_cycle_accumulates_capture_totals():
|
|
from sync.capture import CaptureStats
|
|
s = ServiceState(started_at=_dt.datetime.now(), pid=1)
|
|
s.update_cycle(True, 1.0, capture=CaptureStats(enqueued=5, deferred=2, aged_out=1))
|
|
s.update_cycle(True, 1.0, capture=CaptureStats(enqueued=3, deferred=1, aged_out=0))
|
|
snap = s.snapshot()
|
|
assert snap["capture_enqueued"] == 8
|
|
assert snap["capture_deferred"] == 3
|
|
assert snap["capture_aged_out"] == 1
|
|
|
|
|
|
# --------------------------------------------------------------- status logic
|
|
|
|
def _checker_with(state, cfg, queue=None, compare_mismatch=False):
|
|
"""Build a checker whose SQL queries are stubbed (no real DB)."""
|
|
c = HealthChecker(state, cfg)
|
|
c._queue_status = lambda: queue or {}
|
|
c._drift_tables = lambda: []
|
|
c._dead_rows_sample = lambda: []
|
|
c._last_compare = lambda: {"mismatch": compare_mismatch}
|
|
return c
|
|
|
|
|
|
def test_status_healthy_when_recent_cycle_clean_queue():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
c = _checker_with(state, cfg, queue={"pending": 0, "dead": 0, "error": 0})
|
|
assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
|
|
c._data_health()) == "healthy"
|
|
|
|
|
|
def test_status_unhealthy_when_cycle_lag_exceeds_threshold():
|
|
cfg = _cfg() # poll_interval=10 -> threshold = 30s
|
|
# last cycle was 60s ago
|
|
state = ServiceState(started_at=_dt.datetime.now() - _dt.timedelta(seconds=120), pid=1)
|
|
state.last_cycle_at = _dt.datetime.now() - _dt.timedelta(seconds=60)
|
|
c = _checker_with(state, cfg)
|
|
assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
|
|
c._data_health()) == "unhealthy"
|
|
|
|
|
|
def test_status_degraded_when_dead_rows():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
c = _checker_with(state, cfg, queue={"dead": 2})
|
|
assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
|
|
c._data_health()) == "degraded"
|
|
|
|
|
|
def test_status_degraded_when_error_rows():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
c = _checker_with(state, cfg, queue={"error": 1})
|
|
assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
|
|
c._data_health()) == "degraded"
|
|
|
|
|
|
def test_status_degraded_when_aged_out_accumulated():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
state.update_cycle(True, 1.0) # no capture arg
|
|
state.capture_aged_out = 3 # simulate prior aged-out incidents
|
|
c = _checker_with(state, cfg, queue={})
|
|
snap = c.snapshot()
|
|
assert snap["status"] == "degraded"
|
|
|
|
|
|
def test_status_degraded_when_compare_mismatch():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
c = _checker_with(state, cfg, queue={}, compare_mismatch=True)
|
|
assert c._derive_status(c._service_health(state.snapshot(), _dt.datetime.now()),
|
|
c._data_health()) == "degraded"
|
|
|
|
|
|
# -------------------------------------------------------------- snapshot shape
|
|
|
|
def test_snapshot_has_expected_top_level_keys():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
c = _checker_with(state, cfg, queue={"pending": 0})
|
|
snap = c.snapshot()
|
|
for k in ("service", "status", "checked_at", "pid", "started_at",
|
|
"uptime_seconds", "service_health", "data_health"):
|
|
assert k in snap
|
|
assert "cycle_lag_seconds" in snap["service_health"]
|
|
assert "last_compare" in snap["data_health"]
|
|
|
|
|
|
# ------------------------------------------------------------- HTTP endpoint
|
|
|
|
def test_health_endpoint_returns_200_when_healthy():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
with patch.object(HealthChecker, "_queue_status", return_value={}), \
|
|
patch.object(HealthChecker, "_drift_tables", return_value=[]), \
|
|
patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
|
|
patch.object(HealthChecker, "_last_compare",
|
|
return_value={"available": False}):
|
|
app = create_app(state, cfg)
|
|
client = TestClient(app)
|
|
r = client.get("/health")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["status"] == "healthy"
|
|
assert body["service"] == "DataMacroSync"
|
|
|
|
|
|
def test_health_endpoint_returns_503_when_unhealthy():
|
|
cfg = _cfg()
|
|
# stale cycle -> unhealthy
|
|
state = ServiceState(started_at=_dt.datetime.now() - _dt.timedelta(seconds=120), pid=1)
|
|
state.last_cycle_at = _dt.datetime.now() - _dt.timedelta(seconds=60)
|
|
with patch.object(HealthChecker, "_queue_status", return_value={}), \
|
|
patch.object(HealthChecker, "_drift_tables", return_value=[]), \
|
|
patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
|
|
patch.object(HealthChecker, "_last_compare",
|
|
return_value={"available": False}):
|
|
app = create_app(state, cfg)
|
|
client = TestClient(app)
|
|
r = client.get("/health")
|
|
assert r.status_code == 503
|
|
assert r.json()["status"] == "unhealthy"
|
|
|
|
|
|
def test_health_endpoint_returns_503_when_degraded():
|
|
cfg = _cfg()
|
|
state = _state()
|
|
with patch.object(HealthChecker, "_queue_status",
|
|
return_value={"dead": 1}), \
|
|
patch.object(HealthChecker, "_drift_tables", return_value=[]), \
|
|
patch.object(HealthChecker, "_dead_rows_sample", return_value=[]), \
|
|
patch.object(HealthChecker, "_last_compare",
|
|
return_value={"available": False}):
|
|
app = create_app(state, cfg)
|
|
client = TestClient(app)
|
|
r = client.get("/health")
|
|
assert r.status_code == 503
|
|
assert r.json()["status"] == "degraded"
|