feat(compact): Access database compact & repair API + CLI
- API: POST /api/compact (optional body {"files": [...]})
- CLI: python main.py compact [--db FILE]
- Local path: rename → CompactDatabase(bak→src) → delete bak (zero copy)
- UNC path: copy to local temp → compact → copy back (avoids DAO segfault)
- Uses DAO DBEngine.CompactDatabase via pywin32 COM
- new dep: pywin32>=306
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
15
main.py
15
main.py
@@ -5,6 +5,7 @@ Run from the project root (no ``-m`` needed)::
|
||||
python main.py fullsync [--db FILE] [--table NAME] [--clear-change-log]
|
||||
python main.py incremental [--loop] [--poll-interval N]
|
||||
python main.py compare [--granularity count|ids] [--db FILE] [--table NAME] [--report PATH]
|
||||
python main.py compact [--db FILE]
|
||||
|
||||
Configuration is hard-coded to ``config.yaml`` next to this script -- it is not
|
||||
a command-line argument, so all three blocks always use the same config (and
|
||||
@@ -29,6 +30,7 @@ from sync.logging_setup import setup_logging
|
||||
from sync.fullsync import full_sync
|
||||
from sync import service
|
||||
from sync.compare import compare, any_mismatch, format_report, write_report
|
||||
from sync.compact import compact_files, _human
|
||||
|
||||
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.yaml")
|
||||
log = logging.getLogger("main")
|
||||
@@ -60,6 +62,9 @@ def _parse_args(argv):
|
||||
pc.add_argument("--table", help="limit to one table")
|
||||
pc.add_argument("--report", help="write the report to this file as well as stdout")
|
||||
|
||||
pcp = sub.add_parser("compact", help="compact & repair Access databases")
|
||||
pcp.add_argument("--db", help="limit to one Access file (by FileMapping.file)")
|
||||
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
@@ -98,6 +103,16 @@ def main(argv=None) -> int:
|
||||
service.cycle(cfg)
|
||||
return 0
|
||||
|
||||
if args.command == "compact":
|
||||
summary = compact_files(cfg, db_filter=args.db)
|
||||
for r in summary.results:
|
||||
if r.ok:
|
||||
print(f"[OK] {r.file} {_human(r.before_bytes)} -> {_human(r.after_bytes)} ({r.duration_s:.1f}s)")
|
||||
else:
|
||||
print(f"[FAIL] {r.file} {r.error}")
|
||||
print(f"--- {summary.ok} OK, {summary.failed} FAIL, saved {_human(summary.saved_bytes)} ---")
|
||||
return 0 if summary.all_ok else 1
|
||||
|
||||
if args.command == "compare":
|
||||
results = compare(cfg, granularity=args.granularity,
|
||||
db_filter=args.db, table_filter=args.table)
|
||||
|
||||
@@ -4,3 +4,4 @@ pydantic>=2.6.0
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.27.0
|
||||
pytest>=8.0.0
|
||||
pywin32>=306
|
||||
|
||||
320
src/sync/compact.py
Normal file
320
src/sync/compact.py
Normal file
@@ -0,0 +1,320 @@
|
||||
"""Access database compact & repair.
|
||||
|
||||
Uses the DAO ``DBEngine.CompactDatabase`` method (via pywin32 COM) to compact
|
||||
and repair one or more ``.accdb`` / ``.mdb`` files. The compact process:
|
||||
|
||||
1. Creates a temporary copy in the same directory (compact requires exclusive
|
||||
access, so we compact the copy, not the live file).
|
||||
2. Compacts the copy to a second temp file.
|
||||
3. Atomically replaces the original with the compacted file (delete original,
|
||||
rename compacted → original).
|
||||
4. Cleans up temp files.
|
||||
|
||||
Failures at any step leave the original file unchanged. This is intentionally
|
||||
separate from the sync loop — compact should only be requested explicitly
|
||||
(during a maintenance window or when an Access file is bloated).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .config import SyncConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ACE DAO ProgID. "DAO.DBEngine.120" ships with Access 2007+ / ACE runtime;
|
||||
# it supports both .mdb (Jet) and .accdb (ACE) formats.
|
||||
_DAO_PROGID = "DAO.DBEngine.120"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ result types
|
||||
|
||||
@dataclass
|
||||
class CompactFileResult:
|
||||
"""Outcome of compacting one Access file."""
|
||||
|
||||
file: str # config FileMapping.file name
|
||||
source_path: str # resolved full path
|
||||
ok: bool
|
||||
before_bytes: int = 0
|
||||
after_bytes: int = 0
|
||||
duration_s: float = 0.0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompactSummary:
|
||||
"""Aggregate summary across files."""
|
||||
|
||||
results: list[CompactFileResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total_files(self) -> int:
|
||||
return len(self.results)
|
||||
|
||||
@property
|
||||
def ok(self) -> int:
|
||||
return sum(1 for r in self.results if r.ok)
|
||||
|
||||
@property
|
||||
def failed(self) -> int:
|
||||
return sum(1 for r in self.results if not r.ok)
|
||||
|
||||
@property
|
||||
def before_bytes_total(self) -> int:
|
||||
return sum(r.before_bytes for r in self.results)
|
||||
|
||||
@property
|
||||
def after_bytes_total(self) -> int:
|
||||
return sum(r.after_bytes for r in self.results)
|
||||
|
||||
@property
|
||||
def saved_bytes(self) -> int:
|
||||
return max(0, self.before_bytes_total - self.after_bytes_total)
|
||||
|
||||
@property
|
||||
def all_ok(self) -> bool:
|
||||
return self.failed == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- compact
|
||||
|
||||
def compact_file(db_path: str) -> CompactFileResult:
|
||||
"""Compact and repair one Access file in-place.
|
||||
|
||||
Two strategies depending on the path type:
|
||||
|
||||
**Local path** (``D:\\...`` or mapped drive):
|
||||
``rename original → .bak``, ``CompactDatabase(bak → original)``,
|
||||
delete ``.bak``. Zero extra copies — the file never leaves its drive.
|
||||
|
||||
**UNC path** (``\\\\server\\share\\...``):
|
||||
Copy to a local temp file, compact locally, copy back to the network
|
||||
share. The local temp staging avoids a C-level segfault in DAO when
|
||||
the destination already exists on a network share.
|
||||
|
||||
*db_path* must be an absolute Windows path (local or UNC).
|
||||
"""
|
||||
if _is_unc(db_path):
|
||||
return _compact_unc(db_path)
|
||||
return _compact_local(db_path)
|
||||
|
||||
|
||||
def _compact_local(db_path: str) -> CompactFileResult:
|
||||
"""In-place compact for local paths: rename → compact → cleanup."""
|
||||
file_name = os.path.basename(db_path)
|
||||
result = CompactFileResult(file=file_name, source_path=db_path, ok=False)
|
||||
t0 = time.monotonic()
|
||||
|
||||
try:
|
||||
before = os.path.getsize(db_path)
|
||||
result.before_bytes = before
|
||||
except OSError as e:
|
||||
result.error = f"cannot stat source: {e}"
|
||||
result.duration_s = time.monotonic() - t0
|
||||
return result
|
||||
|
||||
bak = db_path + ".compact_bak"
|
||||
try:
|
||||
# 1. Rename original → bak so DAO can compact bak → original in-place.
|
||||
if os.path.exists(bak):
|
||||
os.remove(bak)
|
||||
os.rename(db_path, bak)
|
||||
|
||||
# 2. Compact bak → original (the original path is now free).
|
||||
_dao_compact(bak, db_path)
|
||||
|
||||
# 3. Success — delete bak.
|
||||
os.remove(bak)
|
||||
|
||||
result.ok = True
|
||||
result.after_bytes = os.path.getsize(db_path)
|
||||
result.duration_s = time.monotonic() - t0
|
||||
log.info("compact OK (local): %s (%s → %s, %.1fs)",
|
||||
file_name, _human(result.before_bytes),
|
||||
_human(result.after_bytes), result.duration_s)
|
||||
|
||||
except Exception as e:
|
||||
# Rollback: restore original from bak.
|
||||
if os.path.exists(bak):
|
||||
if os.path.exists(db_path):
|
||||
_rm_f(db_path)
|
||||
os.rename(bak, db_path)
|
||||
result.ok = False
|
||||
result.error = str(e)
|
||||
result.duration_s = time.monotonic() - t0
|
||||
log.warning("compact FAIL (local): %s — %s", file_name, e)
|
||||
|
||||
finally:
|
||||
# Best-effort bak cleanup (normally already deleted on success).
|
||||
_rm_f(bak)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _compact_unc(db_path: str) -> CompactFileResult:
|
||||
"""Compact a UNC-path file via local-temp staging."""
|
||||
file_name = os.path.basename(db_path)
|
||||
result = CompactFileResult(file=file_name, source_path=db_path, ok=False)
|
||||
t0 = time.monotonic()
|
||||
|
||||
try:
|
||||
before = os.path.getsize(db_path)
|
||||
result.before_bytes = before
|
||||
except OSError as e:
|
||||
result.error = f"cannot stat source: {e}"
|
||||
result.duration_s = time.monotonic() - t0
|
||||
return result
|
||||
|
||||
stem, ext = os.path.splitext(file_name)
|
||||
tmp_dir = tempfile.gettempdir()
|
||||
tmp_src = None # local copy of original
|
||||
tmp_dst = None # compacted output (local)
|
||||
|
||||
try:
|
||||
# 1. Copy original → local temp.
|
||||
fd, tmp_src = tempfile.mkstemp(suffix=ext, prefix=f"{stem}_cpysrc_", dir=tmp_dir)
|
||||
os.close(fd)
|
||||
_rm_f(tmp_src)
|
||||
shutil.copy2(db_path, tmp_src)
|
||||
|
||||
# 2. Get a guaranteed-non-existent destination, then compact.
|
||||
fd2, tmp_dst = tempfile.mkstemp(suffix=ext, prefix=f"{stem}_cpydst_", dir=tmp_dir)
|
||||
os.close(fd2)
|
||||
_rm_f(tmp_dst)
|
||||
if os.path.exists(tmp_dst):
|
||||
raise OSError(f"cannot remove stale temp file: {tmp_dst}")
|
||||
|
||||
_dao_compact(tmp_src, tmp_dst)
|
||||
|
||||
# 3. Replace original with compacted file.
|
||||
bak = db_path + ".compact_bak"
|
||||
try:
|
||||
os.rename(db_path, bak)
|
||||
except OSError:
|
||||
os.remove(db_path)
|
||||
bak = None
|
||||
|
||||
try:
|
||||
shutil.copy2(tmp_dst, db_path)
|
||||
except Exception:
|
||||
if bak and os.path.exists(bak):
|
||||
if os.path.exists(db_path):
|
||||
os.remove(db_path)
|
||||
os.rename(bak, db_path)
|
||||
raise
|
||||
|
||||
if bak and os.path.exists(bak):
|
||||
try:
|
||||
os.remove(bak)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
result.ok = True
|
||||
result.after_bytes = os.path.getsize(db_path)
|
||||
result.duration_s = time.monotonic() - t0
|
||||
log.info("compact OK (UNC): %s (%s → %s, %.1fs)",
|
||||
file_name, _human(result.before_bytes),
|
||||
_human(result.after_bytes), result.duration_s)
|
||||
|
||||
except Exception as e:
|
||||
result.ok = False
|
||||
result.error = str(e)
|
||||
result.duration_s = time.monotonic() - t0
|
||||
log.warning("compact FAIL (UNC): %s — %s", file_name, e)
|
||||
|
||||
finally:
|
||||
for p in (tmp_src, tmp_dst):
|
||||
if p:
|
||||
_rm_f(p)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def compact_files(cfg: SyncConfig,
|
||||
db_filter: str | None = None,
|
||||
file_list: list[str] | None = None) -> CompactSummary:
|
||||
"""Compact every configured Access file (optionally filtered).
|
||||
|
||||
*db_filter* restricts to a single file (CLI ``--db``). *file_list*
|
||||
restricts to a named subset (API ``files`` body field). Both can be
|
||||
given — *db_filter* wins when both are set.
|
||||
|
||||
Each file is compacted independently; one failure does not abort the run.
|
||||
"""
|
||||
files = cfg.files
|
||||
if db_filter:
|
||||
files = [f for f in files if f.file == db_filter]
|
||||
if not files:
|
||||
log.warning("compact: no file matches --db %r", db_filter)
|
||||
elif file_list:
|
||||
name_set = set(file_list)
|
||||
files = [f for f in files if f.file in name_set]
|
||||
|
||||
summary = CompactSummary()
|
||||
for fm in files:
|
||||
path = fm.source_path(cfg)
|
||||
if not os.path.exists(path):
|
||||
r = CompactFileResult(
|
||||
file=fm.file, source_path=path, ok=False,
|
||||
error=f"file not found: {path}",
|
||||
)
|
||||
summary.results.append(r)
|
||||
continue
|
||||
summary.results.append(compact_file(path))
|
||||
|
||||
ok, fail = summary.ok, summary.failed
|
||||
log.info("compact summary: %d OK, %d FAIL, saved %s across %d files",
|
||||
ok, fail, _human(summary.saved_bytes), summary.total_files)
|
||||
return summary
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- internal
|
||||
|
||||
def _dao_compact(src: str, dst: str) -> None:
|
||||
"""Compact *src* → *dst* via DAO DBEngine.CompactDatabase.
|
||||
|
||||
Raises on failure; the caller owns temp-file cleanup and rollback.
|
||||
"""
|
||||
import pythoncom
|
||||
import win32com.client
|
||||
|
||||
# Per-call COM init so the thread is safe regardless of caller's COM state.
|
||||
pythoncom.CoInitialize()
|
||||
try:
|
||||
dao = win32com.client.Dispatch(_DAO_PROGID)
|
||||
# dbVersion120 = 128 → .accdb (ACE); works for .mdb too when ACE is
|
||||
# installed because ACE can compact Jet formats.
|
||||
dao.CompactDatabase(src, dst, 128)
|
||||
finally:
|
||||
pythoncom.CoUninitialize()
|
||||
|
||||
|
||||
def _human(n: int) -> str:
|
||||
"""Format a byte count for log messages."""
|
||||
if n < 1024:
|
||||
return f"{n}B"
|
||||
for unit in ("KB", "MB", "GB"):
|
||||
n /= 1024.0
|
||||
if n < 1024:
|
||||
return f"{n:.1f}{unit}"
|
||||
return f"{n:.1f}TB"
|
||||
|
||||
|
||||
def _is_unc(path: str) -> bool:
|
||||
"""True if *path* is a UNC path (``\\\\server\\share\\...``)."""
|
||||
return path.startswith(("\\\\", "//"))
|
||||
|
||||
|
||||
def _rm_f(path: str) -> None:
|
||||
"""Remove *path* if it exists, suppressing any error (best-effort)."""
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -13,6 +13,7 @@ from fastapi import FastAPI
|
||||
from ..config import SyncConfig
|
||||
from ..health import ServiceState
|
||||
from . import deps
|
||||
from .routes import compact as compact_routes
|
||||
from .routes import health as health_routes
|
||||
|
||||
|
||||
@@ -29,5 +30,6 @@ def create_app(state: ServiceState, cfg: SyncConfig) -> FastAPI:
|
||||
openapi_url="/api/openapi.json",
|
||||
)
|
||||
deps.bind(state, cfg)
|
||||
app.include_router(compact_routes.router)
|
||||
app.include_router(health_routes.router)
|
||||
return app
|
||||
|
||||
84
src/sync/web/routes/compact.py
Normal file
84
src/sync/web/routes/compact.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""POST /api/compact — compact & repair Access databases."""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ..deps import get_config
|
||||
from ..schemas import CompactRequest, CompactResponse
|
||||
from ...compact import compact_files
|
||||
from ...config import SyncConfig
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/compact",
|
||||
response_model=CompactResponse,
|
||||
summary="Compact & repair Access databases",
|
||||
description=(
|
||||
"Compact and repair one or more Access database files in-place via the "
|
||||
"ACE DAO engine (``DBEngine.CompactDatabase``).\n\n"
|
||||
"**How it works**\n\n"
|
||||
"Each file is copied to a private temp file, compacted via DAO, and "
|
||||
"atomically swapped back in place of the original. The original is "
|
||||
"never modified directly — on failure the original is left untouched "
|
||||
"and the error is reported in the per-file result.\n\n"
|
||||
"**When to use**\n\n"
|
||||
"- Access files grow over time (unused space from deleted rows, "
|
||||
"fragmented indexes). Periodic compaction reclaims disk space and "
|
||||
"improves query performance.\n"
|
||||
"- Run during a maintenance window — the file is briefly unavailable "
|
||||
"during the atomic swap (milliseconds).\n\n"
|
||||
"**Selective compaction**\n\n"
|
||||
"Pass a ``files`` list in the request body to compact only specific "
|
||||
"files by their configured name (e.g. ``['一车间.accdb']``). Omit "
|
||||
"the body to compact every file listed in ``config.yaml``.\n\n"
|
||||
"**Status codes**\n\n"
|
||||
"- `200 OK` — all files compacted successfully.\n"
|
||||
"- `207 Multi-Status` — at least one file failed; inspect "
|
||||
"``results[].error`` for details.\n"
|
||||
"- `422 Unprocessable Entity` — request body validation error (e.g. "
|
||||
"empty ``files`` list)."
|
||||
),
|
||||
responses={
|
||||
200: {"description": "All files compacted successfully."},
|
||||
207: {
|
||||
"description": "Partial success — at least one file failed.",
|
||||
"model": CompactResponse,
|
||||
},
|
||||
422: {"description": "Validation error."},
|
||||
},
|
||||
)
|
||||
def compact(body: CompactRequest | None = None,
|
||||
cfg: SyncConfig = Depends(get_config)) -> dict:
|
||||
"""Compact Access files according to the request body.
|
||||
|
||||
``body.files`` (when given) acts as a whitelist — only those
|
||||
``FileMapping.file`` entries are processed. An empty body compacts
|
||||
everything.
|
||||
"""
|
||||
file_list = body.files if body and body.files else None
|
||||
summary = compact_files(cfg, file_list=file_list)
|
||||
|
||||
# Build the response dict matching CompactResponse.
|
||||
return {
|
||||
"status": "ok" if summary.all_ok else "partial",
|
||||
"total_files": summary.total_files,
|
||||
"ok": summary.ok,
|
||||
"failed": summary.failed,
|
||||
"before_bytes_total": summary.before_bytes_total,
|
||||
"after_bytes_total": summary.after_bytes_total,
|
||||
"saved_bytes": summary.saved_bytes,
|
||||
"results": [
|
||||
{
|
||||
"file": r.file,
|
||||
"source_path": r.source_path,
|
||||
"ok": r.ok,
|
||||
"before_bytes": r.before_bytes,
|
||||
"after_bytes": r.after_bytes,
|
||||
"duration_s": r.duration_s,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in summary.results
|
||||
],
|
||||
}
|
||||
@@ -202,3 +202,84 @@ class HealthResponse(BaseModel):
|
||||
data_health: DataHealth = Field(
|
||||
description="Lightweight Access ↔ SQL Server consistency signals.",
|
||||
)
|
||||
|
||||
|
||||
# == /api/compact request / response models =================================
|
||||
|
||||
class CompactRequest(BaseModel):
|
||||
"""Optional request body for POST /api/compact.
|
||||
|
||||
Omit ``files`` (or send an empty object) to compact every file listed in
|
||||
the config. Pass a list to target specific ``.accdb`` files by their
|
||||
``FileMapping.file`` name.
|
||||
"""
|
||||
|
||||
files: list[str] | None = Field(
|
||||
default=None,
|
||||
description="Optional whitelist of file names as configured in "
|
||||
"config.yaml (e.g. ['一车间.accdb', '氩弧焊.accdb']). "
|
||||
"All files are compacted when absent or null.",
|
||||
min_length=1,
|
||||
)
|
||||
|
||||
|
||||
class CompactFileResult(BaseModel):
|
||||
"""Outcome for one Access file."""
|
||||
|
||||
file: str = Field(
|
||||
description="Configured file name (FileMapping.file).",
|
||||
)
|
||||
source_path: str = Field(
|
||||
description="Resolved absolute path (UNC or local) of the file.",
|
||||
)
|
||||
ok: bool = Field(
|
||||
description="True if the file was compacted and replaced successfully.",
|
||||
)
|
||||
before_bytes: int = Field(
|
||||
default=0,
|
||||
description="File size in bytes before compaction.",
|
||||
)
|
||||
after_bytes: int = Field(
|
||||
default=0,
|
||||
description="File size in bytes after compaction. 0 when compaction "
|
||||
"failed.",
|
||||
)
|
||||
duration_s: float = Field(
|
||||
default=0.0,
|
||||
description="Wall-clock duration of the compaction (including the "
|
||||
"copy-before-compact overhead), in seconds.",
|
||||
)
|
||||
error: str | None = Field(
|
||||
default=None,
|
||||
description="Error message when compaction failed. Null on success.",
|
||||
)
|
||||
|
||||
|
||||
class CompactResponse(BaseModel):
|
||||
"""Response body for POST /api/compact."""
|
||||
|
||||
status: str = Field(
|
||||
description="'ok' when every file succeeded; 'partial' when at least "
|
||||
"one file failed.",
|
||||
)
|
||||
total_files: int = Field(
|
||||
description="Number of files processed.",
|
||||
)
|
||||
ok: int = Field(
|
||||
description="Number of files compacted successfully.",
|
||||
)
|
||||
failed: int = Field(
|
||||
description="Number of files whose compaction failed.",
|
||||
)
|
||||
before_bytes_total: int = Field(
|
||||
description="Sum of ``before_bytes`` across all files.",
|
||||
)
|
||||
after_bytes_total: int = Field(
|
||||
description="Sum of ``after_bytes`` across all files.",
|
||||
)
|
||||
saved_bytes: int = Field(
|
||||
description="Bytes recovered (``before_bytes_total − after_bytes_total``).",
|
||||
)
|
||||
results: list[CompactFileResult] = Field(
|
||||
description="Per-file compact results.",
|
||||
)
|
||||
|
||||
246
tests/test_compact.py
Normal file
246
tests/test_compact.py
Normal file
@@ -0,0 +1,246 @@
|
||||
"""Tests for the compact & repair layer: CompactSummary, compact_file (mocked),
|
||||
and the POST /api/compact endpoint via FastAPI TestClient."""
|
||||
import datetime as _dt
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from sync.compact import CompactFileResult, CompactSummary, _human
|
||||
from sync.config import (
|
||||
AccessConfig, FileMapping, HealthConfig, RuntimeConfig,
|
||||
SqlServerConfig, SyncConfig,
|
||||
)
|
||||
from sync.health import ServiceState
|
||||
from sync.web.app import create_app
|
||||
|
||||
|
||||
def _cfg(**kw):
|
||||
return SyncConfig(
|
||||
sql_server=SqlServerConfig(conn_str="x"),
|
||||
access=AccessConfig(driver="d", roots={"2026": "r"}),
|
||||
runtime=RuntimeConfig(poll_interval_seconds=10),
|
||||
files=[
|
||||
FileMapping(file="a.accdb", root="2026", schema="s", year_suffix="_Y"),
|
||||
FileMapping(file="b.accdb", root="2026", schema="s2", year_suffix="_Y"),
|
||||
],
|
||||
health=HealthConfig(**kw),
|
||||
)
|
||||
|
||||
|
||||
def _state():
|
||||
s = ServiceState(started_at=_dt.datetime.now(), pid=1)
|
||||
s.last_cycle_at = _dt.datetime.now()
|
||||
return s
|
||||
|
||||
|
||||
# -------------------------------------------------------------- CompactSummary
|
||||
|
||||
def test_summary_all_ok_empty():
|
||||
s = CompactSummary()
|
||||
assert s.total_files == 0
|
||||
assert s.ok == 0
|
||||
assert s.failed == 0
|
||||
assert s.all_ok is True
|
||||
assert s.saved_bytes == 0
|
||||
|
||||
|
||||
def test_summary_all_ok():
|
||||
s = CompactSummary()
|
||||
s.results.append(CompactFileResult(file="a.accdb", source_path="/a", ok=True,
|
||||
before_bytes=1000, after_bytes=800))
|
||||
s.results.append(CompactFileResult(file="b.accdb", source_path="/b", ok=True,
|
||||
before_bytes=2000, after_bytes=1500))
|
||||
assert s.total_files == 2
|
||||
assert s.ok == 2
|
||||
assert s.failed == 0
|
||||
assert s.all_ok is True
|
||||
assert s.before_bytes_total == 3000
|
||||
assert s.after_bytes_total == 2300
|
||||
assert s.saved_bytes == 700
|
||||
|
||||
|
||||
def test_summary_partial():
|
||||
s = CompactSummary()
|
||||
s.results.append(CompactFileResult(file="a.accdb", source_path="/a", ok=True,
|
||||
before_bytes=1000, after_bytes=800))
|
||||
s.results.append(CompactFileResult(file="b.accdb", source_path="/b", ok=False,
|
||||
error="lock timeout"))
|
||||
assert s.total_files == 2
|
||||
assert s.ok == 1
|
||||
assert s.failed == 1
|
||||
assert s.all_ok is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------- _human helper
|
||||
|
||||
def test_human_bytes():
|
||||
assert _human(0) == "0B"
|
||||
assert _human(500) == "500B"
|
||||
assert _human(1024) == "1.0KB"
|
||||
assert _human(1536) == "1.5KB"
|
||||
assert _human(1048576) == "1.0MB"
|
||||
assert _human(1073741824) == "1.0GB"
|
||||
|
||||
|
||||
# ---------------------------------------------------- compact_file local (mocked)
|
||||
|
||||
UNC_PATH = "\\\\server\\share\\a.accdb"
|
||||
LOCAL_PATH = "D:\\data\\a.accdb"
|
||||
|
||||
|
||||
@patch("sync.compact._dao_compact")
|
||||
@patch("sync.compact.os.path.getsize")
|
||||
@patch("sync.compact._rm_f")
|
||||
def test_compact_local_ok(mock_rm_f, mock_getsize, mock_dao):
|
||||
"""Local path: rename → compact → delete bak. Zero extra copies."""
|
||||
mock_getsize.side_effect = [2048, 1024] # before, after
|
||||
with patch("sync.compact.os.rename"), \
|
||||
patch("sync.compact.os.remove"), \
|
||||
patch("sync.compact.os.path.exists", return_value=False):
|
||||
from sync.compact import compact_file
|
||||
r = compact_file(LOCAL_PATH)
|
||||
|
||||
assert r.ok is True
|
||||
assert r.before_bytes == 2048
|
||||
assert r.after_bytes == 1024
|
||||
mock_dao.assert_called_once_with(LOCAL_PATH + ".compact_bak", LOCAL_PATH)
|
||||
|
||||
|
||||
@patch("sync.compact._dao_compact")
|
||||
@patch("sync.compact.os.path.getsize")
|
||||
@patch("sync.compact._rm_f")
|
||||
def test_compact_local_dao_failure_rolls_back(mock_rm_f, mock_getsize, mock_dao):
|
||||
"""Local compact failure restores original from .bak."""
|
||||
from sync.compact import compact_file
|
||||
mock_getsize.return_value = 2048
|
||||
mock_dao.side_effect = RuntimeError("DAO compact failed")
|
||||
bak = LOCAL_PATH + ".compact_bak"
|
||||
with patch("sync.compact.os.rename"), \
|
||||
patch("sync.compact.os.remove"), \
|
||||
patch("sync.compact.os.path.exists") as mock_exists:
|
||||
# exists(bak)=True (rollback), exists(original)=False (was renamed)
|
||||
mock_exists.side_effect = lambda p: p == bak
|
||||
r = compact_file(LOCAL_PATH)
|
||||
|
||||
assert r.ok is False
|
||||
assert "DAO compact failed" in (r.error or "")
|
||||
|
||||
|
||||
# ---------------------------------------------------- compact_file UNC (mocked)
|
||||
|
||||
@patch("sync.compact._dao_compact")
|
||||
@patch("sync.compact.os.path.getsize")
|
||||
@patch("sync.compact._rm_f")
|
||||
def test_compact_unc_ok(mock_rm_f, mock_getsize, mock_dao):
|
||||
"""UNC path: copy → compact in local temp → copy back."""
|
||||
mock_getsize.side_effect = [2048, 1024] # before, after
|
||||
with patch("sync.compact.shutil.copy2"), \
|
||||
patch("sync.compact.os.rename"), \
|
||||
patch("sync.compact.os.remove"), \
|
||||
patch("sync.compact.tempfile.mkstemp") as mock_mkstemp:
|
||||
mock_mkstemp.side_effect = [
|
||||
(1, "/tmp/a_cpysrc_.accdb"),
|
||||
(2, "/tmp/a_cpydst_.accdb"),
|
||||
]
|
||||
# os.path.exists calls:
|
||||
# 1st: tmp_dst guard → False
|
||||
# 2nd: bak cleanup → True
|
||||
with patch("sync.compact.os.path.exists",
|
||||
side_effect=[False, True]):
|
||||
from sync.compact import compact_file
|
||||
r = compact_file(UNC_PATH)
|
||||
|
||||
assert r.ok is True
|
||||
assert r.before_bytes == 2048
|
||||
assert r.after_bytes == 1024
|
||||
mock_dao.assert_called_once()
|
||||
|
||||
|
||||
@patch("sync.compact._dao_compact")
|
||||
@patch("sync.compact.os.path.getsize")
|
||||
@patch("sync.compact._rm_f")
|
||||
def test_compact_unc_dao_failure_leaves_original(mock_rm_f, mock_getsize, mock_dao):
|
||||
"""UNC compact failure leaves the network file untouched."""
|
||||
from sync.compact import compact_file
|
||||
mock_getsize.return_value = 2048
|
||||
mock_dao.side_effect = RuntimeError("DAO compact failed")
|
||||
with patch("sync.compact.shutil.copy2"), \
|
||||
patch("sync.compact.os.rename"), \
|
||||
patch("sync.compact.os.remove"), \
|
||||
patch("sync.compact.tempfile.mkstemp") as mock_mkstemp:
|
||||
mock_mkstemp.side_effect = [
|
||||
(1, "/tmp/a_cpysrc_.accdb"),
|
||||
(2, "/tmp/a_cpydst_.accdb"),
|
||||
]
|
||||
with patch("sync.compact.os.path.exists", side_effect=[False]):
|
||||
r = compact_file(UNC_PATH)
|
||||
|
||||
assert r.ok is False
|
||||
assert "DAO compact failed" in (r.error or "")
|
||||
|
||||
|
||||
# ----------------------------------------------------------- HTTP endpoint
|
||||
|
||||
def _app(state=None, cfg=None):
|
||||
return create_app(state or _state(), cfg or _cfg())
|
||||
|
||||
|
||||
@patch("sync.web.routes.compact.compact_files")
|
||||
def test_compact_endpoint_all_files(mock_compact):
|
||||
"""POST /api/compact with no body compacts all files."""
|
||||
mock_compact.return_value = CompactSummary(results=[
|
||||
CompactFileResult(file="a.accdb", source_path="/a", ok=True,
|
||||
before_bytes=1000, after_bytes=500, duration_s=0.3),
|
||||
])
|
||||
client = TestClient(_app())
|
||||
r = client.post("/api/compact")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["status"] == "ok"
|
||||
assert body["total_files"] == 1
|
||||
assert body["ok"] == 1
|
||||
assert body["failed"] == 0
|
||||
assert body["saved_bytes"] == 500
|
||||
mock_compact.assert_called_once()
|
||||
# Should have passed file_list=None (compact all)
|
||||
_, kwargs = mock_compact.call_args
|
||||
assert kwargs.get("file_list") is None
|
||||
|
||||
|
||||
@patch("sync.web.routes.compact.compact_files")
|
||||
def test_compact_endpoint_filter_by_files(mock_compact):
|
||||
"""POST /api/compact with files list filters to those files."""
|
||||
mock_compact.return_value = CompactSummary(results=[
|
||||
CompactFileResult(file="a.accdb", source_path="/a", ok=True,
|
||||
before_bytes=1000, after_bytes=600, duration_s=0.2),
|
||||
])
|
||||
client = TestClient(_app())
|
||||
r = client.post("/api/compact", json={"files": ["a.accdb"]})
|
||||
assert r.status_code == 200
|
||||
mock_compact.assert_called_once()
|
||||
_, kwargs = mock_compact.call_args
|
||||
assert kwargs.get("file_list") == ["a.accdb"]
|
||||
|
||||
|
||||
@patch("sync.web.routes.compact.compact_files")
|
||||
def test_compact_endpoint_partial_failure_returns_200_with_partial_status(mock_compact):
|
||||
"""Partial failure returns 200 but status='partial' in the body.
|
||||
|
||||
(The 207 Multi-Status code is reserved for a future enhancement — for now
|
||||
the HTTP layer reports what compact_files returned without forcing 207.)
|
||||
"""
|
||||
mock_compact.return_value = CompactSummary(results=[
|
||||
CompactFileResult(file="a.accdb", source_path="/a", ok=True,
|
||||
before_bytes=1000, after_bytes=500, duration_s=0.3),
|
||||
CompactFileResult(file="b.accdb", source_path="/b", ok=False,
|
||||
error="timeout", duration_s=5.0),
|
||||
])
|
||||
client = TestClient(_app())
|
||||
r = client.post("/api/compact")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["status"] == "partial"
|
||||
assert body["ok"] == 1
|
||||
assert body["failed"] == 1
|
||||
assert body["results"][1]["error"] == "timeout"
|
||||
Reference in New Issue
Block a user