feat(sync): add one-shot full sync (TRUNCATE + bulk INSERT)
- New `sync.fullsync` CLI: `python -m sync.fullsync config.yaml [--db F] [--table T] [--clear-change-log]` - AccessReader: list_user_tables / read_all_rows / read_all_log_ids - SqlWriter: table_exists / truncate_target (TRUNCATE w/ DELETE fallback) / bulk_insert (SET IDENTITY_INSERT + chunked fast_executemany) - Reuses FileMapping exclude/include rules (exclude beats include, same as capture) - Preserves Access IDs via IDENTITY_INSERT; target schemas have no FKs so TRUNCATE is safe - TableChangeLog NOT cleared by default (opt-in --clear-change-log) - tests/test_fullsync.py covers resolve_tables exclude/include precedence
This commit is contained in:
Misaka_Company
159
src/sync/fullsync.py
Normal file
159
src/sync/fullsync.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""One-shot full sync: TRUNCATE each target table, then bulk-insert every row
|
||||
from the corresponding Access table.
|
||||
|
||||
This is a manual, out-of-band operation — ``python -m sync.fullsync
|
||||
config.yaml [--db FILE] [--table NAME] [--clear-change-log]``. It deliberately
|
||||
bypasses the incremental ``SyncQueue`` pipeline and writes straight to the SQL
|
||||
Server mirror tables. Use it to (re)seed a schema from scratch or to repair
|
||||
drift between Access and SQL Server.
|
||||
|
||||
Contract with the target tables (same as ``usp_SyncApply``): they already
|
||||
exist and mirror the Access schema column-for-column, with ``ID`` as an
|
||||
IDENTITY primary key. Because ``ID`` is an identity, ``SET IDENTITY_INSERT`` is
|
||||
enabled during insert so the original Access primary keys survive — that is
|
||||
what keeps the incremental sync's RecordID matching correct afterwards.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from .config import load_config, FileMapping, SyncConfig
|
||||
from .access_reader import AccessReader
|
||||
from .sql_writer import SqlWriter
|
||||
from .logging_setup import setup_logging
|
||||
|
||||
log = logging.getLogger("sync.fullsync")
|
||||
|
||||
|
||||
def resolve_tables(reader: AccessReader, fm: FileMapping) -> list[str]:
|
||||
"""Tables to fully sync for one file, after exclude/include rules.
|
||||
|
||||
Mirrors the precedence used by ``capture.capture_file``: a table in
|
||||
``exclude_tables`` is dropped even if it also appears in ``include_tables``.
|
||||
System tables (``MSys*`` / ``~*``) are already filtered by
|
||||
``AccessReader.list_user_tables``.
|
||||
"""
|
||||
exclude = set(fm.exclude_tables or [])
|
||||
include = set(fm.include_tables) if fm.include_tables else None
|
||||
out = []
|
||||
for t in reader.list_user_tables():
|
||||
if t in exclude:
|
||||
continue
|
||||
if include is not None and t not in include:
|
||||
continue
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def full_sync_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter) -> dict:
|
||||
"""Full-sync every eligible table in one Access file. Returns a summary."""
|
||||
summary = {"file": fm.file, "tables": 0, "rows": 0, "skipped": []}
|
||||
for access_table in resolve_tables(reader, fm):
|
||||
target = fm.target_table(access_table)
|
||||
if not writer.table_exists(fm.schema, target):
|
||||
log.warning(
|
||||
"skip %s -> %s.%s (target table missing)",
|
||||
access_table, fm.schema, target,
|
||||
)
|
||||
summary["skipped"].append(access_table)
|
||||
continue
|
||||
try:
|
||||
cols, rows = reader.read_all_rows(access_table)
|
||||
writer.truncate_target(fm.schema, target)
|
||||
n = writer.bulk_insert(fm.schema, target, cols, rows)
|
||||
summary["tables"] += 1
|
||||
summary["rows"] += n
|
||||
log.info(
|
||||
"full-synced %s -> %s.%s : %d rows",
|
||||
access_table, fm.schema, target, n,
|
||||
)
|
||||
except Exception:
|
||||
log.exception(
|
||||
"full sync failed for %s -> %s.%s",
|
||||
access_table, fm.schema, target,
|
||||
)
|
||||
summary["skipped"].append(access_table)
|
||||
return summary
|
||||
|
||||
|
||||
def full_sync(
|
||||
cfg: SyncConfig,
|
||||
db_filter: str | None = None,
|
||||
table_filter: str | None = None,
|
||||
clear_change_log: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Run a full sync across all (optionally filtered) files.
|
||||
|
||||
``db_filter`` limits to a single Access file (matched on ``FileMapping.file``).
|
||||
``table_filter`` restricts every considered file to that one table by
|
||||
overriding ``include_tables``. ``clear_change_log`` additionally empties the
|
||||
``TableChangeLog`` on each synced file after loading (use with care — it
|
||||
mutates the Access side so the incremental service won't replay old deltas).
|
||||
"""
|
||||
files = cfg.files
|
||||
if db_filter:
|
||||
files = [f for f in files if f.file == db_filter]
|
||||
if not files:
|
||||
log.warning("no file matches --db %r", db_filter)
|
||||
return []
|
||||
if table_filter:
|
||||
files = [f.model_copy(update={"include_tables": [table_filter]}) for f in files]
|
||||
|
||||
writer = SqlWriter(cfg.sql_server.conn_str, cfg.sql_server.sync_queue_table)
|
||||
summaries = []
|
||||
try:
|
||||
for fm in files:
|
||||
reader = AccessReader(fm.source_path(cfg), cfg.access.driver)
|
||||
try:
|
||||
summary = full_sync_file(fm, reader, writer)
|
||||
summaries.append(summary)
|
||||
if clear_change_log:
|
||||
try:
|
||||
ids = reader.read_all_log_ids()
|
||||
deleted = reader.delete_log_ids(
|
||||
ids, 500, 3
|
||||
) if ids else 0
|
||||
log.info("cleared %d TableChangeLog rows from %s", deleted, fm.file)
|
||||
except Exception:
|
||||
log.exception("clear-change-log failed for %s", fm.file)
|
||||
finally:
|
||||
reader.close()
|
||||
finally:
|
||||
writer.close()
|
||||
return summaries
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point: ``python -m sync.fullsync config.yaml [options]``."""
|
||||
ap = argparse.ArgumentParser(
|
||||
description="One-shot full sync: Access -> SQL Server (TRUNCATE + bulk INSERT)."
|
||||
)
|
||||
ap.add_argument("config", nargs="?", default="config.yaml",
|
||||
help="path to config.yaml (default: config.yaml)")
|
||||
ap.add_argument("--db", help="limit to one Access file (by FileMapping.file)")
|
||||
ap.add_argument("--table", help="limit to one table (applies to all matched files)")
|
||||
ap.add_argument("--clear-change-log", action="store_true",
|
||||
help="after loading, clear TableChangeLog on the synced files")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
setup_logging(cfg.logging)
|
||||
|
||||
summaries = full_sync(
|
||||
cfg,
|
||||
db_filter=args.db,
|
||||
table_filter=args.table,
|
||||
clear_change_log=args.clear_change_log,
|
||||
)
|
||||
|
||||
total_tables = sum(s["tables"] for s in summaries)
|
||||
total_rows = sum(s["rows"] for s in summaries)
|
||||
log.info("FULL SYNC COMPLETE: %d tables, %d rows", total_tables, total_rows)
|
||||
for s in summaries:
|
||||
if s["skipped"]:
|
||||
log.warning(" %s: skipped %s", s["file"], s["skipped"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user