feat(sync): add one-shot full sync (TRUNCATE + bulk INSERT)

- New `sync.fullsync` CLI: `python -m sync.fullsync config.yaml [--db F] [--table T] [--clear-change-log]`
- AccessReader: list_user_tables / read_all_rows / read_all_log_ids
- SqlWriter: table_exists / truncate_target (TRUNCATE w/ DELETE fallback) / bulk_insert (SET IDENTITY_INSERT + chunked fast_executemany)
- Reuses FileMapping exclude/include rules (exclude beats include, same as capture)
- Preserves Access IDs via IDENTITY_INSERT; target schemas have no FKs so TRUNCATE is safe
- TableChangeLog NOT cleared by default (opt-in --clear-change-log)
- tests/test_fullsync.py covers resolve_tables exclude/include precedence
This commit is contained in:
Misaka_Company
2026-07-14 18:06:39 +08:00
parent ea48de290f
commit a227aed0cb
4 changed files with 339 additions and 0 deletions

View File

@@ -132,6 +132,93 @@ class SqlWriter:
)
return cur.rowcount
def table_exists(self, schema: str, table: str) -> bool:
"""True if ``[schema].[table]`` exists in the target database."""
cur = self._conn.cursor()
cur.execute(
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
"ON s.schema_id = t.schema_id "
"WHERE s.name = ? AND t.name = ?",
schema, table,
)
return cur.fetchone() is not None
def _has_identity(self, schema: str, table: str) -> bool:
"""True if ``[schema].[table]`` has an IDENTITY column (the ``ID`` PK)."""
cur = self._conn.cursor()
cur.execute(
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
"ON s.schema_id = t.schema_id JOIN sys.columns c "
"ON c.object_id = t.object_id "
"WHERE s.name = ? AND t.name = ? AND c.is_identity = 1",
schema, table,
)
return cur.fetchone() is not None
def truncate_target(self, schema: str, table: str) -> None:
"""Empty ``[schema].[table]`` prior to a full reload.
Prefers ``TRUNCATE TABLE`` (fast, minimal logging). This project's
target schemas have no foreign keys, so TRUNCATE always succeeds; the
``DELETE FROM`` fallback only matters if a future FK is added or the
table sits under snapshot isolation.
"""
full = f"[{schema}].[{table}]"
cur = self._conn.cursor()
try:
cur.execute(f"TRUNCATE TABLE {full}")
except pyodbc.Error:
cur.execute(f"DELETE FROM {full}")
def bulk_insert(
self, schema: str, table: str, columns: list[str], rows: list[tuple]
) -> int:
"""Insert every ``row`` into ``[schema].[table]``.
``columns`` and the tuples in ``rows`` must be positionally aligned.
When the target has an IDENTITY column (the ``ID`` PK in every target
here), ``SET IDENTITY_INSERT`` is enabled so the original Access primary
keys are preserved — required for the incremental sync's RecordID
matching to keep working afterwards.
Rows are inserted in 1000-row chunks. ``fast_executemany`` is tried
first for speed; if a row's types can't be inferred (e.g. a leading
NULL), it transparently retries the chunk without it. Returns the total
number of rows inserted.
"""
if not columns or not rows:
return 0
full = f"[{schema}].[{table}]"
col_list = ", ".join(f"[{c}]" for c in columns)
placeholders = ", ".join("?" for _ in columns)
has_identity = self._has_identity(schema, table)
cur = self._conn.cursor()
if has_identity:
cur.execute(f"SET IDENTITY_INSERT {full} ON")
try:
inserted = 0
fast = True
for i in range(0, len(rows), 1000):
chunk = rows[i:i + 1000]
for attempt in range(2):
try:
cur.fast_executemany = fast
cur.executemany(
f"INSERT INTO {full} ({col_list}) VALUES ({placeholders})",
chunk,
)
break
except pyodbc.Error:
if attempt == 0 and fast:
fast = False # retry this chunk without fast_executemany
continue
raise
inserted += len(chunk)
return inserted
finally:
if has_identity:
cur.execute(f"SET IDENTITY_INSERT {full} OFF")
def close(self) -> None:
"""Close the underlying pyodbc connection."""
self._conn.close()