feat(sync): add one-shot full sync (TRUNCATE + bulk INSERT)
- New `sync.fullsync` CLI: `python -m sync.fullsync config.yaml [--db F] [--table T] [--clear-change-log]` - AccessReader: list_user_tables / read_all_rows / read_all_log_ids - SqlWriter: table_exists / truncate_target (TRUNCATE w/ DELETE fallback) / bulk_insert (SET IDENTITY_INSERT + chunked fast_executemany) - Reuses FileMapping exclude/include rules (exclude beats include, same as capture) - Preserves Access IDs via IDENTITY_INSERT; target schemas have no FKs so TRUNCATE is safe - TableChangeLog NOT cleared by default (opt-in --clear-change-log) - tests/test_fullsync.py covers resolve_tables exclude/include precedence
This commit is contained in:
@@ -132,6 +132,93 @@ class SqlWriter:
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
def table_exists(self, schema: str, table: str) -> bool:
|
||||
"""True if ``[schema].[table]`` exists in the target database."""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
|
||||
"ON s.schema_id = t.schema_id "
|
||||
"WHERE s.name = ? AND t.name = ?",
|
||||
schema, table,
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def _has_identity(self, schema: str, table: str) -> bool:
|
||||
"""True if ``[schema].[table]`` has an IDENTITY column (the ``ID`` PK)."""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
|
||||
"ON s.schema_id = t.schema_id JOIN sys.columns c "
|
||||
"ON c.object_id = t.object_id "
|
||||
"WHERE s.name = ? AND t.name = ? AND c.is_identity = 1",
|
||||
schema, table,
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def truncate_target(self, schema: str, table: str) -> None:
|
||||
"""Empty ``[schema].[table]`` prior to a full reload.
|
||||
|
||||
Prefers ``TRUNCATE TABLE`` (fast, minimal logging). This project's
|
||||
target schemas have no foreign keys, so TRUNCATE always succeeds; the
|
||||
``DELETE FROM`` fallback only matters if a future FK is added or the
|
||||
table sits under snapshot isolation.
|
||||
"""
|
||||
full = f"[{schema}].[{table}]"
|
||||
cur = self._conn.cursor()
|
||||
try:
|
||||
cur.execute(f"TRUNCATE TABLE {full}")
|
||||
except pyodbc.Error:
|
||||
cur.execute(f"DELETE FROM {full}")
|
||||
|
||||
def bulk_insert(
|
||||
self, schema: str, table: str, columns: list[str], rows: list[tuple]
|
||||
) -> int:
|
||||
"""Insert every ``row`` into ``[schema].[table]``.
|
||||
|
||||
``columns`` and the tuples in ``rows`` must be positionally aligned.
|
||||
When the target has an IDENTITY column (the ``ID`` PK in every target
|
||||
here), ``SET IDENTITY_INSERT`` is enabled so the original Access primary
|
||||
keys are preserved — required for the incremental sync's RecordID
|
||||
matching to keep working afterwards.
|
||||
|
||||
Rows are inserted in 1000-row chunks. ``fast_executemany`` is tried
|
||||
first for speed; if a row's types can't be inferred (e.g. a leading
|
||||
NULL), it transparently retries the chunk without it. Returns the total
|
||||
number of rows inserted.
|
||||
"""
|
||||
if not columns or not rows:
|
||||
return 0
|
||||
full = f"[{schema}].[{table}]"
|
||||
col_list = ", ".join(f"[{c}]" for c in columns)
|
||||
placeholders = ", ".join("?" for _ in columns)
|
||||
has_identity = self._has_identity(schema, table)
|
||||
cur = self._conn.cursor()
|
||||
if has_identity:
|
||||
cur.execute(f"SET IDENTITY_INSERT {full} ON")
|
||||
try:
|
||||
inserted = 0
|
||||
fast = True
|
||||
for i in range(0, len(rows), 1000):
|
||||
chunk = rows[i:i + 1000]
|
||||
for attempt in range(2):
|
||||
try:
|
||||
cur.fast_executemany = fast
|
||||
cur.executemany(
|
||||
f"INSERT INTO {full} ({col_list}) VALUES ({placeholders})",
|
||||
chunk,
|
||||
)
|
||||
break
|
||||
except pyodbc.Error:
|
||||
if attempt == 0 and fast:
|
||||
fast = False # retry this chunk without fast_executemany
|
||||
continue
|
||||
raise
|
||||
inserted += len(chunk)
|
||||
return inserted
|
||||
finally:
|
||||
if has_identity:
|
||||
cur.execute(f"SET IDENTITY_INSERT {full} OFF")
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the underlying pyodbc connection."""
|
||||
self._conn.close()
|
||||
|
||||
Reference in New Issue
Block a user