fix(sync): track cleanup state to stop re-deleting log rows and bound SyncQueue
- access_reader.delete_log_ids returns the actual rows deleted (was None). - sql_writer.mark_cleaned flips applied queue rows to 'cleaned' (sets CleanedAt) after their Access log rows are physically removed, so the same IDs are never deleted twice. - sql_writer.purge_cleaned removes 'cleaned' rows older than a retention window (default 24h) so SyncQueue stops growing without bound. - cleanup.cleanup_file marks rows cleaned after a successful delete and returns the real delete count, so the service log reports honest 'cleaned N' instead of a constant. - service.cycle calls purge_cleaned once per pass; config adds cleaned_retention_hours (default 24). - sql/01_sync_queue.sql adds CleanedAt column + IX_SyncQueue_Cleaned idempotently. - tests: unit coverage for mark_cleaned/purge_cleaned/delete_log_ids return count; assert cycle purges each pass.
This commit is contained in:
@@ -90,6 +90,48 @@ class SqlWriter:
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
|
||||
def mark_cleaned(self, source_file: str, source_log_ids: list[int]) -> None:
|
||||
"""Mark applied queue rows for ``source_file`` as ``cleaned``.
|
||||
|
||||
Called after the corresponding Access log rows have been physically
|
||||
deleted. Once a row is ``cleaned``, ``applied_log_ids`` no longer
|
||||
returns it, so the same IDs are never deleted from Access twice. Rows
|
||||
are matched on ``(SourceFile, SourceLogID)`` and constrained to
|
||||
``Status='applied'``, so a row that errored out is never silently
|
||||
marked clean. Chunked to stay under SQL Server's 2100-param limit.
|
||||
"""
|
||||
if not source_log_ids:
|
||||
return
|
||||
cur = self._conn.cursor()
|
||||
for i in range(0, len(source_log_ids), 1000):
|
||||
chunk = source_log_ids[i:i + 1000]
|
||||
placeholders = ",".join("?" * len(chunk))
|
||||
cur.execute(
|
||||
f"UPDATE dbo.SyncQueue SET Status='cleaned', "
|
||||
f"CleanedAt=sysdatetime() "
|
||||
f"WHERE SourceFile=? AND Status='applied' "
|
||||
f"AND SourceLogID IN ({placeholders})",
|
||||
source_file,
|
||||
*chunk,
|
||||
)
|
||||
|
||||
def purge_cleaned(self, retention_hours: int) -> int:
|
||||
"""Delete ``cleaned`` rows older than ``retention_hours``.
|
||||
|
||||
Keeps the table bounded: cleanup marks rows ``cleaned`` every cycle,
|
||||
and this removes the old ones after a short audit/debug window so
|
||||
``dbo.SyncQueue`` stops growing without bound. Returns the number of
|
||||
rows removed.
|
||||
"""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"DELETE FROM dbo.SyncQueue "
|
||||
"WHERE Status='cleaned' "
|
||||
"AND CleanedAt < DATEADD(hour, -?, GETDATE())",
|
||||
retention_hours,
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the underlying pyodbc connection."""
|
||||
self._conn.close()
|
||||
|
||||
Reference in New Issue
Block a user