fix(sync): track cleanup state to stop re-deleting log rows and bound SyncQueue

- access_reader.delete_log_ids returns the actual rows deleted (was None).

- sql_writer.mark_cleaned flips applied queue rows to 'cleaned' (sets CleanedAt) after their Access log rows are physically removed, so the same IDs are never deleted twice.

- sql_writer.purge_cleaned removes 'cleaned' rows older than a retention window (default 24h) so SyncQueue stops growing without bound.

- cleanup.cleanup_file marks rows cleaned after a successful delete and returns the real delete count, so the service log reports honest 'cleaned N' instead of a constant.

- service.cycle calls purge_cleaned once per pass; config adds cleaned_retention_hours (default 24).

- sql/01_sync_queue.sql adds CleanedAt column + IX_SyncQueue_Cleaned idempotently.

- tests: unit coverage for mark_cleaned/purge_cleaned/delete_log_ids return count; assert cycle purges each pass.
This commit is contained in:
Misaka_Company
2026-07-14 16:32:47 +08:00
parent b1b118463d
commit ea48de290f
8 changed files with 164 additions and 9 deletions

View File

@@ -90,6 +90,48 @@ class SqlWriter:
)
return [r[0] for r in cur.fetchall()]
def mark_cleaned(self, source_file: str, source_log_ids: list[int]) -> None:
"""Mark applied queue rows for ``source_file`` as ``cleaned``.
Called after the corresponding Access log rows have been physically
deleted. Once a row is ``cleaned``, ``applied_log_ids`` no longer
returns it, so the same IDs are never deleted from Access twice. Rows
are matched on ``(SourceFile, SourceLogID)`` and constrained to
``Status='applied'``, so a row that errored out is never silently
marked clean. Chunked to stay under SQL Server's 2100-param limit.
"""
if not source_log_ids:
return
cur = self._conn.cursor()
for i in range(0, len(source_log_ids), 1000):
chunk = source_log_ids[i:i + 1000]
placeholders = ",".join("?" * len(chunk))
cur.execute(
f"UPDATE dbo.SyncQueue SET Status='cleaned', "
f"CleanedAt=sysdatetime() "
f"WHERE SourceFile=? AND Status='applied' "
f"AND SourceLogID IN ({placeholders})",
source_file,
*chunk,
)
def purge_cleaned(self, retention_hours: int) -> int:
"""Delete ``cleaned`` rows older than ``retention_hours``.
Keeps the table bounded: cleanup marks rows ``cleaned`` every cycle,
and this removes the old ones after a short audit/debug window so
``dbo.SyncQueue`` stops growing without bound. Returns the number of
rows removed.
"""
cur = self._conn.cursor()
cur.execute(
"DELETE FROM dbo.SyncQueue "
"WHERE Status='cleaned' "
"AND CleanedAt < DATEADD(hour, -?, GETDATE())",
retention_hours,
)
return cur.rowcount
def close(self) -> None:
"""Close the underlying pyodbc connection."""
self._conn.close()