初始化项目:数据同步服务源码与文档
- src: config_loader / source_watcher / excel_parser / sync_writer / runner - 169 Excel 全量同步至 114 SQL Server + Access 生产库 - 新增 .gitignore(忽略 venv、日志、数据、含凭据的 config.yaml 等) - 新增 README.md(用途、安装、配置、运行模式、部署说明)
This commit is contained in:
87
src/sync_writer.py
Normal file
87
src/sync_writer.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""全量同步写入:SQL Server (TRUNCATE+INSERT) + Access (DELETE+INSERT)。"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
import pyodbc
|
||||
|
||||
from .config_loader import build_sql_connstr, build_access_connstr
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_insert(table_expr, fields, import_col):
|
||||
"""构造 INSERT SQL。table_expr 为完整表表达式(含 schema/方括号)。"""
|
||||
cols = list(fields) + [import_col]
|
||||
col_list = ",".join(f"[{c}]" for c in cols)
|
||||
placeholders = ",".join(["?"] * len(cols))
|
||||
return f"INSERT INTO {table_expr} ({col_list}) VALUES ({placeholders})"
|
||||
|
||||
|
||||
def _to_rows(records, fields, import_time):
|
||||
return [tuple(r.get(c) for c in fields) + (import_time,) for r in records]
|
||||
|
||||
|
||||
def sync_sql_server(cfg, records, fields, import_time):
|
||||
if not records:
|
||||
logger.warning("SQL Server: 无数据,跳过")
|
||||
return
|
||||
s = cfg["sql_server"]
|
||||
table_expr = f"{s['schema']}.[{s['table']}]" # procurementVisibilityHub.[请购执行]
|
||||
import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间")
|
||||
insert_sql = _build_insert(table_expr, fields, import_col)
|
||||
rows = _to_rows(records, fields, import_time)
|
||||
batch = cfg.get("sync", {}).get("batch_size", 1000)
|
||||
|
||||
cn = pyodbc.connect(build_sql_connstr(cfg), autocommit=False)
|
||||
try:
|
||||
cur = cn.cursor()
|
||||
logger.info("SQL Server: 清空 %s", table_expr)
|
||||
try:
|
||||
cur.execute(f"TRUNCATE TABLE {table_expr}")
|
||||
except pyodbc.Error as e:
|
||||
logger.warning("TRUNCATE 失败(%s),改用 DELETE", e)
|
||||
cur.execute(f"DELETE FROM {table_expr}")
|
||||
cur.fast_executemany = True
|
||||
for i in range(0, len(rows), batch):
|
||||
cur.executemany(insert_sql, rows[i:i + batch])
|
||||
logger.info("SQL Server: 已插入 %d/%d", min(i + batch, len(rows)), len(rows))
|
||||
cn.commit()
|
||||
logger.info("SQL Server: 完成,共 %d 行", len(rows))
|
||||
except Exception:
|
||||
cn.rollback()
|
||||
raise
|
||||
finally:
|
||||
cn.close()
|
||||
|
||||
|
||||
def sync_access(cfg, records, fields, import_time):
|
||||
if not records:
|
||||
logger.warning("Access: 无数据,跳过")
|
||||
return
|
||||
a = cfg["access"]
|
||||
table_expr = f"[{a['table']}]" # [procurementVisibilityHub_请购执行]
|
||||
import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间")
|
||||
insert_sql = _build_insert(table_expr, fields, import_col)
|
||||
rows = _to_rows(records, fields, import_time)
|
||||
batch = cfg.get("sync", {}).get("batch_size", 1000)
|
||||
|
||||
cn = pyodbc.connect(build_access_connstr(cfg), autocommit=True)
|
||||
try:
|
||||
cur = cn.cursor()
|
||||
logger.info("Access: 清空 %s", table_expr)
|
||||
cur.execute(f"DELETE FROM {table_expr}")
|
||||
total = len(rows)
|
||||
for i in range(0, total, batch):
|
||||
cur.executemany(insert_sql, rows[i:i + batch])
|
||||
logger.info("Access: 已插入 %d/%d", min(i + batch, total), total)
|
||||
logger.info("Access: 完成,共 %d 行", total)
|
||||
finally:
|
||||
cn.close()
|
||||
|
||||
|
||||
def sync_all(cfg, records, fields):
|
||||
import_time = datetime.now().replace(microsecond=0)
|
||||
logger.info("=== 开始全量同步 导入时间=%s 数据=%d 行 ===", import_time, len(records))
|
||||
sync_sql_server(cfg, records, fields, import_time)
|
||||
sync_access(cfg, records, fields, import_time)
|
||||
logger.info("=== 同步完成 ===")
|
||||
Reference in New Issue
Block a user