初始化项目:数据同步服务源码与文档

- src: config_loader / source_watcher / excel_parser / sync_writer / runner
- 169 Excel 全量同步至 114 SQL Server + Access 生产库
- 新增 .gitignore(忽略 venv、日志、数据、含凭据的 config.yaml 等)
- 新增 README.md(用途、安装、配置、运行模式、部署说明)
This commit is contained in:
Misaka_Company
2026-07-15 14:14:39 +08:00
commit 5ae7acffde
9 changed files with 557 additions and 0 deletions

87
src/sync_writer.py Normal file
View File

@@ -0,0 +1,87 @@
"""全量同步写入SQL Server (TRUNCATE+INSERT) + Access (DELETE+INSERT)。"""
import logging
from datetime import datetime
import pyodbc
from .config_loader import build_sql_connstr, build_access_connstr
logger = logging.getLogger(__name__)
def _build_insert(table_expr, fields, import_col):
"""构造 INSERT SQL。table_expr 为完整表表达式(含 schema/方括号)。"""
cols = list(fields) + [import_col]
col_list = ",".join(f"[{c}]" for c in cols)
placeholders = ",".join(["?"] * len(cols))
return f"INSERT INTO {table_expr} ({col_list}) VALUES ({placeholders})"
def _to_rows(records, fields, import_time):
return [tuple(r.get(c) for c in fields) + (import_time,) for r in records]
def sync_sql_server(cfg, records, fields, import_time):
if not records:
logger.warning("SQL Server: 无数据,跳过")
return
s = cfg["sql_server"]
table_expr = f"{s['schema']}.[{s['table']}]" # procurementVisibilityHub.[请购执行]
import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间")
insert_sql = _build_insert(table_expr, fields, import_col)
rows = _to_rows(records, fields, import_time)
batch = cfg.get("sync", {}).get("batch_size", 1000)
cn = pyodbc.connect(build_sql_connstr(cfg), autocommit=False)
try:
cur = cn.cursor()
logger.info("SQL Server: 清空 %s", table_expr)
try:
cur.execute(f"TRUNCATE TABLE {table_expr}")
except pyodbc.Error as e:
logger.warning("TRUNCATE 失败(%s),改用 DELETE", e)
cur.execute(f"DELETE FROM {table_expr}")
cur.fast_executemany = True
for i in range(0, len(rows), batch):
cur.executemany(insert_sql, rows[i:i + batch])
logger.info("SQL Server: 已插入 %d/%d", min(i + batch, len(rows)), len(rows))
cn.commit()
logger.info("SQL Server: 完成,共 %d", len(rows))
except Exception:
cn.rollback()
raise
finally:
cn.close()
def sync_access(cfg, records, fields, import_time):
if not records:
logger.warning("Access: 无数据,跳过")
return
a = cfg["access"]
table_expr = f"[{a['table']}]" # [procurementVisibilityHub_请购执行]
import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间")
insert_sql = _build_insert(table_expr, fields, import_col)
rows = _to_rows(records, fields, import_time)
batch = cfg.get("sync", {}).get("batch_size", 1000)
cn = pyodbc.connect(build_access_connstr(cfg), autocommit=True)
try:
cur = cn.cursor()
logger.info("Access: 清空 %s", table_expr)
cur.execute(f"DELETE FROM {table_expr}")
total = len(rows)
for i in range(0, total, batch):
cur.executemany(insert_sql, rows[i:i + batch])
logger.info("Access: 已插入 %d/%d", min(i + batch, total), total)
logger.info("Access: 完成,共 %d", total)
finally:
cn.close()
def sync_all(cfg, records, fields):
import_time = datetime.now().replace(microsecond=0)
logger.info("=== 开始全量同步 导入时间=%s 数据=%d 行 ===", import_time, len(records))
sync_sql_server(cfg, records, fields, import_time)
sync_access(cfg, records, fields, import_time)
logger.info("=== 同步完成 ===")