feat: migrate configuration to .env environment variables

This commit implements a complete migration from JSON-based configuration
to .env environment variables, providing better security and flexibility.

Key Changes:
- Add python-dotenv dependency for environment variable support
- Create config/env_loader.py with type conversion utilities
- Add from_env() class methods to all config dataclasses
- Update ConfigLoader to prioritize environment variables
- Add save_to_env() method for .env file management
- Implement database connection factory pattern
- Add base DAO and connection classes for better abstraction
- Support both SQL Server and MySQL with unified interface
- Create migration script (scripts/migrate_to_env.py)
- Update GUI to read/write .env files
- Add comprehensive migration documentation

New Files:
- config/env_loader.py - Environment variable loader
- db/base_connection.py - Base database connection interface
- db/base_dao.py - Base DAO with common utilities
- db/connection_factory.py - Factory for creating connections
- db/mysql_connection.py - MySQL-specific connection
- db/sqlserver_connection.py - SQL Server-specific connection
- db/table_name_converter.py - SQL dialect converter
- scripts/migrate_to_env.py - Configuration migration tool
- docs/ENV_MIGRATION.md - Complete migration guide
- .env.example - Environment variable template

Testing:
- Verified MySQL connection (8.0.44)
- Tested all DAO operations
- Confirmed 150 tables accessible
- Validated configuration loading

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Misaka
2026-02-09 22:39:14 +08:00
parent 04b99292ad
commit aaa46ef282
22 changed files with 2545 additions and 563 deletions

View File

@@ -3,29 +3,51 @@
"""
配置加载器
负责加载、合并和验证配置。
负责加载、合并和验证配置,优先从环境变量加载
"""
import json
import os
from typing import Any, Dict
from config.schema import AppConfig
from config.schema import (
AppConfig,
ERPConfig,
DatabaseConfig,
PathConfig,
ExtractionConfig,
ValidationConfig,
DatabaseType,
SQLServerConfig,
MySQLConfig,
)
from config.defaults import DEFAULT_APP_CONFIG, DEFAULT_SETTINGS_DICT
from config.env_loader import get_env, get_env_bool, get_env_int
class ConfigLoader:
"""配置加载器"""
@staticmethod
def load(config_file: str = "config/user_settings.json") -> AppConfig:
def load(config_file: str = "config/user_settings.json", use_env: bool = True) -> AppConfig:
"""
加载配置文件
加载配置
优先级:
1. 环境变量(如果 use_env=True
2. JSON 配置文件(如果存在)
3. 默认配置
Args:
config_file: 配置文件路径
use_env: 是否使用环境变量,默认为 True
Returns:
应用配置对象
"""
# 优先从环境变量加载
if use_env:
return AppConfig.from_env()
# 如果不使用环境变量,则从 JSON 文件加载(向后兼容)
if os.path.exists(config_file):
try:
with open(config_file, "r", encoding="utf-8") as f:
@@ -66,6 +88,63 @@ class ConfigLoader:
print(f"保存配置文件失败: {e}")
return False
@staticmethod
def save_to_env(config: AppConfig, env_file: str = ".env") -> bool:
"""
保存配置到 .env 文件
Args:
config: 应用配置对象
env_file: .env 文件路径
Returns:
保存是否成功
"""
from config.env_loader import save_env_file
env_dict = {
# ERP 配置
"ERP_URL": config.erp.url,
"ERP_USERNAME": config.erp.username,
"ERP_PASSWORD": config.erp.password,
"ERP_HEADLESS": config.erp.headless,
"ERP_IGNORE_HTTPS_ERRORS": config.erp.ignore_https_errors,
"ERP_AUTO_CLOSE_BROWSER": config.erp.auto_close_browser,
# 数据库配置
"DB_TYPE": config.database.db_type.value,
"DB_SERVER": config.database.server,
"DB_NAME": config.database.database,
"DB_USERNAME": config.database.username,
"DB_PASSWORD": config.database.password,
# SQL Server 特定配置
"DB_SQLSERVER_DRIVER": config.database.sqlserver.driver if config.database.sqlserver else "ODBC Driver 18 for SQL Server",
"DB_TRUST_SERVER_CERTIFICATE": config.database.sqlserver.trust_server_certificate if config.database.sqlserver else "yes",
# MySQL 特定配置
"DB_MYSQL_HOST": config.database.mysql.host if config.database.mysql else "",
"DB_MYSQL_PORT": config.database.mysql.port if config.database.mysql else 3306,
"DB_MYSQL_CHARSET": config.database.mysql.charset if config.database.mysql else "utf8mb4",
# 路径配置
"PATH_DATA_DIR": config.paths.data_dir,
"PATH_PRODUCTION_ID_FILE": config.paths.production_id_file,
"PATH_DEFAULT_OUTPUT": config.paths.default_output,
"PATH_VALIDATION_OUTPUT": config.paths.validation_output,
# 数据提取配置
"EXTRACTION_BATCH_SIZE": config.extraction.batch_size,
"EXTRACTION_VERBOSE": config.extraction.verbose,
"EXTRACTION_AUTO_CONVERT": config.extraction.auto_convert,
"EXTRACTION_MERGE_BATCHES": config.extraction.merge_batches,
"EXTRACTION_ENABLE_DB_PERSISTENCE": config.extraction.enable_db_persistence,
# 校验配置
"VALIDATION_DATA_SOURCE": config.validation.data_source,
"VALIDATION_USE_DATABASE": config.validation.use_database,
"VALIDATION_BATCH_SIZE": config.validation.batch_size,
"VALIDATION_ENABLE_CRUD": config.validation.enable_crud_operations,
"VALIDATION_DEFAULT_MANAGER": config.validation.default_manager,
"VALIDATION_MATCH_MODE": config.validation.match_mode,
}
return save_env_file(env_file, env_dict)
@staticmethod
def _merge_settings(defaults: Dict, loaded: Dict) -> Dict:
"""
@@ -109,6 +188,28 @@ class ConfigLoader:
extraction_dict = settings.get("extraction", {})
validation_dict = settings.get("validation", {})
# 解析数据库类型
db_type_str = database_dict.get("db_type", "sqlserver")
try:
db_type = DatabaseType(db_type_str)
except ValueError:
db_type = DatabaseType.SQLSERVER
# 解析 SQL Server 配置
sqlserver_dict = database_dict.get("sqlserver", {})
sqlserver_config = SQLServerConfig(
driver=sqlserver_dict.get("driver", "ODBC Driver 18 for SQL Server"),
trust_server_certificate=sqlserver_dict.get("trust_server_certificate", "yes"),
)
# 解析 MySQL 配置
mysql_dict = database_dict.get("mysql", {})
mysql_config = MySQLConfig(
host=mysql_dict.get("host", database_dict.get("server", "")),
port=mysql_dict.get("port", 3306),
charset=mysql_dict.get("charset", "utf8mb4"),
)
return AppConfig(
erp=ERPConfig(
url=erp_dict.get("url", ""),
@@ -119,14 +220,13 @@ class ConfigLoader:
auto_close_browser=erp_dict.get("auto_close_browser", True),
),
database=DatabaseConfig(
db_type=db_type,
server=database_dict.get("server", ""),
database=database_dict.get("database", ""),
username=database_dict.get("username", ""),
password=database_dict.get("password", ""),
driver=database_dict.get("driver", "ODBC Driver 18 for SQL Server"),
trust_server_certificate=database_dict.get(
"trust_server_certificate", "yes"
),
sqlserver=sqlserver_config,
mysql=mysql_config,
),
paths=PathConfig(
data_dir=paths_dict.get("data_dir", ""),
@@ -156,5 +256,3 @@ class ConfigLoader:
)
# 为了兼容旧代码,导入必要的类型
from config.schema import ERPConfig, DatabaseConfig, PathConfig, ExtractionConfig, ValidationConfig