feat: migrate configuration to .env environment variables
This commit implements a complete migration from JSON-based configuration to .env environment variables, providing better security and flexibility. Key Changes: - Add python-dotenv dependency for environment variable support - Create config/env_loader.py with type conversion utilities - Add from_env() class methods to all config dataclasses - Update ConfigLoader to prioritize environment variables - Add save_to_env() method for .env file management - Implement database connection factory pattern - Add base DAO and connection classes for better abstraction - Support both SQL Server and MySQL with unified interface - Create migration script (scripts/migrate_to_env.py) - Update GUI to read/write .env files - Add comprehensive migration documentation New Files: - config/env_loader.py - Environment variable loader - db/base_connection.py - Base database connection interface - db/base_dao.py - Base DAO with common utilities - db/connection_factory.py - Factory for creating connections - db/mysql_connection.py - MySQL-specific connection - db/sqlserver_connection.py - SQL Server-specific connection - db/table_name_converter.py - SQL dialect converter - scripts/migrate_to_env.py - Configuration migration tool - docs/ENV_MIGRATION.md - Complete migration guide - .env.example - Environment variable template Testing: - Verified MySQL connection (8.0.44) - Tested all DAO operations - Confirmed 150 tables accessible - Validated configuration loading Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
118
config/loader.py
118
config/loader.py
@@ -3,29 +3,51 @@
|
||||
"""
|
||||
配置加载器
|
||||
|
||||
负责加载、合并和验证配置。
|
||||
负责加载、合并和验证配置,优先从环境变量加载。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
from config.schema import AppConfig
|
||||
from config.schema import (
|
||||
AppConfig,
|
||||
ERPConfig,
|
||||
DatabaseConfig,
|
||||
PathConfig,
|
||||
ExtractionConfig,
|
||||
ValidationConfig,
|
||||
DatabaseType,
|
||||
SQLServerConfig,
|
||||
MySQLConfig,
|
||||
)
|
||||
from config.defaults import DEFAULT_APP_CONFIG, DEFAULT_SETTINGS_DICT
|
||||
from config.env_loader import get_env, get_env_bool, get_env_int
|
||||
|
||||
|
||||
class ConfigLoader:
|
||||
"""配置加载器"""
|
||||
|
||||
@staticmethod
|
||||
def load(config_file: str = "config/user_settings.json") -> AppConfig:
|
||||
def load(config_file: str = "config/user_settings.json", use_env: bool = True) -> AppConfig:
|
||||
"""
|
||||
加载配置文件
|
||||
加载配置
|
||||
|
||||
优先级:
|
||||
1. 环境变量(如果 use_env=True)
|
||||
2. JSON 配置文件(如果存在)
|
||||
3. 默认配置
|
||||
|
||||
Args:
|
||||
config_file: 配置文件路径
|
||||
use_env: 是否使用环境变量,默认为 True
|
||||
|
||||
Returns:
|
||||
应用配置对象
|
||||
"""
|
||||
# 优先从环境变量加载
|
||||
if use_env:
|
||||
return AppConfig.from_env()
|
||||
|
||||
# 如果不使用环境变量,则从 JSON 文件加载(向后兼容)
|
||||
if os.path.exists(config_file):
|
||||
try:
|
||||
with open(config_file, "r", encoding="utf-8") as f:
|
||||
@@ -66,6 +88,63 @@ class ConfigLoader:
|
||||
print(f"保存配置文件失败: {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def save_to_env(config: AppConfig, env_file: str = ".env") -> bool:
|
||||
"""
|
||||
保存配置到 .env 文件
|
||||
|
||||
Args:
|
||||
config: 应用配置对象
|
||||
env_file: .env 文件路径
|
||||
|
||||
Returns:
|
||||
保存是否成功
|
||||
"""
|
||||
from config.env_loader import save_env_file
|
||||
|
||||
env_dict = {
|
||||
# ERP 配置
|
||||
"ERP_URL": config.erp.url,
|
||||
"ERP_USERNAME": config.erp.username,
|
||||
"ERP_PASSWORD": config.erp.password,
|
||||
"ERP_HEADLESS": config.erp.headless,
|
||||
"ERP_IGNORE_HTTPS_ERRORS": config.erp.ignore_https_errors,
|
||||
"ERP_AUTO_CLOSE_BROWSER": config.erp.auto_close_browser,
|
||||
# 数据库配置
|
||||
"DB_TYPE": config.database.db_type.value,
|
||||
"DB_SERVER": config.database.server,
|
||||
"DB_NAME": config.database.database,
|
||||
"DB_USERNAME": config.database.username,
|
||||
"DB_PASSWORD": config.database.password,
|
||||
# SQL Server 特定配置
|
||||
"DB_SQLSERVER_DRIVER": config.database.sqlserver.driver if config.database.sqlserver else "ODBC Driver 18 for SQL Server",
|
||||
"DB_TRUST_SERVER_CERTIFICATE": config.database.sqlserver.trust_server_certificate if config.database.sqlserver else "yes",
|
||||
# MySQL 特定配置
|
||||
"DB_MYSQL_HOST": config.database.mysql.host if config.database.mysql else "",
|
||||
"DB_MYSQL_PORT": config.database.mysql.port if config.database.mysql else 3306,
|
||||
"DB_MYSQL_CHARSET": config.database.mysql.charset if config.database.mysql else "utf8mb4",
|
||||
# 路径配置
|
||||
"PATH_DATA_DIR": config.paths.data_dir,
|
||||
"PATH_PRODUCTION_ID_FILE": config.paths.production_id_file,
|
||||
"PATH_DEFAULT_OUTPUT": config.paths.default_output,
|
||||
"PATH_VALIDATION_OUTPUT": config.paths.validation_output,
|
||||
# 数据提取配置
|
||||
"EXTRACTION_BATCH_SIZE": config.extraction.batch_size,
|
||||
"EXTRACTION_VERBOSE": config.extraction.verbose,
|
||||
"EXTRACTION_AUTO_CONVERT": config.extraction.auto_convert,
|
||||
"EXTRACTION_MERGE_BATCHES": config.extraction.merge_batches,
|
||||
"EXTRACTION_ENABLE_DB_PERSISTENCE": config.extraction.enable_db_persistence,
|
||||
# 校验配置
|
||||
"VALIDATION_DATA_SOURCE": config.validation.data_source,
|
||||
"VALIDATION_USE_DATABASE": config.validation.use_database,
|
||||
"VALIDATION_BATCH_SIZE": config.validation.batch_size,
|
||||
"VALIDATION_ENABLE_CRUD": config.validation.enable_crud_operations,
|
||||
"VALIDATION_DEFAULT_MANAGER": config.validation.default_manager,
|
||||
"VALIDATION_MATCH_MODE": config.validation.match_mode,
|
||||
}
|
||||
|
||||
return save_env_file(env_file, env_dict)
|
||||
|
||||
@staticmethod
|
||||
def _merge_settings(defaults: Dict, loaded: Dict) -> Dict:
|
||||
"""
|
||||
@@ -109,6 +188,28 @@ class ConfigLoader:
|
||||
extraction_dict = settings.get("extraction", {})
|
||||
validation_dict = settings.get("validation", {})
|
||||
|
||||
# 解析数据库类型
|
||||
db_type_str = database_dict.get("db_type", "sqlserver")
|
||||
try:
|
||||
db_type = DatabaseType(db_type_str)
|
||||
except ValueError:
|
||||
db_type = DatabaseType.SQLSERVER
|
||||
|
||||
# 解析 SQL Server 配置
|
||||
sqlserver_dict = database_dict.get("sqlserver", {})
|
||||
sqlserver_config = SQLServerConfig(
|
||||
driver=sqlserver_dict.get("driver", "ODBC Driver 18 for SQL Server"),
|
||||
trust_server_certificate=sqlserver_dict.get("trust_server_certificate", "yes"),
|
||||
)
|
||||
|
||||
# 解析 MySQL 配置
|
||||
mysql_dict = database_dict.get("mysql", {})
|
||||
mysql_config = MySQLConfig(
|
||||
host=mysql_dict.get("host", database_dict.get("server", "")),
|
||||
port=mysql_dict.get("port", 3306),
|
||||
charset=mysql_dict.get("charset", "utf8mb4"),
|
||||
)
|
||||
|
||||
return AppConfig(
|
||||
erp=ERPConfig(
|
||||
url=erp_dict.get("url", ""),
|
||||
@@ -119,14 +220,13 @@ class ConfigLoader:
|
||||
auto_close_browser=erp_dict.get("auto_close_browser", True),
|
||||
),
|
||||
database=DatabaseConfig(
|
||||
db_type=db_type,
|
||||
server=database_dict.get("server", ""),
|
||||
database=database_dict.get("database", ""),
|
||||
username=database_dict.get("username", ""),
|
||||
password=database_dict.get("password", ""),
|
||||
driver=database_dict.get("driver", "ODBC Driver 18 for SQL Server"),
|
||||
trust_server_certificate=database_dict.get(
|
||||
"trust_server_certificate", "yes"
|
||||
),
|
||||
sqlserver=sqlserver_config,
|
||||
mysql=mysql_config,
|
||||
),
|
||||
paths=PathConfig(
|
||||
data_dir=paths_dict.get("data_dir", ""),
|
||||
@@ -156,5 +256,3 @@ class ConfigLoader:
|
||||
)
|
||||
|
||||
|
||||
# 为了兼容旧代码,导入必要的类型
|
||||
from config.schema import ERPConfig, DatabaseConfig, PathConfig, ExtractionConfig, ValidationConfig
|
||||
|
||||
Reference in New Issue
Block a user