207 lines
7.8 KiB
Python
207 lines
7.8 KiB
Python
|
import pymssql
|
|||
|
|
import mysql.connector
|
|||
|
|
import logging
|
|||
|
|
from multiprocessing import Pool
|
|||
|
|
|
|||
|
|
# 配置日志
|
|||
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|||
|
|
|
|||
|
|
# SQL Server配置
|
|||
|
|
sql_server_config = {
|
|||
|
|
"server": "203.192.15.17", # SQL Server 的 IP 地址
|
|||
|
|
"port": 28063, # SQL Server 的端口
|
|||
|
|
"user": "zncbuser", # 用户名
|
|||
|
|
"password": "ZZB-Cbindex-data", # 密码
|
|||
|
|
"database": "jydb", # 数据库名称
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# MySQL配置
|
|||
|
|
mysql_config = {
|
|||
|
|
"host": "rm-bp1f85h3xs6mvnf5e3o.mysql.rds.aliyuncs.com", # MySQL 的 IP 地址
|
|||
|
|
"user": "zzb_jydb", # 用户名
|
|||
|
|
"password": "Ysdbsdjs89Yrqwp", # 密码
|
|||
|
|
"database": "zzb_jydb", # 数据库名称
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 分批大小（每次查询和插入的行数）
|
|||
|
|
BATCH_SIZE = 100000
|
|||
|
|
|
|||
|
|
# 最大进程数
|
|||
|
|
MAX_PROCESSES = 1
|
|||
|
|
|
|||
|
|
def sync_table(table_name):
|
|||
|
|
try:
|
|||
|
|
# 连接到SQL Server
|
|||
|
|
sql_server_conn = pymssql.connect(
|
|||
|
|
server=sql_server_config["server"],
|
|||
|
|
port=sql_server_config["port"],
|
|||
|
|
user=sql_server_config["user"],
|
|||
|
|
password=sql_server_config["password"],
|
|||
|
|
database=sql_server_config["database"],
|
|||
|
|
)
|
|||
|
|
sql_server_cursor = sql_server_conn.cursor()
|
|||
|
|
|
|||
|
|
# 连接到MySQL
|
|||
|
|
mysql_conn = mysql.connector.connect(**mysql_config)
|
|||
|
|
mysql_cursor = mysql_conn.cursor()
|
|||
|
|
|
|||
|
|
logging.info(f"Processing table: {table_name}")
|
|||
|
|
|
|||
|
|
# 检查MySQL中是否已存在该表
|
|||
|
|
mysql_cursor.execute(f"SHOW TABLES LIKE '{table_name}'")
|
|||
|
|
table_exists = mysql_cursor.fetchone()
|
|||
|
|
|
|||
|
|
if not table_exists:
|
|||
|
|
# 如果表不存在，创建表
|
|||
|
|
sql_server_cursor.execute(f"""
|
|||
|
|
SELECT
|
|||
|
|
COLUMN_NAME,
|
|||
|
|
DATA_TYPE,
|
|||
|
|
CHARACTER_MAXIMUM_LENGTH,
|
|||
|
|
NUMERIC_PRECISION,
|
|||
|
|
NUMERIC_SCALE
|
|||
|
|
FROM INFORMATION_SCHEMA.COLUMNS
|
|||
|
|
WHERE TABLE_NAME = '{table_name}'
|
|||
|
|
""")
|
|||
|
|
columns = sql_server_cursor.fetchall()
|
|||
|
|
|
|||
|
|
# 生成MySQL的CREATE TABLE语句
|
|||
|
|
create_table_sql = f"CREATE TABLE {table_name} ("
|
|||
|
|
for col in columns:
|
|||
|
|
col_name = col[0]
|
|||
|
|
col_type = col[1]
|
|||
|
|
|
|||
|
|
# 获取字段长度
|
|||
|
|
char_length = col[2]
|
|||
|
|
numeric_precision = col[3]
|
|||
|
|
numeric_scale = col[4]
|
|||
|
|
|
|||
|
|
# 简单类型映射（可能需要根据实际情况调整）
|
|||
|
|
if col_type == "varchar":
|
|||
|
|
col_type = "VARCHAR(255)"
|
|||
|
|
elif col_type == "int":
|
|||
|
|
col_type = "INT"
|
|||
|
|
elif col_type == "datetime":
|
|||
|
|
col_type = "DATETIME"
|
|||
|
|
elif col_type == "decimal":
|
|||
|
|
if numeric_precision and numeric_scale:
|
|||
|
|
col_type = f"DECIMAL({numeric_precision}, {numeric_scale})"
|
|||
|
|
else:
|
|||
|
|
col_type = "DECIMAL(10, 2)" # 默认值
|
|||
|
|
elif col_type == "money":
|
|||
|
|
col_type = "DECIMAL(19, 4)"
|
|||
|
|
|
|||
|
|
elif col_type == "smallmoney":
|
|||
|
|
col_type = "DECIMAL(19, 4)"
|
|||
|
|
elif col_type == "image":
|
|||
|
|
col_type = "LONGBLOB"
|
|||
|
|
|
|||
|
|
# 设置列的 NULL 属性
|
|||
|
|
if col_name.lower() == "id":
|
|||
|
|
# ID 列不允许 NULL
|
|||
|
|
create_table_sql += f"`{col_name}` {col_type} NOT NULL, "
|
|||
|
|
else:
|
|||
|
|
# 其他列允许 NULL
|
|||
|
|
create_table_sql += f"`{col_name}` {col_type} , "
|
|||
|
|
|
|||
|
|
# 添加主键约束（假设 ID 是主键）
|
|||
|
|
create_table_sql = create_table_sql.rstrip(", ") + f", PRIMARY KEY ({columns[0][0]}))"
|
|||
|
|
logging.info(f"Create table SQL: {create_table_sql}")
|
|||
|
|
|
|||
|
|
# 在MySQL中创建表
|
|||
|
|
mysql_cursor.execute(create_table_sql)
|
|||
|
|
logging.info(f"Table {table_name} created in MySQL.")
|
|||
|
|
else:
|
|||
|
|
logging.info(f"Table {table_name} already exists in MySQL. Updating data...")
|
|||
|
|
|
|||
|
|
# 获取表的列信息
|
|||
|
|
sql_server_cursor.execute(f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '{table_name}'")
|
|||
|
|
columns = sql_server_cursor.fetchall()
|
|||
|
|
|
|||
|
|
# 获取目标数据库中该表的 id 最大值
|
|||
|
|
mysql_cursor.execute(f"SELECT MAX({columns[0][0]}) FROM {table_name}")
|
|||
|
|
max_id = mysql_cursor.fetchone()[0]
|
|||
|
|
if max_id is None:
|
|||
|
|
max_id = 0 # 如果表中没有数据，设置 max_id 为 0
|
|||
|
|
logging.info(f"Target table {table_name} has max ID: {max_id}")
|
|||
|
|
|
|||
|
|
# 获取SQL Server中的数据（分批查询）
|
|||
|
|
offset = 0
|
|||
|
|
while True:
|
|||
|
|
# 使用 ROW_NUMBER() 实现分页查询
|
|||
|
|
sql_server_cursor.execute(f"""
|
|||
|
|
SELECT * FROM (
|
|||
|
|
SELECT *, ROW_NUMBER() OVER (ORDER BY {columns[0][0]}) AS RowNum
|
|||
|
|
FROM {table_name}
|
|||
|
|
WHERE {columns[0][0]} > {max_id}
|
|||
|
|
) AS SubQuery
|
|||
|
|
WHERE RowNum BETWEEN {offset + 1} AND {offset + BATCH_SIZE}
|
|||
|
|
""")
|
|||
|
|
rows = sql_server_cursor.fetchall()
|
|||
|
|
if not rows:
|
|||
|
|
logging.info(f"表:{table_name} 数据已经是最新的,不需要更新")
|
|||
|
|
break # 如果没有数据了，退出循环
|
|||
|
|
|
|||
|
|
insert_values = [row[:-1] for row in rows]
|
|||
|
|
|
|||
|
|
# 批量插入数据
|
|||
|
|
if insert_values:
|
|||
|
|
# 动态生成插入语句的列名和占位符
|
|||
|
|
placeholders = ", ".join(["%s"] * len(insert_values[0]))
|
|||
|
|
columns_list = ", ".join([col[0] for col in columns])
|
|||
|
|
insert_sql = f"INSERT INTO {table_name} ({columns_list}) VALUES ({placeholders})"
|
|||
|
|
# 执行批量插入
|
|||
|
|
try:
|
|||
|
|
mysql_cursor.executemany(insert_sql, insert_values)
|
|||
|
|
mysql_conn.commit()
|
|||
|
|
logging.info(f"Inserted {len(insert_values)} rows into {table_name}.")
|
|||
|
|
except mysql.connector.errors.DataError as e:
|
|||
|
|
logging.error(f"DataError: {e}")
|
|||
|
|
mysql_conn.rollback()
|
|||
|
|
|
|||
|
|
offset += BATCH_SIZE
|
|||
|
|
logging.info(f"Processed {offset} rows in {table_name}...")
|
|||
|
|
|
|||
|
|
# 关闭连接
|
|||
|
|
sql_server_cursor.close()
|
|||
|
|
sql_server_conn.close()
|
|||
|
|
mysql_cursor.close()
|
|||
|
|
mysql_conn.close()
|
|||
|
|
|
|||
|
|
logging.info(f"Sync completed for table: {table_name}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logging.error(f"Failed to sync table {table_name}. Error: {e}")
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
try:
|
|||
|
|
# 连接到SQL Server
|
|||
|
|
sql_server_conn = pymssql.connect(
|
|||
|
|
server=sql_server_config["server"],
|
|||
|
|
port=sql_server_config["port"],
|
|||
|
|
user=sql_server_config["user"],
|
|||
|
|
password=sql_server_config["password"],
|
|||
|
|
database=sql_server_config["database"],
|
|||
|
|
)
|
|||
|
|
sql_server_cursor = sql_server_conn.cursor()
|
|||
|
|
|
|||
|
|
# 获取SQL Server中的所有表
|
|||
|
|
sql_server_cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME")
|
|||
|
|
tables = sql_server_cursor.fetchall()
|
|||
|
|
|
|||
|
|
# 使用进程池并发处理每个表
|
|||
|
|
with Pool(processes=MAX_PROCESSES) as pool:
|
|||
|
|
pool.map(sync_table, [table[0] for table in tables])
|
|||
|
|
|
|||
|
|
logging.info("All tables synced successfully!")
|
|||
|
|
except Exception as e:
|
|||
|
|
logging.error(f"Main function failed. Error: {e}")
|
|||
|
|
finally:
|
|||
|
|
# 关闭连接
|
|||
|
|
if 'sql_server_cursor' in locals():
|
|||
|
|
sql_server_cursor.close()
|
|||
|
|
if 'sql_server_conn' in locals():
|
|||
|
|
sql_server_conn.close()
|
|||
|
|
|
|||
|
|
# 启动主函数
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|