MySQL python 数据迁移脚本
admin
2023-07-14 11:44:20
0

用python写了个数据迁移脚本,主要是利用从库将大的静态表导出表空间,载导入到目标实例中。

#!/usr/bin/env python3
#-*- coding:utf8 -*-
#author:zhanbin.liu
#!!!!!DB必须同版本
#python3环境   pip3 install pymysql paramiko

import os
#from pathlib import Path
import sys
import pymysql
import paramiko

#每次只能迁移一个DB下的表,到指定DB
#GRANT SELECT, CREATE, RELOAD, ALTER, LOCK TABLES ON *.* TO 'data_migration'@'192.168.%' IDENTIFIED BY  'data_migration@123';
tables='sqlauto_cluster,sqlauto_user'       #以,分割的字符串,如a,b,c
tableList = tables.split(',')
sourceIp = '192.168.1.101'
sourceDataBase = '/data/mysql/3306/data'
sourceDbName = 'inception_web'
sourceDataDir = os.path.join(sourceDataBase,sourceDbName)
desIp = '192.168.1.102'
desDataBase = '/data/mysql/3306/data'
desDbName = 'inception_web'
desDataDir = os.path.join(desDataBase,desDbName)

# for table in tableList:
#     desFile = Path("%s/%s.ibd" %(desDataDir,table))
#     print(desFile)
#     if desFile.is_file():
#         print("ok")
#     else:
#         print("no")

comUser = 'data_migration'
comPwd = 'data_migration@123'
comPort = 3306

client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

def table_judge():
    print("table_judge")
    sourceTableExist = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
    desTableExist = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
    sourceTables = []
    desTables = []
    cursor_source = sourceTableExist.cursor()
    cursor_des = desTableExist.cursor()

    for table in tableList:
        #print(table)
        cursor_source.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (sourceDbName,table))
        sourceTable_tmp = cursor_source.fetchall()
        cursor_des.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (desDbName,table))
        desTable_tmp = cursor_des.fetchall()
        #print(desTable_tmp)
        if sourceTable_tmp is ():
            sourceTables.append(table)
        if desTable_tmp is not ():
            desTables.append(desTable_tmp[0][0])
    sourceTableExist.close()
    desTableExist.close()

    s=d=0
    if sourceTables != []:
        print('迁移源不存在将要迁移的表:',sourceIp,sourceDbName, sourceTables,' 请检查')
        s=1
    if desTables != []:
        print('目标库存在将要迁移的表:',desIp,desDbName,desTables,' 请移除')
        d=1
    if s == 1 or d == 1:
        sys.exit()

def data_sync():
    print('data_sync')
    db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
    db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
    cursor_db_source = db_source.cursor()
    cursor_db_des = db_des.cursor()

    for table in tableList:
        print("正在同步表:",table)
        cursor_db_source.execute("show create table %s;" % (table))
        createTableSQL = cursor_db_source.fetchall()[0][1]
        print(createTableSQL)
        try:
            cursor_db_des.execute(createTableSQL)
        except Exception as error:
            print(error)
        cursor_db_source.execute("flush table %s with read lock;" % (table))
        cursor_db_des.execute("alter table %s discard tablespace;" % (table))

        client.connect(sourceIp, 22, 'root')
        stdin1, stdout1, stderr1 = client.exec_command("scp %s  %s:%s " % (sourceDataDir+"/"+table+".ibd", desIp, desDataDir))
        stdin2, stdout2, stderr2 = client.exec_command("scp %s  %s:%s " % (sourceDataDir+"/"+table+".cfg", desIp, desDataDir))
        a_e_1 = stderr1.readlines()
        a_e_2 = stderr2.readlines()
        if a_e_1 != [] or a_e_2 != []:
            print(a_e_1,a_e_2)
            sys.exit()
        client.close()

        client.connect(desIp, 22, 'root')
        stdin3, stdout3, stderr3 = client.exec_command("chown -R mysql.mysql %s*" % (desDataDir+"/"+table))
        a_e_3 = stderr3.readlines()
        if a_e_3 != []:
            print(a_e_1, a_e_2)
            sys.exit()
        client.close()
        #cursor_db_source.execute("select sleep(10);")
        cursor_db_source.execute("unlock tables;")
        cursor_db_des.execute("alter table %s import tablespace;" % (table))
        print("同步完成")

    cursor_db_source.close()
    cursor_db_des.close()

def data_checksum():
    print('data_checksum')
    db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
    db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
    cursor_db_source = db_source.cursor()
    cursor_db_des = db_des.cursor()

    for table in tableList:
        print("正在校验表:", table)
        cursor_db_source.execute("checksum table %s;" % (table))
        ck_s = cursor_db_source.fetchall()[0][1]
        cursor_db_des.execute("checksum table %s;" % (table))
        ck_d = cursor_db_des.fetchall()[0][1]
        if ck_s != ck_d:
            print("表不一致:",table)
        else:
            print("表一致:",table)

    cursor_db_source.close()
    cursor_db_des.close()

if __name__ == "__main__":
    table_judge()
    data_sync()
    data_checksum()
    print('haha')

相关内容

热门资讯

黄河科技学院2026年招生简章 长按图片识别二维码或点击 “阅读原文” 查看电子招生简章。
医路起航,从“心” 开始!黄河... 6月1日上午,黄河科技学院附属医院2022级临床医学本科实习生入院岗前培训在大医讲堂顺利举办。院领导...
问题居然在实体卡槽上!美版iP... 6月2日消息,日前,又有博主提前把还没发布的iPhone 18 Pro电池参数给曝光了出来,根据爆料...
2026年618手机购机攻略:... 每年618都是手机换新的最佳窗口期,2026年大促叠加数码国补、平台满减、品牌降价三重福利,新机老机...
8.99万打穿底价!史无前例的... 科技的浪漫,不是将人类送入遥不可及的星辰,而是把曾经高不可攀的科幻,变成柴米油盐里的“论斤卖”。 如...
苹果首款折叠屏iPhone U... PChome 6月2日消息,据知名数码博主@i冰宇宙最新曝光的机模谍照,苹果首款折叠屏手机(内部命名...
特朗普:美伊暂停谈判?假新闻! 当地时间6月2日,美国总统特朗普否认伊朗已停止与调解方沟通,称美国与伊朗之间的对话一直在持续进行。特...
白宫记协晚宴改期至7月,特朗普... 当地时间6月2日,美国总统特朗普表示,此前因突发暴力事件而中断的白宫记协晚宴已确定改期至7月24日举...
特朗普,开掉“不听话的情报头子... 苦撑15个月后,美国国家情报总监加巴德即将离开白宫。她对辞职的解释是:丈夫近期确诊罕见骨癌,自己希望...
面粉染头模仿黄仁勋,农村青年走... “先赔偿5000元肖像侵权使用费,再删除所有视频,要么就起诉你。”这是黄仁勋的模仿者杨洋最近频繁收到...