Mysql 跨库数据迁移 -- python 脚本

 

 

示例代码

import pymysql

class DataMigrate:

    def __init__(self, env):
        self.num = 50
        self.ENV = env

        # From
        self.db_from = self.connect_jingyi(1)
        self.cursor_from = self.db_from.cursor()
        self.from_sql = 'select first_project, status from product '

        # To
        self.db_to = self.connect_jingyi()
        self.cursor_to = self.db_to.cursor()
        self.to_sql = "insert into product_copy(first_project, status) value(%s,%s)"

    def connect_jingyi(self, is_jingyi=0):
        if is_jingyi:
            if self.ENV == 'test':
                return pymysql.connect(host='****',
                                       port=3306,
                                       user='***',
                                       passwd='***',
                                       db='from_db',
                                       charset='utf8'
                                       )
            else:
                return pymysql.connect(host='****',
                                       port=3306,
                                       user='***',
                                       passwd='***',
                                       db='from_db',
                                       charset='utf8'
                                       )
        else:
            if self.ENV == 'test':
                return pymysql.connect(host='****',
                                       port=3306,
                                       user='***',
                                       passwd='***',
                                       db='to_db',
                                       charset='utf8'
                                       )
            elif self.ENV == 'pre':
                return pymysql.connect(host='*****',
                                       port=3306,
                                       user='***',
                                       passwd='***',
                                       db='to_db',
                                       charset='utf8'
                                       )

    def migrate_date(self):
        len_from = self.cursor_from.execute(self.from_sql)
        for i in range(int(len_from / self.num)):
            print(i)
            data1 = self.cursor_from.fetchmany(self.num)
            self.cursor_to.executemany(self.to_sql, data1)

        data2 = self.cursor_from.fetchall()
        self.cursor_to.executemany(self.to_sql, data2)

        self.db_to.commit()

        self.db_from.close()
        self.db_to.close()


if __name__ == '__main__':
    data_migrate = DataMigrate('self-test')
    data_migrate.migrate_date()

 

函数解析

fetchone() 获取游标下一条数据（元组）

fetchmany()  获取多条数据（元组列表）

fetchall()   获取所有数据（元组列表）

看下源码

...
    def fetchone(self):
        """Fetch the next row"""
        self._check_executed()
        if self._rows is None or self.rownumber >= len(self._rows):
            return None
        result = self._rows[self.rownumber]
        self.rownumber += 1
        return result

    def fetchmany(self, size=None):
        """Fetch several rows"""
        self._check_executed()
        if self._rows is None:
            return ()
        end = self.rownumber + (size or self.arraysize)
        result = self._rows[self.rownumber:end]
        self.rownumber = min(end, len(self._rows))
        return result

    def fetchall(self):
        """Fetch all the rows"""
        self._check_executed()
        if self._rows is None:
            return ()
        if self.rownumber:
            result = self._rows[self.rownumber:]
        else:
            result = self._rows
        self.rownumber = len(self._rows)
        return result
...