转换mongodb数据库到mysql每天凌晨同步数据

# coding=utf-8

# /usr/bin/python3.6 generate_stars.py
# 定时任务每天凌晨第一分
# 1 0 * * * /usr/bin/python3.6 /root/backend/services/app/app/generate_stars.py
# 测试定时任务1分一次
# */1 * * * * /usr/bin/python3.6 /root/backend/services/app/app/generate_stars.py

import random
import json
import datetime
import time
from pymongo import MongoClient
from io import BytesIO
import pymysql.cursors

import logging

# 默认的日志级别设置为WARNING（日志级别等级CRITICAL > ERROR > WARNING > INFO > DEBUG > NOTSET）
logging.basicConfig(level=logging.DEBUG,
                    format="%(asctime)s %(name)s [line:%(lineno)d] %(levelname)s %(message)s",
                    datefmt='%a, %d %b %Y %H:%M:%S',
                    filename="/root/backend/db/cront.log",
                    filemode="a")

dddd = MongoClient('120.xx.26.xx:20002', username='xxx', password='xxx')
mongo_tag = MongoClient('xxxxx.mongodb.rds.aliyuncs.com:3717', username='xx', password='xxx')


def get_stars_accounts():

    current = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    logging.info("ok start-%s"%(current))

    stars = dddd['goodlook']['star'].find()
    conn = pymysql.connect(host='rm-xxxx.mysql.rds.aliyuncs.com',
                           port=3306,
                           user='dba',
                           password='xxx',
                           db='app',
                           charset='utf8')
    cursor = conn.cursor()
    # conn.ping(reconnect=True)

    for star in stars:
        print(star)
        updated_at = star.get("updated_at")
        created_at = star.get("created_at")
        print(type(updated_at))
        star_id = star.get("star_id")

        birthday = star.get("birthday")
        biography = star.get("biography")
        dress_comment = star.get("dress_comment")

        country = star.get("nation")
        height = star.get("height")
        avatar_url = star.get("avatar")
        images_count = star.get("image_count")

        gender = star.get("gender")
        if gender == "女":
            gender = 'F'
        else:
            gender = 'M'

        # big_image_url = star.get("image_url")[0]
        names_list = star.get("names")

        chinese_name = None
        english_name = None
        alias_name = None

        for li in names_list:
            if li.get("type") == "Chinese":
                chinese_name = li.get("name")
            elif li.get("type") == "English":
                english_name = li.get("name")
            else:
                alias_name = li.get("name")

        print(chinese_name, english_name, alias_name)
        if english_name and chinese_name:
            name = english_name
            other_name = chinese_name
        elif english_name and not chinese_name:
            name = english_name
            other_name = ""
        elif chinese_name and not english_name:
            name = chinese_name
            other_name = ""
        else:
            pass

        # 拿tag_id;直到昭君那边数据里直接带tag_id，即可删除
        print("name", name)
        tag = mongo_tag['admin']['tag'].find_one({"key": "star", "value": name})
        if tag is not None:
            tag_id = str(tag.get("_id"))
            print("tag_id", tag_id)

            sql = 'select star_id, updated_at from star where star_id="{star_id}"'.format(star_id=star_id)
            cursor.execute(sql)
            ret = cursor.fetchone()

            if ret is not None:
                # 表里存在
                mydb_updated_at = ret[1]
                if updated_at > mydb_updated_at:
                    # 更新
                    update_sql = 'update star set gender="{gender}", height="{height}", country="{country}", birthday="{birthday}", avatar_url="{avatar_url}", name="{name}", other_name="{other_name}", alias_name="{alias_name}", biography="{biography}", dress_comment="{dress_comment}", created_at="{created_at}", updated_at="{updated_at}" where star_id="{star_id}"'.format(
                        gender=gender, height=height, country=country, birthday=birthday,
                        avatar_url=avatar_url, name=name, other_name=other_name, alias_name=alias_name,
                        biography=biography, dress_comment=dress_comment, created_at=created_at,
                        updated_at=updated_at,
                        star_id=star_id)

                    cursor.execute(update_sql)
                    conn.commit()

                else:
                    # 说明数据没有变化
                    pass
            else:
                # 表里不存在直接插入
                try:
                    insert_sql = 'insert into star(tag_id, star_id, gender, height, country, birthday, avatar_url, name, other_name, alias_name, biography, dress_comment,created_at,updated_at) values("{tag_id}", "{star_id}", "{gender}", "{height}", "{country}", "{birthday}","{avatar_url}", "{name}", "{other_name}", "{alias_name}", "{biography}","{dress_comment}", "{created_at}", "{updated_at}")'.format(
                        tag_id=tag_id, star_id=star_id, gender=gender, height=height, country=country,
                        birthday=birthday,
                        avatar_url=avatar_url, name=name, other_name=other_name, alias_name=alias_name,
                        biography=biography, dress_comment=dress_comment, created_at=created_at,
                        updated_at=updated_at)
                    cursor.execute(insert_sql)
                    conn.commit()
                except pymysql.err.IntegrityError as e:
                    pass
        else:
            pass

    cursor.close()
    conn.close()
    current = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    logging.info("ok over-%s" % (current))


get_stars_accounts()