大数量导入

大数量导入数据库：

def import_data():
    """
    导入日线数据
    """
    #offset：当偏移量大于800万时，offset limit模式性能下降严重，查询一次要12秒……
    #改成直接定位主键id查询。

    from_table = BS_Profit_Data
    with concurrent.futures.ThreadPoolExecutor() as executor:
        with session_scope() as session:
            onerow = session.query(func.min(from_table.id), func.max(from_table.id)).one()
            minid = onerow[0]
            maxid = onerow[1]

            if not minid:  #没有数据
                return

            pagesize = 5000
            ahead_id = minid
            next_id = ahead_id + pagesize
            i = 0  #计数
            while True:
                if ahead_id > maxid:
                    break
                rp = session.query(from_table).filter(from_table.id >= ahead_id, from_table.id < next_id)
                to_data = _build_result_data(rp)
                executor.submit(_insert_data, to_data, i + 1)
                i += 1
                ahead_id = next_id
                next_id = ahead_id + pagesize
    
    _logger.info("数据导入完成")