Python读取文件并写入ODPS

前提：

　　将本地csv文件，用pandas读取，并进行数据ETL，最后导入到ODPS表中。

代码如下：

import pandas as pd
from tqdm import tqdm_notebook
from odps import ODPS
from odps import options
from odps.df import DataFrame


from odps.df import DataFrame

# 查看相对路径
%pwd

odps = ODPS('ODPS_ak', 'ODPS_pw', 'ODPS项目空间',
            endpoint='http://service.cn-shanghai.maxcompute.aliyun.com/api',
            tunnel_endpoint="http://dt.cn-shanghai.maxcompute.aliyun.com")

# 读取数据
data = pd.read_csv('/home/linxz/datadir/202108.csv',encoding='utf-8') # 07已同步

# 查看数据是否全部读取成功
data.describe()

# 数据ETL
a = data[['company_id','ofr_id','bid_id','security_id','bond_id','listed_market','short_name','time','bid_price','bid_volume','ofr_price','ofr_volume','bid_yield','bid_net_price','bid_flag_bargain','bid_flag_relation','bid_exercise','bid_price_description','bid_quote_type','ofr_yield','ofr_net_price','ofr_flag_bargain','ofr_flag_relation','ofr_exercise']]
b = data[['ofr_price_description']]
b_new = b.replace('[,]','，',regex = True) # 使用正则表达式，将数据中的英文逗号改为中文

c = data[['ofr_quote_type','bid_ss_detect','ofr_ss_detect']]
data_join = a.join(b_new)
data_new = data_join.join(c)
data_new.head()

# 新增最后面一列，作为分区
data_new['month_date'] = '202108'
data_new.head()

# 最后导入ODPS表中 
options.connect_timeout=200
options.tunnel.use_instance_tunnel = True
options.tunnel.limit_instance_tunnel = False
odps_awake_model_result = DataFrame(data_new)
print(odps_awake_model_result)
odps_awake_model_result.persist('ODPS项目空间.o_sumscope_bond_relation_detail_d', partitions=['month_date'], odps=odps)

# 导出方式二：导出到本地
# 导出修改后的csv文件
# data_new.to_csv('/home/linxz/datadir/202101_new.csv')