Pandas DataFrame四种写入方法效率对比
该测试程序是将protocol变量转换为DataFrame类型为了便于使用numpy等数学工具
ImuEncode_pb2 是使用protocol生成的model， 具体使用方法可参考Protocol buffer 指南。
这是我使用的一些变量的声明
import time
import ImuEncode_pb2
import numpy as np
import pandas as pd
from pandas.core.frame import DataFrame
from ImuSignal import LegString
from ImuSignal import LegPositionString
from ImuSignal import SignalAttributeDict
from OnlineClassifier import OnlineClassifier
ImuSignalDataset = ImuEncode_pb2.ImuSignalDataset()
for i in range(50):
imusignal =ImuSignalDataset.imusignal.add()
for j in range(4):
signal = imusignal.signal.add()
signal.TimeStamp = 0
signal.acc.AccX = 0
signal.acc.AccY = 0
signal.acc.AccZ = 0
signal.angle.Roll = 0
signal.angle.Pitch = 0
signal.anglerate.Course = 0
signal.quaternion.v = 0
signal.quaternion.x = 0
signal.quaternion.y = 0
signal.quaternion.z = 0
signal.leg = ImuEncode_pb2.Signal.RT
signal.legposition = ImuEncode_pb2.Signal.Foot
rate = 100 #sample rate
period = 0.5 #classify period
Leg = ['RT','LT']
Leg_Position = ['Foot','Thigh']
angle = ['Roll','Pitch','Course']
accel = ['AccX','AccY','AccZ']
特别注意下面代码段的加入，这是对于内存的声明，我是随便赋值不知道Python变量的声明原理后续完善
SignalFrame = DataFrame()
for leg in Leg:
for leg_position in Leg_Position:
for signal in accel + angle:
SignalFrame[leg_position+"_"+signal+"_"+leg] = np.arange(999999999,999999999+int(period*rate),1)/1000000
该方法不简单并且耗时应该是最长的，测试耗时为： 0.09793972969055176
SignalFrame = DataFrame()
for leg in Leg:
for leg_position in Leg_Position:
for signal in accel + angle:
SignalFrame[leg_position+"_"+signal+"_"+leg] = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
SignalFrame = pd.concat([pd.DataFrame(data={"Foot_AccX_RT":imusignal.signal[0].acc.AccX, "Foot_AccY_RT":imusignal.signal[0].acc.AccY, "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
"Foot_Roll_RT":imusignal.signal[0].angle.Roll, "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch, "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
"Thigh_AccX_RT":imusignal.signal[1].acc.AccX, "Thigh_AccY_RT":imusignal.signal[1].acc.AccY, "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
"Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch, "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
"Foot_AccX_LT":imusignal.signal[2].acc.AccX, "Foot_AccY_LT":imusignal.signal[2].acc.AccY, "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
"Foot_Roll_LT":imusignal.signal[2].angle.Roll, "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch, "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
"Thigh_AccX_LT":imusignal.signal[3].acc.AccX, "Thigh_AccY_LT":imusignal.signal[3].acc.AccY, "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
"Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch, "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
},index=[0]) for imusignal in ImuSignalDataset.imusignal],ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)
该方法不简单并且耗时长，测试耗时为： 0.04797029495239258
SignalFrame = DataFrame()
for leg in Leg:
for leg_position in Leg_Position:
for signal in accel + angle:
SignalFrame[leg_position+"_"+signal+"_"+leg] = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
for imusignal in ImuSignalDataset.imusignal:
SignalFrame = SignalFrame.append({"Foot_AccX_RT":imusignal.signal[0].acc.AccX, "Foot_AccY_RT":imusignal.signal[0].acc.AccY, "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
"Foot_Roll_RT":imusignal.signal[0].angle.Roll, "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch, "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
"Thigh_AccX_RT":imusignal.signal[1].acc.AccX, "Thigh_AccY_RT":imusignal.signal[1].acc.AccY, "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
"Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch, "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
"Foot_AccX_LT":imusignal.signal[2].acc.AccX, "Foot_AccY_LT":imusignal.signal[2].acc.AccY, "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
"Foot_Roll_LT":imusignal.signal[2].angle.Roll, "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch, "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
"Thigh_AccX_LT":imusignal.signal[3].acc.AccX, "Thigh_AccY_LT":imusignal.signal[3].acc.AccY, "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
"Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch, "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
},ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)
该方法简单并且耗时仅长于最快的实现方法，测试耗时为：0.016989469528198242
SignalFrame = DataFrame()
for leg in Leg:
for leg_position in Leg_Position:
for signal in accel + angle:
SignalFrame[leg_position+"_"+signal+"_"+leg] = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
for SignalData in imusignal.signal:
signal_dict = SignalAttributeDict(SignalData)
for signal in angle + accel:
SignalFrame.at[i,LegPositionString(SignalData.legposition)+"_"+signal+"_"+LegString(SignalData.leg)]=signal_dict[signal]
i+=1
endtime = time.time()
print("cost time:",endtime-starttime)
该方法不简单，如果数据量较大可能会影响数据的准确度，但是耗时应该是最短的，测试耗时为：0.012990951538085938
SignalFrame = DataFrame()
for leg in Leg:
for leg_position in Leg_Position:
for signal in accel + angle:
SignalFrame[leg_position+"_"+signal+"_"+leg] = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
SignalFrame.loc[i]=[imusignal.signal[0].acc.AccX , imusignal.signal[0].acc.AccY , imusignal.signal[0].acc.AccZ,\
imusignal.signal[0].angle.Roll , imusignal.signal[0].angle.Pitch , imusignal.signal[0].anglerate.Course,\
imusignal.signal[1].acc.AccX , imusignal.signal[1].acc.AccY , imusignal.signal[1].acc.AccZ,\
imusignal.signal[1].angle.Roll , imusignal.signal[1].angle.Pitch , imusignal.signal[1].anglerate.Course,\
imusignal.signal[2].acc.AccX , imusignal.signal[2].acc.AccY , imusignal.signal[2].acc.AccZ,\
imusignal.signal[2].angle.Roll , imusignal.signal[2].angle.Pitch , imusignal.signal[2].anglerate.Course,\
imusignal.signal[3].acc.AccX , imusignal.signal[3].acc.AccY , imusignal.signal[3].acc.AccZ,\
imusignal.signal[3].angle.Roll , imusignal.signal[3].angle.Pitch , imusignal.signal[3].anglerate.Course]
i+=1
endtime = time.time()
print("cost time:",endtime-starttime)
用于生成ImuEncode_pb2的ImuEncode.proto文件如下
syntax = "proto2";
package ImuTutorial;
message Signal {
enum Leg {
RT = 0;
LT = 1;
}
enum LegPosition{
Foot = 0;
Thigh = 1;
}
required Leg leg = 1 [default = RT];
required LegPosition legposition = 2 [default = Foot];
optional float TimeStamp = 3;
optional int64 time = 4;
optional int32 id = 5;
message Acc{
required float AccX = 1;
required float AccY = 2;
required float AccZ = 3;
}
message Angle{
optional float Course = 1;
required float Roll = 2;
required float Pitch = 3;
}
message AngleRate{
required float Course = 1;
optional float Roll = 2;
optional float Pitch = 3;
}
message Quaternion{
optional float v = 1;
required float x = 2;
required float y = 3;
required float z = 4;
}
required Acc acc = 6;
required Angle angle= 7;
required AngleRate anglerate= 8;
optional Quaternion quaternion = 9;
}
message ImuSignal {
required uint64 time = 1;
repeated Signal signal = 2;
}
message ImuSignalDataset {
repeated ImuSignal imusignal = 1;
}
message Datalength {
required fixed32 length = 1;
}
任世事无常，勿忘初心
