数据预处理之清洗
''' 数据的清洗 1.检测与处理重复值 2.检测与处理缺失值 3.检测与处理异常值 ''' import numpy as np import pandas as pd # 去重:按照A列去重，同时保留第一个数据且在原表去重 data = pd.DataFrame({'A': [1, 1, 2, 2], 'B': ['a', 'b', 'a', 'b']}) print(data) # 对A列去重 # data.drop_duplicates('A', 'first', inplace=True) # print(data) # 对所有列去重 # data.drop_duplicates('first',inplace=True) # print(data) # 检测重复值:默认按照行 duplicates = data.duplicated() print(duplicates) # 检测重复值:按照列 duplicates = data.duplicated(['A']) print(duplicates) # 处理缺失值 # 删除法 dates = pd.date_range('20130101', periods=6) df = pd.DataFrame(np.arange(24).reshape((6, 4)), index=dates, columns=['A', 'B', 'C', 'D']) df.iloc[0, 1] = np.nan df.iloc[1, 2] = np.nan print(df) df1 = df.dropna(axis=0, how='any', inplace=False) print(df1) # 替换法 df2 = df.fillna(value=0) print(df2) # 判断是否有缺失值 df3 = df.isnull() print(df3) # 统计数据中一共出现多少缺失值 print(df3.sum()) # 每一列缺失值个数（默认形式） print(df3.sum(axis=1)) # 每一行缺失值个数 print(df3.sum().sum()) # 数据整体出现缺失值个数 # 判断整个数据中是否存在缺失数据 df4 = np.any(df.isnull()) == True print(df4) 输出结果 A B 0 1 a 1 1 b 2 2 a 3 2 b 0 False 1 False 2 False 3 False dtype: bool 0 False 1 True 2 False 3 True dtype: bool A B C D 2013-01-01 0 NaN 2.0 3 2013-01-02 4 5.0 NaN 7 2013-01-03 8 9.0 10.0 11 2013-01-04 12 13.0 14.0 15 2013-01-05 16 17.0 18.0 19 2013-01-06 20 21.0 22.0 23 A B C D 2013-01-03 8 9.0 10.0 11 2013-01-04 12 13.0 14.0 15 2013-01-05 16 17.0 18.0 19 2013-01-06 20 21.0 22.0 23 A B C D 2013-01-01 0 0.0 2.0 3 2013-01-02 4 5.0 0.0 7 2013-01-03 8 9.0 10.0 11 2013-01-04 12 13.0 14.0 15 2013-01-05 16 17.0 18.0 19 2013-01-06 20 21.0 22.0 23 A B C D 2013-01-01 False True False False 2013-01-02 False False True False 2013-01-03 False False False False 2013-01-04 False False False False 2013-01-05 False False False False 2013-01-06 False False False False A 0 B 1 C 1 D 0 dtype: int64 2013-01-01 1 2013-01-02 1 2013-01-03 0 2013-01-04 0 2013-01-05 0 2013-01-06 0 Freq: D, dtype: int64 2 True
''' 数据清洗之缺失值插值法 ''' import numpy as np from scipy.interpolate import interp1d import matplotlib.pyplot as mp # 创建待插值的数据 x = np.linspace(0, 10 * np.pi, 20) y = np.cos(x) # 分别使用linear和quadratic插值 f1 = interp1d(x, y, kind='linear') fq = interp1d(x, y, kind='quadratic') # 设置x的最大值和最小值以防止插值数据越界 xint = np.linspace(x.min(), x.max(), 1000) yint1 = f1(xint) yintq = fq(xint) # 线性插值和二阶插值效果 mp.plot(x, y, color='pink') mp.plot(xint, yint1, color='blue', label='Linear') mp.plot(xint, yintq, color='orangered', label='Quadratic') mp.legend() mp.show()
''' 数据清洗---异常值处理 ''' import numpy as np import pandas as pd df = pd.DataFrame(np.random.randn(4, 4) * 4 + 3) print(df) # 离差标准化 df_norm = (df - df.min()) / (df.max() - df.min()) print(df_norm) # 标准差标准化 df_norm = (df - df.mean()) / df.std() print(df_norm) 输出结果： 0 1 2 3 0 5.163514 1.087991 11.144715 -1.082642 1 0.932404 2.191543 7.219776 3.437653 2 9.983589 0.218654 6.576483 0.321555 3 5.253907 3.771512 8.672818 4.734098 0 1 2 3 0 0.467465 0.244687 1.000000 0.000000 1 0.000000 0.555296 0.140819 0.777118 2 1.000000 0.000000 0.000000 0.241406 3 0.477452 1.000000 0.458894 1.000000 0 1 2 3 0 -0.045926 -0.475946 1.352387 -1.089516 1 -1.190051 0.244107 -0.583957 0.588309 2 1.257460 -1.043177 -0.901322 -0.568312 3 -0.021483 1.275017 0.132892 1.069518
