pandas基础(3)_数据处理
1：删除重复数据
使用duplicate()函数检测重复的行，返回元素为bool类型的Series对象，每个元素对应一行，如果该行不是第一次出现，则元素为true
>>> df =DataFrame(np.random.randint(0,150,size=(6,3)),columns=['Chinese','maths','Chinese'],index=['zhangsan','lisi','wangwu','lisi','xiaowu','zhangsan'])
>>> df
Chinese maths Chinese
zhangsan 17 58 70
lisi 88 20 137
wangwu 130 29 57
lisi 71 20 65
xiaowu 133 60 6
zhangsan 96 48 60
>>> df.duplicated()
zhangsan False
lisi False
wangwu False
lisi False
xiaowu False
zhangsan False
dtype: bool
>>> df =DataFrame(np.random.randint(0,2,size=(6,2)),columns=['Chinese','maths'],index=['zhangsan','lisi','wangwu','lisi','xiaowu','zhangsan'])
>>> df
Chinese maths
zhangsan 1 1
lisi 1 0
wangwu 0 0
lisi 1 0
xiaowu 1 1
zhangsan 0 0
>>> df.duplicated ()
zhangsan False
lisi False
wangwu False
lisi True
xiaowu True
zhangsan True
dtype: bool
>>> #如果出现的数据一样，则会返回true
>>> #调用drop_duplicates()可以删除重复的数据
>>> df.drop_duplicates ()
Chinese maths
zhangsan 1 1
lisi 1 0
wangwu 0 0
>>> #删除的是行
>>> #rename()函数替换索引
>>> #map():新建一列
>>> #replace()替换元素
2：异常值检测和过滤
>>> #使用describe（）函数查看每一列的描述统计量
>>> df =DataFrame(np.random.randint(0,150,size=(6,2)),columns=['Chinese','maths'],index=[list('ABCDEF')])
>>> df
Chinese maths
A 119 25
B 28 33
C 10 134
D 44 121
E 44 119
F 91 46
>>> df.describe ()
Chinese maths
count 6.000000 6.000000
mean 56.000000 79.666667#平均值
std 40.943864 50.014665
min 10.000000 25.000000
25% 32.000000 36.250000
50% 44.000000 82.500000
75% 79.250000 120.500000
max 119.000000 134.000000
>>> #std是标准方差
>>> df.std ()
Chinese 40.943864
maths 50.014665
dtype: float64
>>> df.std(axis=1)
A 66.468037
B 3.535534
C 87.681241
D 54.447222
E 53.033009
F 31.819805
dtype: float64
>>> #每个人的标准差
>>> np.abs(df)>df.std()*2
Chinese maths
A True False
B False False
C False True
D False True
E False True
F True False
>>> #当某个方差大于标准方差的2倍时认为这两个数特殊，返回true,这时筛选出来
>>> df.any(axis=1)
A True
B True
C True
D True
E True
F True
dtype: bool
>>> df2=np.abs(df)>df.std()*2
>>> df3=df2.any(axis=1)
>>> df[df3]
Chinese maths
A 119 25
C 10 134
D 44 121
E 44 119
F 91 46
>>> df2=np.abs(df)>df.std()*2
>>> df2
Chinese maths
A True False
B False False
C False True
D False True
E False True
F True False
>>> df2.any()
Chinese True
maths True
dtype: bool
>>> df2.all()
Chinese False
maths False
dtype: bool
>>> df3=df2.any(axis=1)
>>> df3
A True
B False
C True
D True
E True
F True
dtype: bool
>>> df[df3]
Chinese maths
A 119 25
C 10 134
D 44 121
E 44 119
F 91 46
3：随机排序
>>> x=np.random.permutation (6)
>>> x
array([4, 5, 1, 0, 3, 2])
>>> df.take(x)
Chinese maths
E 44 119
F 91 46
B 28 33
A 119 25
D 44 121
C 10 134
>>> #使用take(函数排序，可以借助np.random.pemutation()函数随机排序，可以用来随机抽样
4：数据聚合
>>> #通常是每一个数组生成一个具体的值
>>> #1分组 2用函数处理 3合并
>>> #核心函数groupby()
>>> df = DataFrame({'item':['apple','banana','orange','banana','orange','apple'],'price':[4,3,3,2.5,4,2],'color':['red','yellow','yellow','green','green','green']})
>>> df
color item price
0 red apple 4.0
1 yellow banana 3.0
2 yellow orange 3.0
3 green banana 2.5
4 green orange 4.0
5 green apple 2.0
>>> df.groupby('item')
<pandas.core.groupby.DataFrameGroupBy object at 0x000000000E8EE240>
>>> g=df.groupby('item')
>>> g
<pandas.core.groupby.DataFrameGroupBy object at 0x000000000E76A828>
>>> g.groups
{'orange': Int64Index([2, 4], dtype='int64'), 'apple': Int64Index([0, 5], dtype='int64'), 'banana': Int64Index([1, 3], dtype='int64')}
>>> #分组
>>> g['price'].mean ()
item
apple 3.00
banana 2.75
orange 3.50
Name: price, dtype: float64
>>> m=g['price'].mean ()
>>> type(m)
<class 'pandas.core.series.Series'>
>>> df_mean=DataFrame(m)
>>> df_mean
price
item
apple 3.00
banana 2.75
orange 3.50
>>> pd.merge(df,df_mean,left_on='item',right_index=True)
color item price_x price_y
0 red apple 4.0 3.00
5 green apple 2.0 3.00
1 yellow banana 3.0 2.75
3 green banana 2.5 2.75
2 yellow orange 3.0 3.50
4 green orange 4.0 3.50
>>> #以多个属性进行分组
>>> df.groupby(['color','item']).sum()
price
color item
green apple 2.0
banana 2.5
orange 4.0
red apple 4.0
yellow banana 3.0
orange 3.0
>>> #最终变成了多重索引结构
