处理缺失值

Julia 中的数据操作

Katerina Zahradova

Instructor

查找缺失值

# 检查是否存在缺失值
describe(penguins, :nmissing)

7×2 DataFrame
Row variable           nmissing
    Symbol             Int64
_________________________________
1   species            0
2   island             5
3   culmen_length_mm   0
4   culmen_depth_mm    0
5   flipper_length_mm  0
6   body_mass_g        23
7   sex                0

ismissing()

# 查找含缺失值的行
penguins[ismissing.(penguins.island),:]

5x7 DataFrame
Row species   island    culmen_length_mm  culmen_depth_mm  ... 
    String15  String15  Float64           Float64          ...
___________________________________________________________________
1   Adelie    missing   39.5              17.4             ...
2   Adelie    missing   40.3              18.0             ...
3   Chinstrip missing   46.7              18.3             ...
4   Gentoo    missing   49.3              13.6             ...
5   Gentoo    missing   43.9              17.8             ...

ismissing()

# 查找含缺失值的行
penguins[ismissing.(penguins.island), :species, :sex]

5x3 DataFrame
Row species   island    sex
    String15  String15  String7
________________________________
1   Adelie    missing   MALE
2   Adelie    missing   FEMALE
3   Chinstrip missing   MALE
4   Gentoo    missing   MALE
5   Gentoo    missing   FEMALE

dropmissing()

# 删除所有缺失值
dropmissing!(penguins)

describe(penguins)

7×2 DataFrame
Row variable           nmissing
    Symbol             Int64
1   species            0
2   island             0
3   culmen_length_mm   0
4   culmen_depth_mm    0
5   flipper_length_mm  0
6   body_mass_g        0
7   sex                0

# 仅删除 island 列中的缺失
dropmissing!(penguins, :island)

describe(penguins)

7×2 DataFrame
Row variable           nmissing
    Symbol             Int64
1   species            0
2   island             0
3   culmen_length_mm   0
4   culmen_depth_mm    0
5   flipper_length_mm  0
6   body_mass_g        23
7   sex                0

replace()

# 用指定值替换缺失
replace!(penguins.body_mass_g, missing => 0)

# 用均值替换缺失
replace!(penguins.body_mass_g, missing => mean(skipmissing(penguins.body_mass_g)))

按组汇总替换

企鹅对比

¹ 图片来源 www.bas.ac.uk/about/antarctica/wildlife/penguins/

使用 groupby() 替换

# 遍历分组，并用各组四舍五入的均值替换
for group in groupby(penguins, :species)
    group[ismissing.(group.body_mass_g), :body_mass_g] .= round(mean(skipmissing(group.body_mass_g)))
end

# 检查缺失值
describe(penguins, :nmissing)

7×2 DataFrame
Row variable           nmissing
    Symbol             Int64
1   species            0
...
6   body_mass_g        0
7   sex                0

使用多列分组替换

# 遍历更多分组，并用各组四舍五入的均值替换
for group in groupby(penguins, [:species, :sex])
    group[ismissing.(group.body_mass_g), :body_mass_g] .= round(mean(skipmissing(group.body_mass_g)))
end

数据不足

# 当组内没有记录会发生什么
for group in groupby(penguins, [:species, :sex, :flipper_length_mm, :culmen_length_mm])
    group[ismissing.(group.body_mass_g), :body_mass_g] .= round(mean(skipmissing(group.body_mass_g)))
end

ArgumentError: median of an empty array is undefined, Any[]

速查表：查找与删除缺失值

ismissing(var): 若 var = missing 返回 true，否则返回 false
ismissing.(df.col): 返回 true/false 向量
df[ismissing.(df.col),:]: 返回 col 为 missing 的行
dropmissing(df): 删除含有 missing 的所有行
dropmissing!(df, :col): 删除 col 中含 missing 的行；就地修改 df

速查表：替换缺失值

replace!(df.col, missing => mean(skipmissing(df.col))): 将 col 中的 missing 用其均值替换（计算时跳过缺失）

按组替换 missing

for group in groupby(df, :col)
  group[ismissing.(group.col),:col] = value  # 或用该组均值
end

Vamos praticar!

Julia 中的数据操作