linux 中awk 根据多列读数据进行去重复

 

001、

(base) [b20223040323@admin1 test2]$ ls
test.txt
(base) [b20223040323@admin1 test2]$ cat test.txt   ## 测试数据如下；根据第一列和第三列对数据进行去重复
ID=gene-RIN1    rna-XM_018043206.1      3615
ID=gene-STRIP2  rna-XM_018046935.1      3917
ID=gene-STRIP2  rna-XM_018046937.1      3917
ID=gene-STRIP2  rna-XM_018046937.1      3917
ID=gene-SLC35D2 rna-XM_018052476.1      1603
ID=gene-DENND6B rna-XM_018048970.1      4265
ID=gene-PSMD13  rna-XM_005700085.3      1683
ID=gene-LAT     rna-XM_018040785.1      1790
ID=gene-LAT     rna-XM_018040784.1      1790
ID=gene-PSMD14  rna-XM_005676052.2      1908

 

002、

(base) [b20223040323@admin1 test2]$ awk '{if(ay1[$1] == "") {ay1[$1] = "xxx"; tmp = $1$3; print $0; next}; if(tmp == $1$3) {print $0, "dup"}}' test.txt
ID=gene-RIN1    rna-XM_018043206.1      3615            ## 标记处重复
ID=gene-STRIP2  rna-XM_018046935.1      3917
ID=gene-STRIP2  rna-XM_018046937.1      3917 dup
ID=gene-STRIP2  rna-XM_018046937.1      3917 dup
ID=gene-SLC35D2 rna-XM_018052476.1      1603
ID=gene-DENND6B rna-XM_018048970.1      4265
ID=gene-PSMD13  rna-XM_005700085.3      1683
ID=gene-LAT     rna-XM_018040785.1      1790
ID=gene-LAT     rna-XM_018040784.1      1790 dup
ID=gene-PSMD14  rna-XM_005676052.2      1908
(base) [b20223040323@admin1 test2]$ awk '{if(ay1[$1] == "") {ay1[$1] = "xxx"; tmp = $1$3; print $0; next}; if(tmp == $1$3) {next}}' test.txt
ID=gene-RIN1    rna-XM_018043206.1      3615               ## 去重复
ID=gene-STRIP2  rna-XM_018046935.1      3917
ID=gene-SLC35D2 rna-XM_018052476.1      1603
ID=gene-DENND6B rna-XM_018048970.1      4265
ID=gene-PSMD13  rna-XM_005700085.3      1683
ID=gene-LAT     rna-XM_018040785.1      1790
ID=gene-PSMD14  rna-XM_005676052.2      1908

 

003、简洁做法

[root@pc1 test1]# ls
test.txt
[root@pc1 test1]# cat test.txt                       ## 测试文本
ID=gene-RIN1    rna-XM_018043206.1      3615
ID=gene-STRIP2  rna-XM_018046935.1      3917
ID=gene-STRIP2  rna-XM_018046937.1      3917
ID=gene-STRIP2  rna-XM_018046937.1      3917
ID=gene-SLC35D2 rna-XM_018052476.1      1603
ID=gene-DENND6B rna-XM_018048970.1      4265
ID=gene-PSMD13  rna-XM_005700085.3      1683
ID=gene-LAT     rna-XM_018040785.1      1790
ID=gene-LAT     rna-XM_018040784.1      1790
ID=gene-PSMD14  rna-XM_005676052.2      1908
[root@pc1 test1]# awk '!ay1[$1$3]++' test.txt        ## 根据1、3列去重复，并保持原来的顺序
ID=gene-RIN1    rna-XM_018043206.1      3615
ID=gene-STRIP2  rna-XM_018046935.1      3917
ID=gene-SLC35D2 rna-XM_018052476.1      1603
ID=gene-DENND6B rna-XM_018048970.1      4265
ID=gene-PSMD13  rna-XM_005700085.3      1683
ID=gene-LAT     rna-XM_018040785.1      1790
ID=gene-PSMD14  rna-XM_005676052.2      1908

 。