Hudi表数据重复原因

测试中，发现虽然显示设置为 upsert，且也按规范设置了 primaryKey、preCombineField，type 等，但查出的结果仍然存在重复。反复测试，重复的数据稳定为 2，且同一数据的一笔提交时间也保持不变。结果显示同一数据分区相同，但来自不同的 HDFS 文件。

相关Issue

这个问题的原因是 drop 掉一张 hudi 表时，对应的表文件没有被删除，导致以同样路径再次创建后，select 仍然能查出来，这时显示重复的原因。但反复 create 和 drop，总是只有两个文件，并没有增加新的。

Flink SQL> CREATE TABLE `default_catalog`.`default_database`.`t06` (
>   `ds` BIGINT,
>   `ut` VARCHAR(2147483647),
>   `pk` BIGINT NOT NULL,
>   `f0` BIGINT,
>   `f1` BIGINT,
>   `f2` BIGINT,
>   `f3` BIGINT,
>   CONSTRAINT `PK_3610` PRIMARY KEY (`pk`) NOT ENFORCED
> ) PARTITIONED BY (`ds`)
> WITH (
>   'connector' = 'hudi',
>   'index.type' = 'BUCKET',
>   'payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.compaction.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.datasource.write.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'path' = 'hdfs:///user/root/hudi/t06',
>   'connector' = 'hudi',
>   'precombine.field' = 'ut',
>   'table.type' = 'MERGE_ON_READ',
>   'write.operation' = 'upsert',
>   'hoodie.bucket.index.num.buckets' = '2'
> )
> ;
[INFO] Execute statement succeed.

Flink SQL> insert into t06 (ds,ut,pk,f0) select 20230101 ds,CAST(current_timestamp AS STRING) ut,1001 pk,1 f0;
[INFO] Submitting SQL update statement to the cluster...
[INFO] SQL update statement has been successfully submitted to the cluster:
Job ID: e4e4c8e8ab8c904ad9238336cbf8fb8f


Flink SQL> select * from t06;
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| op |                   ds |                             ut |                   pk |                   f0 |                   f1 |                   f2 |                   f3 |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| +I |             20230101 |        2023-05-16 12:44:54.004 |                 1001 |                    1 |               <NULL> |               <NULL> |               <NULL> |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
Received a total of 1 row

Flink SQL> drop table t06;
[INFO] Execute statement succeed.

Flink SQL> CREATE TABLE `default_catalog`.`default_database`.`t06` (
>   `ds` BIGINT,
>   `ut` VARCHAR(2147483647),
>   `pk` BIGINT NOT NULL,
>   `f0` BIGINT,
>   `f1` BIGINT,
>   `f2` BIGINT,
>   `f3` BIGINT,
>   CONSTRAINT `PK_3610` PRIMARY KEY (`pk`) NOT ENFORCED
> ) PARTITIONED BY (`ds`)
> WITH (
>   'connector' = 'hudi',
>   'index.type' = 'BUCKET',
>   'payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.compaction.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.datasource.write.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'path' = 'hdfs:///user/root/hudi/t06',
>   'connector' = 'hudi',
>   'precombine.field' = 'ut',
>   'table.type' = 'MERGE_ON_READ',
>   'write.operation' = 'upsert',
>   'hoodie.bucket.index.num.buckets' = '2'
> );
[INFO] Execute statement succeed.

Flink SQL> select * from t06;
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| op |                   ds |                             ut |                   pk |                   f0 |                   f1 |                   f2 |                   f3 |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| +I |             20230101 |        2023-05-16 12:44:54.004 |                 1001 |                    1 |               <NULL> |               <NULL> |               <NULL> |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
Received a total of 1 row

Flink SQL> insert into t06 (ds,ut,pk,f0) select 20230101 ds,CAST(current_timestamp AS STRING) ut,1001 pk,1 f0;
[INFO] Submitting SQL update statement to the cluster...
[INFO] SQL update statement has been successfully submitted to the cluster:
Job ID: 28e29ce22999e479dd14fc1d17bbda1e


Flink SQL> select * from t06;
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| op |                   ds |                             ut |                   pk |                   f0 |                   f1 |                   f2 |                   f3 |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| +I |             20230101 |        2023-05-16 12:45:41.939 |                 1001 |                    1 |               <NULL> |               <NULL> |               <NULL> |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
Received a total of 1 row

Flink SQL> drop table t06;
[INFO] Execute statement succeed.

Flink SQL> CREATE TABLE `default_catalog`.`default_database`.`t06` (
>   `ds` BIGINT,
>   `ut` VARCHAR(2147483647),
>   `pk` BIGINT NOT NULL,
>   `f0` BIGINT,
>   `f1` BIGINT,
>   `f2` BIGINT,
>   `f3` BIGINT,
>   CONSTRAINT `PK_3610` PRIMARY KEY (`pk`) NOT ENFORCED
> ) PARTITIONED BY (`ds`)
> WITH (
>   'connector' = 'hudi',
>   'index.type' = 'BUCKET',
>   'payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.compaction.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'hoodie.datasource.write.payload.class' = 'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
>   'path' = 'hdfs:///user/root/hudi/t06',
>   'connector' = 'hudi',
>   'precombine.field' = 'ut',
>   'table.type' = 'MERGE_ON_READ',
>   'write.operation' = 'upsert',
>   'hoodie.bucket.index.num.buckets' = '2'
> );
[INFO] Execute statement succeed.

Flink SQL> select * from t06;
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| op |                   ds |                             ut |                   pk |                   f0 |                   f1 |                   f2 |                   f3 |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| +I |             20230101 |        2023-05-16 12:45:41.939 |                 1001 |                    1 |               <NULL> |               <NULL> |               <NULL> |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
Received a total of 1 row

Flink SQL>  insert into t06 (ds,ut,pk,f0) select 20230101 ds,CAST(current_timestamp AS STRING) ut,1001 pk,10 f0;
[INFO] Submitting SQL update statement to the cluster...
[INFO] SQL update statement has been successfully submitted to the cluster:
Job ID: b9c7ef4dcdbcf743e609edb92216272e


Flink SQL> select * from t06;
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| op |                   ds |                             ut |                   pk |                   f0 |                   f1 |                   f2 |                   f3 |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
| +I |             20230101 |        2023-05-16 12:46:33.156 |                 1001 |                   10 |               <NULL> |               <NULL> |               <NULL> |
+----+----------------------+--------------------------------+----------------------+----------------------+----------------------+----------------------+----------------------+
Received a total of 1 row

Flink SQL>