【Spark】java.lang.NoSuchMethodException: org.apache.hadoop.hive.ql.metadata.Hive.loadPartition

2/11/14 19:02:23 ERROR [main] SparkUncaughtExceptionHandler: Uncaught exception in thread Thread[main,5,main]
java.lang.NoSuchMethodException: org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(org.apache.hadoop.fs.Path, java.lang.String, java.util.Map, boolean, boolean, boolean, boolean, boolean, boolean)
        at java.lang.Class.getMethod(Class.java:1786)
        at org.apache.spark.sql.hive.client.Shim.findMethod(HiveShim.scala:175)
        at org.apache.spark.sql.hive.client.Shim_v2_1.loadPartitionMethod$lzycompute(HiveShim.scala:1129)
        at org.apache.spark.sql.hive.client.Shim_v2_1.loadPartitionMethod(HiveShim.scala:1128)
        at org.apache.spark.sql.hive.client.Shim_v2_1.loadPartition(HiveShim.scala:1225)
        at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$loadPartition$1.apply$mcV$sp(HiveClientImpl.scala:956)
        at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$loadPartition$1.apply(HiveClientImpl.scala:954)
        at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$loadPartition$1.apply(HiveClientImpl.scala:954)
        at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$withHiveState$1.apply(HiveClientImpl.scala:326)
        at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:257)
        at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:256)
        at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:309)
        at org.apache.spark.sql.hive.client.HiveClientImpl.loadPartition(HiveClientImpl.scala:954)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$loadPartition$1.apply$mcV$sp(HiveExternalCatalog.scala:914)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$loadPartition$1.apply(HiveExternalCatalog.scala:902)
        at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$loadPartition$1.apply(HiveExternalCatalog.scala:902)
        at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:102)
        at org.apache.spark.sql.hive.HiveExternalCatalog.loadPartition(HiveExternalCatalog.scala:902)
        at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadPartition(ExternalCatalogWithListener.scala:185)
        at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:318)
        at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:100)
        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104)
        at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:102)
        at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:115)
        at org.apache.spark.sql.execution.adaptive.QueryStage.executeCollect(QueryStage.scala:243)
        at org.apache.spark.sql.Dataset$$anonfun$6.apply(Dataset.scala:195)
        at org.apache.spark.sql.Dataset$$anonfun$6.apply(Dataset.scala:195)
        at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3441)
        at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:92)
        at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:139)
        at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:87)
        at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withAction(Dataset.scala:3440)
        at org.apache.spark.sql.Dataset.<init>(Dataset.scala:195)
        at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:80)
        at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:652)
        at com.kering.etl.runner.BaozunOdsBatchEtl$.sink(BaozunOdsBatchEtl.scala:62)
        at com.kering.etl.runner.BaozunOdsBatchEtl$.execute(BaozunOdsBatchEtl.scala:134)
        at com.kering.etl.runner.BaozunOdsBatchEtl$.main(BaozunOdsBatchEtl.scala:138)
        at com.kering.etl.runner.BaozunOdsBatchEtl.main(BaozunOdsBatchEtl.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

解决办法:检查插入的字段数据类型和表字段类型是否一致。

 

posted @ 2022-11-14 19:04  梦醒江南·Infinite  阅读(480)  评论(0编辑  收藏  举报