We have a big data cluster where we have data in Kakfa topics and we load it to hive using spark job (with java 8).I was using Cloudera 7.1.7 version and spark version (2.4.7.7.1.7.1000-141) SP1 and SP2 and even 7.1.6 version. still getting some exceptions. I think there is some permission issue which is not allowing to write to hive. Because i have tried to load some data in table using spark shell for testing, it is getting written in HDFS but not in hive. Below is the screenshot of spark job exceptions and screenshot of the security issue in spark shell -
WARN metadata.Hive: No partition is generated by dynamic partitioning
ERROR streaming.AKafkaSparkStreamingService: null; org.apache.spark.sql.AnalysisException: null;
org.apache.spark.sql.hive.client.HiveClientImpl.loadDynamicPartitions(HiveClientImpl.scala:937)
org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$loadDynamicPartitions$1.apply(HiveExternalCatalog.scala:897)
org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadDynamicPartitions(ExternalCatalogWithListener.scala:185)
org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:212)
org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:101)
org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectBase$class.run(CreateHiveTableAsSelectCommand.scala:55)
org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand.run(CreateHiveTableAsSelectCommand.scala:103)
org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104)
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:704)
org.apache.spark.sql.DataFrameWriter.createTable(DataFrameWriter.scala:502)
org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:481)
org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:429)
com.gamma.skybase.spark.services.loader.TransformedStreamHiveLoader.onTabularDataset(TransformedStreamHiveLoader.java:45)
com.gamma.skybase.spark.services.streaming.avro.AKafkaAvroSparkStreamingService.onInitDataset(AKafkaAvroSparkStreamingService.java:138)
com.gamma.skybase.spark.services.streaming.AKafkaSparkStreamingService.lambda$start$f87052e0$1(AKafkaSparkStreamingService.java:84)
org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:416)
org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:256)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:750) Caused by: java.lang.NullPointerException
org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:3047)
org.apache.spark.sql.hive.client.Shim_cdpd.loadDynamicPartitions(HiveShim.scala:1605)
org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$loadDynamicPartitions$1.apply$mcV$sp(HiveClientImpl.scala:940)
ERROR scheduler.JobScheduler: Error running job streaming job 1692949410000 ms.0
com.gamma.components.exceptions.AppUnexpectedException: Failed processing : , e -> null;
AKafkaSparkStreamingService.lambda$start$f87052e0$1(AKafkaSparkStreamingService.java:87)
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:704)
org.apache.spark.sql.DataFrameWriter.createTable(DataFrameWriter.scala:502)
org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:481)
org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:429)
com.gamma.skybase.spark.services.loader.TransformedStreamHiveLoader.onTabularDataset(TransformedStreamHiveLoader.java:45)
com.gamma.skybase.spark.services.streaming.avro.AKafkaAvroSparkStreamingService.onInitDataset(AKafkaAvroSparkStreamingService.java:138)
com.gamma.skybase.spark.services.streaming.AKafkaSparkStreamingService.lambda$start$f87052e0$1(AKafkaSparkStreamingService.java:84)
Caused by: java.lang.NullPointerException
org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:3047)
org.apache.spark.sql.hive.client.Shim_cdpd.loadDynamicPartitions(HiveShim.scala:1605) org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$loadDynamicPartitions$1.apply$mcV$sp(HiveClientImpl.scala:940)
enter image description hereenter image description hereenter image description here
Your help will be highly appreciated. Thank you.
