1

I'm running a jar file with Spark-submit, but i keep getting this error:

Error: Failed to load class antarctic.DataQuality.

This is the command:

spark-submit --class antarctic.DataQuality --master local[*] --deploy-mode client --jars "path/to/file.jar" "arg 1" "arg 2" "arg 3" "arg 4"

This is the structure of the Scala file: Scala Structure

The command is being run in target/scala-2.11

I also have this enviroment variables defined:

JAVA_HOME: C:\Program Files\Java\jdk1.8.0_251
HADOOP_HOME: C:\winutils
SPARK_HOME: C:\Users\Spark
SPARK_USER: C:\Users\Spark

build.sbt:

name := "DataQuality"

version := "0.1"

scalaVersion := "2.11.8"

val sparkVersion = "2.3.0"
// https://mvnrepository.com/artifact/org.apache.spark/spark-core
libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % sparkVersion,
  "org.apache.spark" %% "spark-sql" % sparkVersion,
  "org.apache.spark" %% "spark-mllib" % sparkVersion,
  "org.apache.spark" %% "spark-streaming" % sparkVersion,
  "org.apache.spark" %% "spark-hive" % sparkVersion
)

// https://github.com/awslabs/deequ
libraryDependencies += "com.amazon.deequ" % "deequ" % "1.0.2-antarctic" from "file:///C:/Users/AKAINIX ANALYTICS/Documents/Lucas/Antarctic/Bitbucket/plataforma-dataquality/Backend/deequ/target/deequ-1.0.2-antarctic.jar"

//https://github.com/spray/spray-json
//libraryDependencies += "io.spray" %%  "spray-json" % "1.3.5"

//https://github.com/scopt/scopt
libraryDependencies += "com.github.scopt" %% "scopt" % "4.0.0-RC2"

//https://circe.github.io/circe/
val circeVersion = "0.7.0"
libraryDependencies ++= Seq(
  "io.circe"  %% "circe-core"     % circeVersion,
  "io.circe"  %% "circe-generic"  % circeVersion,
  "io.circe"  %% "circe-parser"   % circeVersion
)

// https://mvnrepository.com/artifact/mysql/mysql-connector-java
libraryDependencies += "mysql" % "mysql-connector-java" % "8.0.19"

// https://mvnrepository.com/artifact/org.postgresql/postgresql
libraryDependencies += "org.postgresql" % "postgresql" % "42.2.5"

//https://github.com/lightbend/config
libraryDependencies += "com.typesafe" % "config" % "1.4.0"

// https://mvnrepository.com/artifact/org.mongodb.spark/mongo-spark-connector
libraryDependencies += "org.mongodb.spark" %% "mongo-spark-connector" % "2.3.0"

0 Answers0