0

I have added job-server 0.9.0 dependencies in build.sbt by add 

scalaVersion := "2.11.0"
resolvers += "Job Server Bintray" at "https://dl.bintray.com/spark-jobserver/maven"
libraryDependencies ++= Seq(
"spark.jobserver" %% "job-server-api" % "0.9.0",
"org.apache.spark" %% "spark-core" % "2.3.2",
"org.apache.spark" %% "spark-sql" % "2.3.2",
)

However, when I try to test the code like this(copyed from https://github.com/spark-jobserver/spark-jobserver#new-sparkjob-api-with-spark-v21):

import com.typesafe.config.Config
import org.apache.spark.sql.SparkSession
import org.scalactic._
import spark.jobserver.api.{JobEnvironment, SingleProblem, ValidationProblem}
import spark.jobserver.SparkSessionJob
import scala.util.Try
object WordCountExampleSparkSession extends SparkSessionJob {
  type JobData = Seq[String]
  type JobOutput = collection.Map[String, Long]
  override def runJob(sparkSession: SparkSession, runtime: JobEnvironment, data: JobData): JobOutput =
sparkSession.sparkContext.parallelize(data).countByValue

  override def validate(sparkSession: SparkSession, runtime: JobEnvironment, config: Config): JobData Or Every[ValidationProblem] = {
Try(config.getString("input.string").split(" ").toSeq)
  .map(words => Good(words))
  .getOrElse(Bad(One(SingleProblem("No input.string param"))))
  }
}

IntelliJ shows error:"Can not resolve symbol 'SparkSessionJob'" And the jar file looks like this: enter image description here

Ben
  • 1
  • 2

1 Answers1

0

I figured out, the sparkSessionJob located in job-server-extras, so I just have to add

libraryDependencies += "spark.jobserver" %% "job-server-api" % "0.9.0"
Ben
  • 1
  • 2