I am trying to expose an Apache Spark ML code as a REST service using Scalatra.
But while calling the GET on the REST endpoint, I receive the following error:
java.lang.ClassNotFoundException: scala.Some at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) ~[spark-core_2.11-2.0.0.jar:2.0.0]
Do I need to add some additional dependencies?
I have the following dependencies listed in my SBT file:
lazy val scalatraVersion = "2.3.0"
lazy val sparkVersion = "2.0.0"
libraryDependencies ++= Seq(
"org.scalatra" %% "scalatra" % scalatraVersion,
"org.scalatra" %% "scalatra-json" % scalatraVersion,
"org.scalatra" %% "scalatra-commands" % scalatraVersion,
"org.json4s" %% "json4s-jackson" % "3.2.9",
"ch.qos.logback" % "logback-classic" % "1.1.3" ,
"org.eclipse.jetty" % "jetty-webapp" % "9.2.10.v20150310",
"com.typesafe.akka" %% "akka-actor" % "2.3.4",
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion,
"org.apache.spark" %% "spark-mllib" % sparkVersion,
"com.github.fommil.netlib" % "all" % "1.1.2"
)
In my scala code IssuePredictor.predict(), I am trying to load the CrossValidatorModel from an already saved file. And this is where the error is thrown:
(I know this because, the println statements before this line are printed, but those after it aren't)
val cvModel = CrossValidatorModel.load("/Users/Bjghosh/MyFiles/Apache Spark/ml/tables/IssuePredictorCVModelNB_2.0")
I call my scala code that loads the model, gets the prediction and returns, as shown below:
class ScalatraStep1 extends ScalatraServlet {
notFound { "Route not implemented" }
get("/predict/:rawText") {
Try { params("rawText").toString } match {
case Success(id) => { println(s"Predict rawText: ${params("rawText")} ")
val predictor = new IssuePredictor
val prediction = predictor.predict(params("rawText"))
s"Prediction for rawText: ${params("rawText")} = $prediction} "
}
case Failure(e) => BadRequest(reason = "Can't understand rawText")
}
}
}
Error Trace
ERROR org.apache.spark.util.Utils - Exception encountered java.lang.ClassNotFoundException: scala.Some at java.net.URLClassLoader$1.run(URLClassLoader.java:372) ~[na:1.8.0_25] at java.net.URLClassLoader$1.run(URLClassLoader.java:361) ~[na:1.8.0_25] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_25] at java.net.URLClassLoader.findClass(URLClassLoader.java:360) ~[na:1.8.0_25] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_25] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_25] at java.lang.Class.forName0(Native Method) ~[na:1.8.0_25] at java.lang.Class.forName(Class.java:344) ~[na:1.8.0_25] at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) ~[spark-core_2.11-2.0.0.jar:2.0.0] at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1613) [na:1.8.0_25] at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1518) [na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1774) [na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) [na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [na:1.8.0_25] at java.io.ObjectInputStream.defaultReadObject(ObjectInputStream.java:501) [na:1.8.0_25] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply$mcV$sp(AccumulatorV2.scala:167) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply(AccumulatorV2.scala:166) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply(AccumulatorV2.scala:166) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1253) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2.readObject(AccumulatorV2.scala:166) [spark-core_2.11-2.0.0.jar:2.0.0] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_25] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_25] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_25] at java.lang.reflect.Method.invoke(Method.java:483) ~[na:1.8.0_25] at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017) [na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896) [na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) [na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) [na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [na:1.8.0_25] at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) [na:1.8.0_25] at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) [spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) [spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:253) [spark-core_2.11-2.0.0.jar:2.0.0] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_25] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_25] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_25] 12:07:36.845 [Executor task launch worker-0] ERROR org.apache.spark.executor.Executor - Exception in task 0.0 in stage 0.0 (TID 0) java.io.IOException: java.lang.ClassNotFoundException: scala.Some at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1260) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2.readObject(AccumulatorV2.scala:166) ~[spark-core_2.11-2.0.0.jar:2.0.0] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_25] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_25] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_25] at java.lang.reflect.Method.invoke(Method.java:483) ~[na:1.8.0_25] at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017) ~[na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896) ~[na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) ~[na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) ~[na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) ~[na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) ~[na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) ~[na:1.8.0_25] at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:253) ~[spark-core_2.11-2.0.0.jar:2.0.0] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_25] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_25] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_25] Caused by: java.lang.ClassNotFoundException: scala.Some at java.net.URLClassLoader$1.run(URLClassLoader.java:372) ~[na:1.8.0_25] at java.net.URLClassLoader$1.run(URLClassLoader.java:361) ~[na:1.8.0_25] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_25] at java.net.URLClassLoader.findClass(URLClassLoader.java:360) ~[na:1.8.0_25] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_25] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_25] at java.lang.Class.forName0(Native Method) ~[na:1.8.0_25] at java.lang.Class.forName(Class.java:344) ~[na:1.8.0_25] at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) ~[spark-core_2.11-2.0.0.jar:2.0.0] at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1613) ~[na:1.8.0_25] at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1518) ~[na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1774) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) ~[na:1.8.0_25] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) ~[na:1.8.0_25] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[na:1.8.0_25] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[na:1.8.0_25] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) ~[na:1.8.0_25] at java.io.ObjectInputStream.defaultReadObject(ObjectInputStream.java:501) ~[na:1.8.0_25] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply$mcV$sp(AccumulatorV2.scala:167) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply(AccumulatorV2.scala:166) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.AccumulatorV2$$anonfun$readObject$1.apply(AccumulatorV2.scala:166) ~[spark-core_2.11-2.0.0.jar:2.0.0] at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1253) ~[spark-core_2.11-2.0.0.jar:2.0.0]
Regards, Bhaskar
