1

I am trying to write the following Spark dataframe into Couchbase:

import com.couchbase.spark.sql._
   import com.couchbase.spark._
   import com.couchbase.client.java.document.JsonDocument
   object Test{
   case class Person(name: String, relation: String, uid:String)
  def main(args: Array[String]) {
    val conf = new SparkConf()
    conf.set("spark.couchbase.nodes", "xxx.cloud.io")
    conf.set("spark.couchbase.username", "Administrator")
    conf.set("com.couchbase.password", "xxx")
    conf.set("com.couchbase.bucket.xx", "xx")
    conf.set("com.couchbase.connectTimeout", "50000")

    val sc = new SparkContext(conf)
    val hiveCtx = new HiveContext(sc)

    import hiveCtx.implicits._


    val people = sc.parallelize(Seq(
    Person("user::michael", "Michael", "27"),
    Person("user::tom", "Tom", "33")
      )).toDF()

    val couchbaseConf =  scala.collection.immutable.Map("bucket" -> "xx", "idField" -> "uid")

    people.write.couchbase(couchbaseConf)

  
  }

}

I use this spark-submit:

spark-submit --class Test  --master yarn-cluster --deploy-mode cluster --queue default --jars spark-connector_2.10-1.2.0.jar,couchbase-client-1.4.12.jar,core-io-1.3.3.jar,java-client-2.4.2.jar test_2.10-1.0.jar

but I face the following error:

java.lang.NoClassDefFoundError: rx/functions/Func2

Any information which could help me is highly appreciated.

Matthew Groves
  • 25,181
  • 9
  • 71
  • 121
R.al
  • 45
  • 6

0 Answers0