writing SparkRDD to a HBase table using Scala

Question

I am trying to write a SparkRDD to HBase table using scala(haven't used before). The entire code is this :

import org.apache.hadoop.hbase.client.{HBaseAdmin, Result}
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor}
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.io.ImmutableBytesWritable    
import scala.collection.JavaConverters._
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark._
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.rdd.PairRDDFunctions
import org.apache.spark.SparkContext._
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.client._

object HBaseWrite {
   def main(args: Array[String]) {
     val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
     val sc = new SparkContext(sparkConf)
     val conf = HBaseConfiguration.create()
     val outputTable = "tablename"

     System.setProperty("user.name", "hdfs")
     System.setProperty("HADOOP_USER_NAME", "hdfs")
     conf.set("hbase.master", "localhost:60000")
     conf.setInt("timeout", 120000)
     conf.set("hbase.zookeeper.quorum", "localhost")
     conf.set("zookeeper.znode.parent", "/hbase-unsecure")
     conf.setInt("hbase.client.scanner.caching", 10000)
     sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
     val jobConfig: JobConf = new JobConf(conf,this.getClass)
     jobConfig.setOutputFormat(classOf[TableOutputFormat])
     jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
     val x = 12
     val y = 15
     val z = 25
     var newarray = Array(x,y,z)
     val newrddtohbase = sc.parallelize(newarray)
     def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
          val p = new Put(Bytes.toBytes(a))
          p.add(Bytes.toBytes("columnfamily"),
          Bytes.toBytes("col_1"), Bytes.toBytes(a))
          new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
     }
     new PairRDDFunctions(newrddtohbase.map(convert)).saveAsHadoopDataset(jobConfig)
     sc.stop()
   }
}

The error I get after doing HBaseWrite(main(Array()) is this:

org.apache.spark.SparkException: Task not serializable

How do I proceed to get it done?

pass your `convert` method as `function literal` inside `map` method, that solves the issue. — Shankar, Oct 22 '16 at 05:42

score 2 · Answer 1 · answered Oct 22 '16 at 04:09

The thing you are doing wrong here is defining the convert inside main If you write this code in this way it may work :

    object HBaseWrite {
       def main(args: Array[String]) {
         val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
         val sc = new SparkContext(sparkConf)
         val conf = HBaseConfiguration.create()
         val outputTable = "tablename"

         System.setProperty("user.name", "hdfs")
         System.setProperty("HADOOP_USER_NAME", "hdfs")
         conf.set("hbase.master", "localhost:60000")
         conf.setInt("timeout", 120000)
         conf.set("hbase.zookeeper.quorum", "localhost")
         conf.set("zookeeper.znode.parent", "/hbase-unsecure")
         conf.setInt("hbase.client.scanner.caching", 10000)
         sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
         val jobConfig: JobConf = new JobConf(conf,this.getClass)
         jobConfig.setOutputFormat(classOf[TableOutputFormat])
         jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
         val x = 12
         val y = 15
         val z = 25
         var newarray = Array(x,y,z)
         val newrddtohbase = sc.parallelize(newarray)
         val convertFunc = convert _
         new PairRDDFunctions(newrddtohbase.map(convertFunc)).saveAsHadoopDataset(jobConfig)
         sc.stop()
       }
       def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }
    }

P.S.: The code is not tested , but it should work !

org.apache.spark.SparkException: Task not serializable at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:166) at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:158) at org.apache.spark.SparkContext.clean(SparkContext.scala:1446) at org.apache.spark.rdd.RDD.map(RDD.scala:286) — Ravi Ranjan, Oct 22 '16 at 04:54
I have not been able to add entire error as it says there are a lot of code in the question and rejects the edit submission. — Ravi Ranjan, Oct 22 '16 at 04:55
@RaviRanjan: Add it as a gist and then provide th link here ! — Shivansh, Oct 22 '16 at 14:12
Create a gist and add the full stack error there and add the link here ! — Shivansh, Oct 22 '16 at 14:23

score 0 · Accepted Answer · answered Oct 22 '16 at 06:06

For example, the below method takes Int as argument and returns Double

var toDouble: (Int) => Double = a => {
    a.toDouble
}

You can use toDouble(2) and it returns 2.0

The same way you can convert your method to function literal as below.

val convert: (Int) => Tuple2[ImmutableBytesWritable,Put] = a => {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }

I wonder why would someone downvote this. It has removed the error for me. — Ravi Ranjan, Oct 24 '16 at 03:45

writing SparkRDD to a HBase table using Scala

2 Answers2

Linked