1
Issue Description:

Spark Version: 1.6.2
Execution: Spark-shell (REPL) master = local[2] (tried local[*])

example.json is as below:

{"name":"D2" ,"lovesPandas":"Y"}
{"name":"D3" ,"lovesPandas":"Y"}
{"name":"D4" ,"lovesPandas":"Y"}
{"name":"D5" ,"lovesPandas":"Y"}

Code executing in Spark-shell local mode:

import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._ 
import org.apache.spark.streaming.kafka._
import org.apache.spark.sql._
import org.json4s._
import org.json4s.jackson.JsonMethods._
import _root_.kafka.serializer.StringDecoder
import _root_.kafka.serializer.Decoder
import _root_.kafka.utils.VerifiableProperties
import org.apache.hadoop.hbase._
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapred.JobConf
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext

val ssc = new  StreamingContext(sc,  Seconds(2) )
val messages = ssc.textFileStream("C:\\pdtemp\\test\\example.json")

messages.print()

I tried the saveAsTextFiles but it is not saving any files too.

this does not work -- shows no output -- tried the same with reading stream from Kafka on a spark-shell

tried the following too -- does not work:

messages.foreachRDD(rdd => rdd.foreach(print))

Also, tried parsing the schema converting to dataframe but nothing seems to work

normal json parsing is working and i can print the contents of normal //RDD/DF //to console in Spark-shell

Can anyone help, please?

RGuy
  • 11
  • 2
  • I tried spark-submit and jar file option with --master local[2]. However, it is not working either. – RGuy Jul 31 '16 at 15:48
  • ok solved the issue....sorry, oversight...did not start the streaming context ssc.start() ; ssc.awaitTermination() – RGuy Jul 31 '16 at 21:45

0 Answers0