You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-user@hadoop.apache.org by siva kumar <si...@gmail.com> on 2016/02/05 14:20:35 UTC
spark streaming
Hi ,
Im trying to pull data from twitter and do some processing out of
it. I found this code
import org.apache.spark.streaming._import
org.apache.spark.streaming.twitter._import
org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport
scala.collection.mutable.HashMapimport java.io.Fileimport
org.apache.log4j.Loggerimport org.apache.log4j.Levelimport
sys.process.stringSeqToProcess
/** Configures the Oauth Credentials for accessing Twitter */def
configureTwitterCredentials(apiKey: String, apiSecret: String,
accessToken: String, accessTokenSecret: String) {
val configs = new HashMap[String, String] ++= Seq(
"apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" ->
accessToken, "accessTokenSecret" -> accessTokenSecret)
println("Configuring Twitter OAuth")
configs.foreach{ case(key, value) =>
if (value.trim.isEmpty) {
throw new Exception("Error setting authentication - value for "
+ key + " not set")
}
val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
System.setProperty(fullKey, value.trim)
println("\tProperty " + fullKey + " set as [" + value.trim + "]")
}
println()}
// Configure Twitter credentialsval apiKey =
"xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret =
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken =
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val
accessTokenSecret =
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey,
apiSecret, accessToken, accessTokenSecret)
import org.apache.spark.streaming.twitter._val ssc = new
StreamingContext(sc, Seconds(2))val tweets =
TwitterUtils.createStream(ssc, None)val twt =
tweets.window(Seconds(60))
case class Tweet(createdAt:Long, text:String)twt.map(status=>
Tweet(status.getCreatedAt().getTime()/1000,
status.getText())).foreachRDD(rdd=>
// Below line works only in spark 1.3.0.
// For spark 1.1.x and spark 1.2.x,
// use rdd.registerTempTable("tweets") instead.
rdd.toDF().registerAsTable("tweets"))
twt.print
ssc.start()
But getting the below error
import org.apache.spark.streaming.twitter._
<console>:19: error: object twitter is not a member of package
org.apache.spark.streaming
import org.apache.spark.streaming.twitter._
I using spark 1.3.0
The same worked for spark 0.9.0 . Any help?
Re: spark streaming
Posted by Ted Yu <yu...@gmail.com>.
Siva:
How did you build your project ?
You can use mvn denpendency:tree to find out the version
of spark-streaming-twitter
or take a look at ./external/twitter/pom.xml
ls ~/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/
1.4.0-SNAPSHOT/ 1.5.0-SNAPSHOT/ 1.5.2/
1.6.0-SNAPSHOT/ 2.0.0-SNAPSHOT/
maven-metadata-local.xml
$ jar tvf
/home/hbase/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/1.5.2/spark-streaming-twitter_2.10-1.5.2.jar
| head
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/twitter/
4749 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterUtils.class
3519 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream.class
1387 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream$$anonfun$1.class
FYI
On Fri, Feb 5, 2016 at 1:32 PM, Rohan Rajeevan <ro...@gmail.com>
wrote:
> Did you try using Twitter4J? http://twitter4j.org/en/index.html
>
> On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
>
>> Hi ,
>> Im trying to pull data from twitter and do some processing out
>> of it. I found this code
>>
>> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
>> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
>> val configs = new HashMap[String, String] ++= Seq(
>> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
>> println("Configuring Twitter OAuth")
>> configs.foreach{ case(key, value) =>
>> if (value.trim.isEmpty) {
>> throw new Exception("Error setting authentication - value for " + key + " not set")
>> }
>> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
>> System.setProperty(fullKey, value.trim)
>> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
>> }
>> println()}
>> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
>> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
>> case class Tweet(createdAt:Long, text:String)twt.map(status=>
>> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
>> // Below line works only in spark 1.3.0.
>> // For spark 1.1.x and spark 1.2.x,
>> // use rdd.registerTempTable("tweets") instead.
>> rdd.toDF().registerAsTable("tweets"))
>> twt.print
>> ssc.start()
>>
>> But getting the below error
>>
>> import org.apache.spark.streaming.twitter._
>> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
>> import org.apache.spark.streaming.twitter._
>>
>> I using spark 1.3.0
>>
>>
>> The same worked for spark 0.9.0 . Any help?
>>
>
>
Re: spark streaming
Posted by Ted Yu <yu...@gmail.com>.
Siva:
How did you build your project ?
You can use mvn denpendency:tree to find out the version
of spark-streaming-twitter
or take a look at ./external/twitter/pom.xml
ls ~/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/
1.4.0-SNAPSHOT/ 1.5.0-SNAPSHOT/ 1.5.2/
1.6.0-SNAPSHOT/ 2.0.0-SNAPSHOT/
maven-metadata-local.xml
$ jar tvf
/home/hbase/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/1.5.2/spark-streaming-twitter_2.10-1.5.2.jar
| head
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/twitter/
4749 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterUtils.class
3519 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream.class
1387 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream$$anonfun$1.class
FYI
On Fri, Feb 5, 2016 at 1:32 PM, Rohan Rajeevan <ro...@gmail.com>
wrote:
> Did you try using Twitter4J? http://twitter4j.org/en/index.html
>
> On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
>
>> Hi ,
>> Im trying to pull data from twitter and do some processing out
>> of it. I found this code
>>
>> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
>> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
>> val configs = new HashMap[String, String] ++= Seq(
>> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
>> println("Configuring Twitter OAuth")
>> configs.foreach{ case(key, value) =>
>> if (value.trim.isEmpty) {
>> throw new Exception("Error setting authentication - value for " + key + " not set")
>> }
>> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
>> System.setProperty(fullKey, value.trim)
>> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
>> }
>> println()}
>> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
>> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
>> case class Tweet(createdAt:Long, text:String)twt.map(status=>
>> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
>> // Below line works only in spark 1.3.0.
>> // For spark 1.1.x and spark 1.2.x,
>> // use rdd.registerTempTable("tweets") instead.
>> rdd.toDF().registerAsTable("tweets"))
>> twt.print
>> ssc.start()
>>
>> But getting the below error
>>
>> import org.apache.spark.streaming.twitter._
>> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
>> import org.apache.spark.streaming.twitter._
>>
>> I using spark 1.3.0
>>
>>
>> The same worked for spark 0.9.0 . Any help?
>>
>
>
Re: spark streaming
Posted by Ted Yu <yu...@gmail.com>.
Siva:
How did you build your project ?
You can use mvn denpendency:tree to find out the version
of spark-streaming-twitter
or take a look at ./external/twitter/pom.xml
ls ~/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/
1.4.0-SNAPSHOT/ 1.5.0-SNAPSHOT/ 1.5.2/
1.6.0-SNAPSHOT/ 2.0.0-SNAPSHOT/
maven-metadata-local.xml
$ jar tvf
/home/hbase/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/1.5.2/spark-streaming-twitter_2.10-1.5.2.jar
| head
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/twitter/
4749 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterUtils.class
3519 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream.class
1387 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream$$anonfun$1.class
FYI
On Fri, Feb 5, 2016 at 1:32 PM, Rohan Rajeevan <ro...@gmail.com>
wrote:
> Did you try using Twitter4J? http://twitter4j.org/en/index.html
>
> On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
>
>> Hi ,
>> Im trying to pull data from twitter and do some processing out
>> of it. I found this code
>>
>> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
>> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
>> val configs = new HashMap[String, String] ++= Seq(
>> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
>> println("Configuring Twitter OAuth")
>> configs.foreach{ case(key, value) =>
>> if (value.trim.isEmpty) {
>> throw new Exception("Error setting authentication - value for " + key + " not set")
>> }
>> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
>> System.setProperty(fullKey, value.trim)
>> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
>> }
>> println()}
>> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
>> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
>> case class Tweet(createdAt:Long, text:String)twt.map(status=>
>> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
>> // Below line works only in spark 1.3.0.
>> // For spark 1.1.x and spark 1.2.x,
>> // use rdd.registerTempTable("tweets") instead.
>> rdd.toDF().registerAsTable("tweets"))
>> twt.print
>> ssc.start()
>>
>> But getting the below error
>>
>> import org.apache.spark.streaming.twitter._
>> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
>> import org.apache.spark.streaming.twitter._
>>
>> I using spark 1.3.0
>>
>>
>> The same worked for spark 0.9.0 . Any help?
>>
>
>
Re: spark streaming
Posted by Ted Yu <yu...@gmail.com>.
Siva:
How did you build your project ?
You can use mvn denpendency:tree to find out the version
of spark-streaming-twitter
or take a look at ./external/twitter/pom.xml
ls ~/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/
1.4.0-SNAPSHOT/ 1.5.0-SNAPSHOT/ 1.5.2/
1.6.0-SNAPSHOT/ 2.0.0-SNAPSHOT/
maven-metadata-local.xml
$ jar tvf
/home/hbase/.m2/repository/org/apache/spark/spark-streaming-twitter_2.10/1.5.2/spark-streaming-twitter_2.10-1.5.2.jar
| head
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/
0 Wed Oct 28 09:20:40 PDT 2015 org/apache/spark/streaming/twitter/
4749 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterUtils.class
3519 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream.class
1387 Wed Oct 28 09:20:40 PDT 2015
org/apache/spark/streaming/twitter/TwitterInputDStream$$anonfun$1.class
FYI
On Fri, Feb 5, 2016 at 1:32 PM, Rohan Rajeevan <ro...@gmail.com>
wrote:
> Did you try using Twitter4J? http://twitter4j.org/en/index.html
>
> On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
>
>> Hi ,
>> Im trying to pull data from twitter and do some processing out
>> of it. I found this code
>>
>> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
>> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
>> val configs = new HashMap[String, String] ++= Seq(
>> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
>> println("Configuring Twitter OAuth")
>> configs.foreach{ case(key, value) =>
>> if (value.trim.isEmpty) {
>> throw new Exception("Error setting authentication - value for " + key + " not set")
>> }
>> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
>> System.setProperty(fullKey, value.trim)
>> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
>> }
>> println()}
>> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
>> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
>> case class Tweet(createdAt:Long, text:String)twt.map(status=>
>> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
>> // Below line works only in spark 1.3.0.
>> // For spark 1.1.x and spark 1.2.x,
>> // use rdd.registerTempTable("tweets") instead.
>> rdd.toDF().registerAsTable("tweets"))
>> twt.print
>> ssc.start()
>>
>> But getting the below error
>>
>> import org.apache.spark.streaming.twitter._
>> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
>> import org.apache.spark.streaming.twitter._
>>
>> I using spark 1.3.0
>>
>>
>> The same worked for spark 0.9.0 . Any help?
>>
>
>
Re: spark streaming
Posted by Rohan Rajeevan <ro...@gmail.com>.
Did you try using Twitter4J? http://twitter4j.org/en/index.html
On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
> Hi ,
> Im trying to pull data from twitter and do some processing out of
> it. I found this code
>
> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
> val configs = new HashMap[String, String] ++= Seq(
> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
> println("Configuring Twitter OAuth")
> configs.foreach{ case(key, value) =>
> if (value.trim.isEmpty) {
> throw new Exception("Error setting authentication - value for " + key + " not set")
> }
> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
> System.setProperty(fullKey, value.trim)
> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
> }
> println()}
> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
> case class Tweet(createdAt:Long, text:String)twt.map(status=>
> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
> // Below line works only in spark 1.3.0.
> // For spark 1.1.x and spark 1.2.x,
> // use rdd.registerTempTable("tweets") instead.
> rdd.toDF().registerAsTable("tweets"))
> twt.print
> ssc.start()
>
> But getting the below error
>
> import org.apache.spark.streaming.twitter._
> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
> import org.apache.spark.streaming.twitter._
>
> I using spark 1.3.0
>
>
> The same worked for spark 0.9.0 . Any help?
>
Re: spark streaming
Posted by Rohan Rajeevan <ro...@gmail.com>.
Did you try using Twitter4J? http://twitter4j.org/en/index.html
On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
> Hi ,
> Im trying to pull data from twitter and do some processing out of
> it. I found this code
>
> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
> val configs = new HashMap[String, String] ++= Seq(
> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
> println("Configuring Twitter OAuth")
> configs.foreach{ case(key, value) =>
> if (value.trim.isEmpty) {
> throw new Exception("Error setting authentication - value for " + key + " not set")
> }
> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
> System.setProperty(fullKey, value.trim)
> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
> }
> println()}
> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
> case class Tweet(createdAt:Long, text:String)twt.map(status=>
> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
> // Below line works only in spark 1.3.0.
> // For spark 1.1.x and spark 1.2.x,
> // use rdd.registerTempTable("tweets") instead.
> rdd.toDF().registerAsTable("tweets"))
> twt.print
> ssc.start()
>
> But getting the below error
>
> import org.apache.spark.streaming.twitter._
> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
> import org.apache.spark.streaming.twitter._
>
> I using spark 1.3.0
>
>
> The same worked for spark 0.9.0 . Any help?
>
Re: spark streaming
Posted by Rohan Rajeevan <ro...@gmail.com>.
Did you try using Twitter4J? http://twitter4j.org/en/index.html
On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
> Hi ,
> Im trying to pull data from twitter and do some processing out of
> it. I found this code
>
> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
> val configs = new HashMap[String, String] ++= Seq(
> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
> println("Configuring Twitter OAuth")
> configs.foreach{ case(key, value) =>
> if (value.trim.isEmpty) {
> throw new Exception("Error setting authentication - value for " + key + " not set")
> }
> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
> System.setProperty(fullKey, value.trim)
> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
> }
> println()}
> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
> case class Tweet(createdAt:Long, text:String)twt.map(status=>
> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
> // Below line works only in spark 1.3.0.
> // For spark 1.1.x and spark 1.2.x,
> // use rdd.registerTempTable("tweets") instead.
> rdd.toDF().registerAsTable("tweets"))
> twt.print
> ssc.start()
>
> But getting the below error
>
> import org.apache.spark.streaming.twitter._
> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
> import org.apache.spark.streaming.twitter._
>
> I using spark 1.3.0
>
>
> The same worked for spark 0.9.0 . Any help?
>
Re: spark streaming
Posted by Rohan Rajeevan <ro...@gmail.com>.
Did you try using Twitter4J? http://twitter4j.org/en/index.html
On Fri, Feb 5, 2016 at 5:20 AM, siva kumar <si...@gmail.com> wrote:
> Hi ,
> Im trying to pull data from twitter and do some processing out of
> it. I found this code
>
> import org.apache.spark.streaming._import org.apache.spark.streaming.twitter._import org.apache.spark.storage.StorageLevelimport scala.io.Sourceimport scala.collection.mutable.HashMapimport java.io.Fileimport org.apache.log4j.Loggerimport org.apache.log4j.Levelimport sys.process.stringSeqToProcess
> /** Configures the Oauth Credentials for accessing Twitter */def configureTwitterCredentials(apiKey: String, apiSecret: String, accessToken: String, accessTokenSecret: String) {
> val configs = new HashMap[String, String] ++= Seq(
> "apiKey" -> apiKey, "apiSecret" -> apiSecret, "accessToken" -> accessToken, "accessTokenSecret" -> accessTokenSecret)
> println("Configuring Twitter OAuth")
> configs.foreach{ case(key, value) =>
> if (value.trim.isEmpty) {
> throw new Exception("Error setting authentication - value for " + key + " not set")
> }
> val fullKey = "twitter4j.oauth." + key.replace("api", "consumer")
> System.setProperty(fullKey, value.trim)
> println("\tProperty " + fullKey + " set as [" + value.trim + "]")
> }
> println()}
> // Configure Twitter credentialsval apiKey = "xxxxxxxxxxxxxxxxxxxxxxxxx"val apiSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessToken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"val accessTokenSecret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"configureTwitterCredentials(apiKey, apiSecret, accessToken, accessTokenSecret)
> import org.apache.spark.streaming.twitter._val ssc = new StreamingContext(sc, Seconds(2))val tweets = TwitterUtils.createStream(ssc, None)val twt = tweets.window(Seconds(60))
> case class Tweet(createdAt:Long, text:String)twt.map(status=>
> Tweet(status.getCreatedAt().getTime()/1000, status.getText())).foreachRDD(rdd=>
> // Below line works only in spark 1.3.0.
> // For spark 1.1.x and spark 1.2.x,
> // use rdd.registerTempTable("tweets") instead.
> rdd.toDF().registerAsTable("tweets"))
> twt.print
> ssc.start()
>
> But getting the below error
>
> import org.apache.spark.streaming.twitter._
> <console>:19: error: object twitter is not a member of package org.apache.spark.streaming
> import org.apache.spark.streaming.twitter._
>
> I using spark 1.3.0
>
>
> The same worked for spark 0.9.0 . Any help?
>