You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sm...@apache.org on 2017/02/08 04:32:47 UTC
[1/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Repository: incubator-streams
Updated Branches:
refs/heads/master 571d4066e -> b71cce83d
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java
new file mode 100644
index 0000000..9a7e547
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.google.api.services.youtube.YouTube;
+import org.joda.time.DateTime;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test for YoutubeProvider.
+ */
+public class YoutubeProviderTest {
+
+ /**
+ * Test that every collector will be run and that data queued from the collectors will be processed.
+ */
+ @Test
+ public void testDataCollectorRunsPerUser() {
+ Random random = new Random(System.currentTimeMillis());
+ int numUsers = random.nextInt(1000);
+ List<UserInfo> userList = new LinkedList<>();
+
+ for ( int i = 0; i < numUsers; ++i ) {
+ userList.add(new UserInfo());
+ }
+
+ YoutubeConfiguration config = new YoutubeConfiguration();
+ config.setYoutubeUsers(userList);
+ config.setApiKey("API_KEY");
+ YoutubeProvider provider = buildProvider(config);
+
+ try {
+ provider.prepare(null);
+ provider.startStream();
+ int datumCount = 0;
+ while (provider.isRunning()) {
+ datumCount += provider.readCurrent().size();
+ }
+ assertEquals(numUsers, datumCount);
+ } finally {
+ provider.cleanUp();
+ }
+ }
+
+ @Test
+ public void testConfigSetterGetter() {
+ YoutubeConfiguration config = new YoutubeConfiguration();
+ config.setApiKey("API_KEY");
+ config.setVersion("fake_version_1");
+ YoutubeConfiguration newConfig = new YoutubeConfiguration();
+ newConfig.setApiKey("API_KEY");
+ config.setVersion("fake_version_2");
+
+ YoutubeProvider provider = buildProvider(config);
+
+ assertEquals(provider.getConfig(), config);
+
+ provider.setConfig(newConfig);
+ assertEquals(provider.getConfig(), newConfig);
+ }
+
+ @Test
+ public void testUserInfoWithDefaultDates() {
+ YoutubeConfiguration config = new YoutubeConfiguration();
+ config.setApiKey("API_KEY");
+ YoutubeProvider provider = buildProvider(config);
+
+ DateTime afterDate = new DateTime(System.currentTimeMillis());
+ DateTime beforeDate = afterDate.minus(10000);
+
+ provider.setDefaultAfterDate(afterDate);
+ provider.setDefaultBeforeDate(beforeDate);
+
+ Set<String> users = new HashSet<>();
+ users.add("test_user_1");
+ users.add("test_user_2");
+ users.add("test_user_3");
+
+ provider.setUserInfoWithDefaultDates(users);
+
+ List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
+
+ for (UserInfo user : youtubeUsers) {
+ assert (user.getAfterDate().equals(afterDate));
+ assert (user.getBeforeDate().equals(beforeDate));
+ }
+ }
+
+ @Test
+ public void testUserInfoWithAfterDate() {
+ YoutubeConfiguration config = new YoutubeConfiguration();
+ config.setApiKey("API_KEY");
+ YoutubeProvider provider = buildProvider(config);
+
+ Map<String, DateTime> users = new HashMap<>();
+ users.put("user1", new DateTime(System.currentTimeMillis()));
+ users.put("user3", new DateTime(System.currentTimeMillis()));
+ users.put("user4", new DateTime(System.currentTimeMillis()));
+
+ provider.setUserInfoWithAfterDate(users);
+
+ List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
+
+ for (UserInfo user : youtubeUsers) {
+ assert (user.getAfterDate().equals(users.get(user.getUserId())));
+ }
+ }
+
+ private YoutubeProvider buildProvider(YoutubeConfiguration config) {
+ return new YoutubeProvider(config) {
+
+ @Override
+ protected YouTube createYouTubeClient() throws IOException {
+ return mock(YouTube.class);
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
+ final BlockingQueue<StreamsDatum> q = queue;
+ return () -> {
+ try {
+ q.put(new StreamsDatum(null));
+ } catch (InterruptedException ie) {
+ fail("Test was interrupted");
+ }
+ };
+ }
+ };
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java
new file mode 100644
index 0000000..1d92a3e
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.local.queues.ThroughputQueue;
+import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Activity;
+import com.google.api.services.youtube.model.ActivityContentDetails;
+import com.google.api.services.youtube.model.ActivityContentDetailsUpload;
+import com.google.api.services.youtube.model.ActivityListResponse;
+import com.google.api.services.youtube.model.Video;
+import com.google.api.services.youtube.model.VideoListResponse;
+import com.google.api.services.youtube.model.VideoSnippet;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test for YoutubeUserActivityCollector.
+ */
+public class YoutubeUserActivityCollectorTest {
+ private static final String USER_ID = "fake_user_id";
+ private static final String IN_RANGE_IDENTIFIER = "data in range";
+ private YoutubeConfiguration config;
+
+ private static VideoListResponse createMockVideoListResponse(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before, boolean page) {
+ VideoListResponse feed = new VideoListResponse();
+ List<Video> list = new LinkedList<>();
+
+ for (int i = 0; i < numAfter; ++i) {
+ com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime(after.getMillis() + 1000000);
+ Video video = new Video();
+ video.setSnippet(new VideoSnippet());
+ video.getSnippet().setPublishedAt(published);
+ list.add(video);
+ }
+ for (int i = 0; i < numInRange; ++i) {
+ DateTime published;
+ if ((before == null && after == null) || before == null) {
+ published = DateTime.now(); // no date range or end time date range so just make the time now.
+ } else if (after == null) {
+ published = before.minusMillis(100000); //no beginning to range
+ } else { // has to be in range
+ long range = before.getMillis() - after.getMillis();
+ published = after.plus(range / 2); //in the middle
+ }
+ com.google.api.client.util.DateTime ytPublished = new com.google.api.client.util.DateTime(published.getMillis());
+ Video video = new Video();
+ video.setSnippet(new VideoSnippet());
+ video.getSnippet().setPublishedAt(ytPublished);
+ video.getSnippet().setTitle(IN_RANGE_IDENTIFIER);
+ list.add(video);
+ }
+ for (int i = 0; i < numBefore; ++i) {
+ com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime((after.minusMillis(100000)).getMillis());
+ Video video = new Video();
+ video.setSnippet(new VideoSnippet());
+ video.getSnippet().setPublishedAt(published);
+ list.add(video);
+ }
+ if (page) {
+ feed.setNextPageToken("A");
+ } else {
+ feed.setNextPageToken(null);
+ }
+
+ feed.setItems(list);
+
+ return feed;
+ }
+
+ @Before
+ public void setup() {
+ this.config = new YoutubeConfiguration();
+ this.config.setApiKey("API_KEY");
+ }
+
+ @Test
+ public void testGetVideos() throws IOException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(0, 1, 0, now, now.minus(10000));
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ List<Video> video = collector.getVideoList("fake_video_id");
+
+ assertNotNull(video.get(0));
+ }
+
+ @Test
+ public void testGetVideosNull() throws IOException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(0, 0, 0, now.plus(10000), now.minus(10000));
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ List<Video> video = collector.getVideoList("fake_video_id");
+
+ assertEquals(video.size(), 0);
+ }
+
+ @Test
+ public void testProcessActivityFeed() throws IOException, InterruptedException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(0, 0, 5, now.plus(3000000), now.minus(1000000));
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ ActivityListResponse feed = buildActivityListResponse(1);
+
+ collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
+
+ assertEquals(collector.getDatumQueue().size(), 5);
+ }
+
+ @Test
+ public void testProcessActivityFeedBefore() throws IOException, InterruptedException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(5, 0, 0, now, now);
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ ActivityListResponse feed = buildActivityListResponse(1);
+
+ collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
+
+ assertEquals(collector.getDatumQueue().size(), 0);
+ }
+
+ @Test
+ public void testProcessActivityFeedAfter() throws IOException, InterruptedException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(0, 5, 0, now, now);
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ ActivityListResponse feed = buildActivityListResponse(1);
+
+ collector.processActivityFeed(feed, new DateTime(now.getMillis() - 100000), null);
+
+ assertEquals(collector.getDatumQueue().size(), 5);
+ }
+
+ @Test
+ public void testProcessActivityFeedMismatchCount() throws IOException, InterruptedException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ ActivityListResponse feed = buildActivityListResponse(1);
+
+ collector.processActivityFeed(feed, new DateTime(now), null);
+
+ assertEquals(collector.getDatumQueue().size(), 5);
+ }
+
+ @Test
+ public void testProcessActivityFeedMismatchCountInRange() throws IOException, InterruptedException {
+ DateTime now = new DateTime(System.currentTimeMillis());
+ YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
+
+ BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
+ YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
+
+ ActivityListResponse feed = buildActivityListResponse(1);
+
+ collector.processActivityFeed(feed, new DateTime(now), new DateTime(now).minus(100000));
+
+ assertEquals(collector.getDatumQueue().size(), 5);
+ }
+
+ private ActivityListResponse buildActivityListResponse(int num) {
+ ActivityListResponse activityListResponse = new ActivityListResponse();
+ List<Activity> items = new ArrayList<>();
+
+ for ( int x = 0; x < num; x++ ) {
+ Activity activity = new Activity();
+
+ ActivityContentDetails contentDetails = new ActivityContentDetails();
+ ActivityContentDetailsUpload upload = new ActivityContentDetailsUpload();
+ upload.setVideoId("video_id_" + x);
+ contentDetails.setUpload(upload);
+
+ activity.setId("id_" + x);
+ activity.setContentDetails(contentDetails);
+
+ items.add(activity);
+ }
+
+ activityListResponse.setItems(items);
+
+ return activityListResponse;
+ }
+
+ private YouTube buildYouTube(int numBeforeRange, int numAfterRange, int numInRange, DateTime afterDate, DateTime beforeDate) {
+
+ return createYoutubeMock(numBeforeRange, numAfterRange, numInRange, afterDate, beforeDate);
+
+ }
+
+ private YouTube createYoutubeMock(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ YouTube youtube = mock(YouTube.class);
+
+ final YouTube.Videos videos = createMockVideos(numBefore, numAfter, numInRange, after, before);
+ doAnswer(invocationOnMock -> videos).when(youtube).videos();
+
+ return youtube;
+ }
+
+ private YouTube.Videos createMockVideos(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ YouTube.Videos videos = mock(YouTube.Videos.class);
+
+ try {
+ YouTube.Videos.List list = createMockVideosList(numBefore, numAfter, numInRange, after, before);
+ when(videos.list(anyString())).thenReturn(list);
+ } catch (IOException ex) {
+ fail("Exception thrown while creating mock");
+ }
+
+ return videos;
+ }
+
+ private YouTube.Videos.List createMockVideosList(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ YouTube.Videos.List list = mock(YouTube.Videos.List.class);
+
+ when(list.setMaxResults(anyLong())).thenReturn(list);
+ when(list.setPageToken(anyString())).thenReturn(list);
+ when(list.setId(anyString())).thenReturn(list);
+ when(list.setKey(anyString())).thenReturn(list);
+
+ VideoListResponseAnswer answer = new VideoListResponseAnswer(numBefore, numAfter, numInRange, after, before);
+ try {
+ doAnswer(answer).when(list).execute();
+ } catch (IOException ioe) {
+ fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
+ }
+
+ return list;
+ }
+
+ private static class VideoListResponseAnswer implements Answer<VideoListResponse> {
+ private int afterCount = 0;
+ private int beforeCount = 0;
+ private int inCount = 0;
+ private int maxBatch = 100;
+
+ private int numAfter;
+ private int numInRange;
+ private int numBefore;
+ private DateTime after;
+ private DateTime before;
+
+ private VideoListResponseAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ this.numBefore = numBefore;
+ this.numAfter = numAfter;
+ this.numInRange = numInRange;
+ this.after = after;
+ this.before = before;
+ }
+
+ @Override
+ public VideoListResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
+ int totalCount = 0;
+ int batchAfter = 0;
+ int batchBefore = 0;
+ int batchIn = 0;
+ inCount = 0;
+ afterCount = 0;
+ beforeCount = 0;
+
+ if (afterCount != numAfter) {
+ if (numAfter - afterCount >= maxBatch) {
+ afterCount += maxBatch;
+ batchAfter += maxBatch;
+ totalCount += batchAfter;
+ } else {
+ batchAfter += numAfter - afterCount;
+ totalCount += numAfter - afterCount;
+ afterCount = numAfter;
+ }
+ }
+ if (totalCount < maxBatch && inCount != numInRange) {
+ if (numInRange - inCount >= maxBatch - totalCount) {
+ inCount += maxBatch - totalCount;
+ batchIn += maxBatch - totalCount;
+ totalCount += batchIn;
+ } else {
+ batchIn += numInRange - inCount;
+ totalCount += numInRange - inCount;
+ inCount = numInRange;
+ }
+ }
+ if (totalCount < maxBatch && beforeCount != numBefore) {
+ if (numBefore - batchBefore >= maxBatch - totalCount) {
+ batchBefore += maxBatch - totalCount;
+ totalCount = maxBatch;
+ beforeCount += batchBefore;
+ } else {
+ batchBefore += numBefore - beforeCount;
+ totalCount += numBefore - beforeCount;
+ beforeCount = numBefore;
+ }
+ }
+
+ return createMockVideoListResponse(batchBefore, batchAfter, batchIn, after, before, numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java
new file mode 100644
index 0000000..d473abe
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.api.services.youtube.model.Video;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class YoutubeEventClassifierTest {
+
+ private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
+ private final String testObjectNode = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#somethingElse\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
+
+ @Test
+ public void testVideoClassification() {
+ Class klass = YoutubeEventClassifier.detectClass(testVideo);
+
+ assertEquals(klass, Video.class);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testExceptionClassification() {
+ YoutubeEventClassifier.detectClass("");
+ }
+
+ @Test
+ public void testObjectNodeClassification() {
+ Class klass = YoutubeEventClassifier.detectClass(testObjectNode);
+
+ assertEquals(klass, ObjectNode.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java
new file mode 100644
index 0000000..e4affad
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.extensions.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Provider;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.youtube.model.Video;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for YoutubeVideoSerDe.
+ */
+public class YoutubeVideoSerDeTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoSerDeTest.class);
+ private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
+ private ObjectMapper objectMapper;
+
+ /**
+ * setup for test.
+ */
+ @Before
+ public void setup() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ @Test
+ public void testVideoObject() {
+ LOGGER.info("raw: {}", testVideo);
+
+ try {
+ Activity activity = new Activity();
+
+ Video video = objectMapper.readValue(testVideo, Video.class);
+
+ YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
+ LOGGER.info("activity: {}", activity);
+
+ assertNotNull(activity);
+ assert (activity.getId().contains("id:youtube:post"));
+ assertEquals(activity.getVerb(), "post");
+
+ Provider provider = activity.getProvider();
+ assertEquals(provider.getId(), "id:providers:youtube");
+ assertEquals(provider.getDisplayName(), "YouTube");
+
+ ActivityObject actor = activity.getActor();
+ assert (actor.getId().contains("id:youtube:"));
+ assertNotNull(actor.getDisplayName());
+ assertNotNull(actor.getSummary());
+
+ assertNotNull(activity.getTitle());
+ assertNotNull(activity.getUrl());
+ assertNotNull(activity.getContent());
+
+ assertEquals(activity.getPublished().getClass(), DateTime.class);
+
+ Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
+
+ assertNotNull(extensions.get("youtube"));
+ assertNotNull(extensions.get("likes"));
+
+ assertTrue(testActivityObject(activity));
+ } catch (Exception ex) {
+ LOGGER.error("Exception while testing the Ser/De functionality of the Video deserializer: {}", ex);
+ }
+ }
+
+ private boolean testActivityObject(Activity activity) {
+ boolean valid = false;
+
+ ActivityObject obj = activity.getObject();
+
+ if ( obj.getObjectType().equals("video")
+ && !(obj.getImage() == null)
+ && !obj.getUrl().equals("null")
+ && obj.getUrl().contains("https://www.youtube.com/watch?v=")) {
+ valid = true;
+ }
+
+ return valid;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
index f551d93..fac0b2e 100644
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
@@ -18,7 +18,8 @@
package org.apache.streams.youtube.test.providers;
-import com.youtube.provider.YoutubeChannelProvider;
+import org.apache.streams.youtube.provider.YoutubeChannelProvider;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
index 34442a2..df1eaef 100644
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
@@ -18,7 +18,7 @@
package org.apache.streams.youtube.test.providers;
-import com.youtube.provider.YoutubeUserActivityProvider;
+import org.apache.streams.youtube.provider.YoutubeUserActivityProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-cassandra/pom.xml b/streams-plugins/streams-plugin-cassandra/pom.xml
index 9316bc3..8009bfd 100644
--- a/streams-plugins/streams-plugin-cassandra/pom.xml
+++ b/streams-plugins/streams-plugin-cassandra/pom.xml
@@ -157,11 +157,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml
index 30d7973..7487745 100644
--- a/streams-plugins/streams-plugin-elasticsearch/pom.xml
+++ b/streams-plugins/streams-plugin-elasticsearch/pom.xml
@@ -158,11 +158,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml
index 89c5f6c..d521c88 100644
--- a/streams-plugins/streams-plugin-hbase/pom.xml
+++ b/streams-plugins/streams-plugin-hbase/pom.xml
@@ -157,11 +157,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-hive/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml
index 133f30f..7b9da30 100644
--- a/streams-plugins/streams-plugin-hive/pom.xml
+++ b/streams-plugins/streams-plugin-hive/pom.xml
@@ -157,11 +157,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-pig/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-pig/pom.xml b/streams-plugins/streams-plugin-pig/pom.xml
index 57317e9..f85d00e 100644
--- a/streams-plugins/streams-plugin-pig/pom.xml
+++ b/streams-plugins/streams-plugin-pig/pom.xml
@@ -157,11 +157,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-plugins/streams-plugin-scala/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-scala/pom.xml b/streams-plugins/streams-plugin-scala/pom.xml
index b8ef652..70bd622 100644
--- a/streams-plugins/streams-plugin-scala/pom.xml
+++ b/streams-plugins/streams-plugin-scala/pom.xml
@@ -20,7 +20,6 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.streams.plugins</groupId>
<artifactId>streams-plugin-scala</artifactId>
<version>0.5-incubating-SNAPSHOT</version>
<packaging>maven-plugin</packaging>
@@ -154,11 +153,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-testing/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/streams-testing/src/test/resources/log4j.properties b/streams-testing/src/test/resources/log4j.properties
index c6c20ae..c183d78 100644
--- a/streams-testing/src/test/resources/log4j.properties
+++ b/streams-testing/src/test/resources/log4j.properties
@@ -18,7 +18,7 @@ log4j.logger.ch.qos.logback=ERROR
log4j.logger.com.foursquare=WARN
log4j.logger.com.google=WARN
log4j.logger.com.mongodb=WARN
-log4j.logger.com.youtube=WARN
+log4j.logger.org.apache.streams.youtube=WARN
log4j.logger.io.dropwizard=WARN
log4j.logger.org.apache.streams=WARN
log4j.logger.org.apache.spark=WARN
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-testing/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/streams-testing/src/test/resources/logback-test.xml b/streams-testing/src/test/resources/logback-test.xml
index 2dfa4b2..9dd8bf3 100644
--- a/streams-testing/src/test/resources/logback-test.xml
+++ b/streams-testing/src/test/resources/logback-test.xml
@@ -21,7 +21,7 @@
<logger name="com.mongodb" level="WARN"/>
<logger name="com.foursquare" level="WARN"/>
<logger name="com.google" level="WARN"/>
- <logger name="com.youtube" level="WARN"/>
+ <logger name="org.apache.streams.youtube" level="WARN"/>
<logger name="io.dropwizard" level="WARN"/>
<logger name="org.apache.streams" level="WARN"/>
<logger name="org.apache.spark" level="WARN"/>
[6/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeIT.java
deleted file mode 100644
index 6bb2f57..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusPersonSerDeIT.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import com.google.gplus.serializer.util.GPlusPersonDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusPersonSerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusPersonSerDeIT.class);
- private ObjectMapper objectMapper;
- private GooglePlusActivityUtil googlePlusActivityUtil;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusActivityUtil = new GooglePlusActivityUtil();
- }
-
- @Test
- public void testPersonObjects() {
- InputStream is = GooglePlusPersonSerDeIT.class.getResourceAsStream("/google_plus_person_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- Person person = objectMapper.readValue(line, Person.class);
-
- GooglePlusActivityUtil.updateActivity(person, activity);
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assertTrue (activity.getId().contains("id:googleplus:update"));
- assertEquals(activity.getVerb(), "update");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:googleplus");
- assertEquals(provider.getDisplayName(), "GooglePlus");
-
- ActivityObject actor = activity.getActor();
- assertNotNull(actor.getImage());
- assertTrue (actor.getId().contains("id:googleplus:"));
- assertNotNull(actor.getUrl());
-
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeIT.java
deleted file mode 100644
index b75021c..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/processor/GooglePlusActivitySerDeIT.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.processor;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.gplus.serializer.util.GPlusActivityDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Map;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusActivitySerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivitySerDeIT.class);
- private ObjectMapper objectMapper;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testActivityObjects() {
- InputStream is = GooglePlusActivitySerDeIT.class.getResourceAsStream("/google_plus_activity_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- com.google.api.services.plus.model.Activity googlePlusActivity =
- objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
-
- GooglePlusActivityUtil.updateActivity(googlePlusActivity, activity);
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assert (activity.getId().contains("id:googleplus:post"));
- assertEquals(activity.getVerb(), "post");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:googleplus");
- assertEquals(provider.getDisplayName(), "GooglePlus");
-
- ActivityObject actor = activity.getActor();
- assertNotNull(actor.getImage());
- assert (actor.getId().contains("id:googleplus:"));
- assertNotNull(actor.getUrl());
-
- assertNotNull(activity.getPublished());
- assertNotNull(activity.getTitle());
- assertNotNull(activity.getUrl());
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().getExtensions(activity);
- assertNotNull(extensions);
-
- if (activity.getContent() != null) {
- assertNotNull(extensions.get("rebroadcasts"));
- assertNotNull(extensions.get("keywords"));
- assertNotNull(extensions.get("likes"));
- assert (((Map<String, Object>) extensions.get("rebroadcasts")).containsKey("count"));
- assert (((Map<String, Object>) extensions.get("likes")).containsKey("count"));
- }
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
deleted file mode 100644
index b3590c4..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestAbstractGPlusProvider.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.GPlusOAuthConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.google.api.services.plus.Plus;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.mockito.Mockito.mock;
-
-/**
- * Unit tests for {@link com.google.gplus.provider.AbstractGPlusProvider}
- */
-public class TestAbstractGPlusProvider extends RandomizedTest {
-
- /**
- * Test that every collector will be run and that data queued from the collectors will be processed.
- */
- @Test
- @Repeat(iterations = 3)
- public void testDataCollectorRunsPerUser() {
- int numUsers = randomIntBetween(1, 1000);
- List<UserInfo> userList = new LinkedList<>();
- for (int i = 0; i < numUsers; ++i) {
- userList.add(new UserInfo());
- }
- GPlusConfiguration config = new GPlusConfiguration();
- GPlusOAuthConfiguration oauth = new GPlusOAuthConfiguration();
- oauth.setAppName("a");
- oauth.setPathToP12KeyFile("a");
- oauth.setServiceAccountEmailAddress("a");
- config.setOauth(oauth);
- config.setGooglePlusUsers(userList);
- AbstractGPlusProvider provider = new AbstractGPlusProvider(config) {
-
- @Override
- protected Plus createPlusClient() throws IOException {
- return mock(Plus.class);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- final BlockingQueue<StreamsDatum> q = queue;
- return () -> {
- try {
- q.put(new StreamsDatum(null));
- } catch (InterruptedException ie) {
- fail("Test was interrupted");
- }
- };
- }
- };
-
- try {
- provider.prepare(null);
- provider.startStream();
- int datumCount = 0;
- while (provider.isRunning()) {
- datumCount += provider.readCurrent().size();
- }
- assertEquals(numUsers, datumCount);
- } finally {
- provider.cleanUp();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
deleted file mode 100644
index ffff3e0..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserActivityCollector.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.ActivityFeed;
-import com.google.gplus.serializer.util.GPlusActivityDeserializer;
-import org.joda.time.DateTime;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Unit tests for {@link com.google.gplus.provider.GPlusUserActivityCollector}
- */
-public class TestGPlusUserActivityCollector extends RandomizedTest {
-
- private static final String ACTIVITY_TEMPLATE = "{ \"kind\": \"plus#activity\", \"etag\": \"\\\"Vea_b94Y77GDGgRK7gFNPnolKQw/v1-6aVSBGT4qiStMoz7f2_AN2fM\\\"\", \"title\": \"\", \"published\": \"%s\", \"updated\": \"2014-10-27T06:26:33.927Z\", \"id\": \"z13twrlznpvtzz52w22mdt1y0k3of1djw04\", \"url\": \"https://plus.google.com/116771159471120611293/posts/GR7CGR8N5VL\", \"actor\": { \"id\": \"116771159471120611293\", \"displayName\": \"displayName\", \"url\": \"https://plus.google.com/116771159471120611293\", \"image\": { \"url\": \"https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50\" } }, \"verb\": \"share\", \"object\": { \"objectType\": \"activity\", \"id\": \"z13zgvtiurjgfti1v234iflghvq2c1dge04\", \"actor\": { \"id\": \"104954254300557350002\", \"displayName\": \"displayName\", \"url\": \"https://plus.google.com/104954254300557350002\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-SO1scj4p2LA/AAAAAAAAAAI/AAAAAAAAI-s/efA
9LBVe144/photo.jpg?sz=50\" } }, \"content\": \"\", \"url\": \"https://plus.google.com/104954254300557350002/posts/AwewXhtn7ws\", \"replies\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/comments\" }, \"plusoners\": { \"totalItems\": 9, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/plusoners\" }, \"resharers\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/resharers\" }, \"attachments\": [ { \"objectType\": \"photo\", \"id\": \"104954254300557350002.6074732746360957410\", \"content\": \"26/10/2014 - 1\", \"url\": \"https://plus.google.com/photos/104954254300557350002/albums/6074732747132702225/6074732746360957410\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w506-h750/2014%2B-%2B1\", \"type\": \"ima
ge/jpeg\" }, \"fullImage\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w600-h1141/2014%2B-%2B1\", \"type\": \"image/jpeg\", \"height\": 1141, \"width\": 600 } } ] }, \"annotation\": \"Truth \U0001f61c\", \"provider\": { \"title\": \"Reshared Post\" }, \"access\": { \"kind\": \"plus#acl\", \"description\": \"Public\", \"items\": [ { \"type\": \"public\" } ] } }";
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final String IN_RANGE_IDENTIFIER = "data in range";
-
- static {
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- /**
- * Creates a randomized activity and randomized date range.
- *
- * <p/>
- * The activity feed is separated into three chunks,
- * |. . . data too recent to be in date range . . .||. . . data in date range. . .||. . . data too old to be in date range|
- * [index 0, ............................................................................................., index length-1]
- *
- * <p/>
- * Inside of those chunks data has no order, but the list is ordered by those three chunks.
- *
- * <p/>
- * The test will check to see if the num of data in the date range make onto the output queue.
- */
- @Test
- @Repeat(iterations = 3)
- public void testWithBeforeAndAfterDates() throws InterruptedException {
- //initialize counts assuming no date ranges will be used
- int numActivities = randomIntBetween(0, 1000);
- int numActivitiesInDateRange = numActivities;
- int numberOutOfRange = 0;
- int numBeforeRange = 0;
- int numAfterRange = 0;
- //determine if date ranges will be used
- DateTime beforeDate = null;
- DateTime afterDate = null;
- if (randomInt() % 2 == 0) {
- beforeDate = DateTime.now().minusDays(randomIntBetween(1,5));
- }
- if (randomInt() % 2 == 0) {
- if (beforeDate == null) {
- afterDate = DateTime.now().minusDays(randomIntBetween(1, 10));
- } else {
- afterDate = beforeDate.minusDays(randomIntBetween(1, 10));
- }
- }
- //update counts if date ranges are going to be used.
- if (beforeDate != null || afterDate != null ) { //assign amount to be in range
- numActivitiesInDateRange = randomIntBetween(0, numActivities);
- numberOutOfRange = numActivities - numActivitiesInDateRange;
- }
- if (beforeDate == null && afterDate != null) { //assign all out of range to be before the start of the range
- numBeforeRange = numberOutOfRange;
- } else if (beforeDate != null && afterDate == null) { //assign all out of range to be after the start of the range
- numAfterRange = numberOutOfRange;
- } else if (beforeDate != null && afterDate != null) { //assign half before range and half after the range
- numAfterRange = (numberOutOfRange / 2) + (numberOutOfRange % 2);
- numBeforeRange = numberOutOfRange / 2;
- }
-
- Plus plus = createMockPlus(numBeforeRange, numAfterRange, numActivitiesInDateRange, afterDate, beforeDate);
- BackOffStrategy strategy = new ConstantTimeBackOffStrategy(1);
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- UserInfo userInfo = new UserInfo();
- userInfo.setUserId("A");
- userInfo.setAfterDate(afterDate);
- userInfo.setBeforeDate(beforeDate);
- GPlusUserActivityCollector collector = new GPlusUserActivityCollector(plus, datums, strategy, userInfo);
- collector.run();
-
- assertEquals(numActivitiesInDateRange, datums.size());
- while (!datums.isEmpty()) {
- StreamsDatum datum = datums.take();
- assertNotNull(datum);
- assertNotNull(datum.getDocument());
- assertTrue(datum.getDocument() instanceof String);
- assertTrue(((String)datum.getDocument()).contains(IN_RANGE_IDENTIFIER)); //only in range documents are on the out going queue.
- }
- }
-
-
- private Plus createMockPlus(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
- Plus plus = mock(Plus.class);
- final Plus.Activities activities = createMockPlusActivities(numBefore, numAfter, numInRange, after, before);
- doAnswer(invocationOnMock -> activities).when(plus).activities();
- return plus;
- }
-
- private Plus.Activities createMockPlusActivities(
- final int numBefore,
- final int numAfter,
- final int numInRange,
- final DateTime after,
- final DateTime before) {
- Plus.Activities activities = mock(Plus.Activities.class);
- try {
- Plus.Activities.List list = createMockPlusActivitiesList(numBefore, numAfter, numInRange, after, before);
- when(activities.list(anyString(), anyString())).thenReturn(list);
- } catch (IOException ioe) {
- fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
- return activities;
- }
-
- private Plus.Activities.List createMockPlusActivitiesList(
- final int numBefore,
- final int numAfter,
- final int numInRange,
- final DateTime after,
- final DateTime before) {
- Plus.Activities.List list = mock(Plus.Activities.List.class);
- when(list.setMaxResults(anyLong())).thenReturn(list);
- when(list.setPageToken(anyString())).thenReturn(list);
- ActivityFeedAnswer answer = new ActivityFeedAnswer(numBefore, numAfter, numInRange, after, before);
- try {
- doAnswer(answer).when(list).execute();
- } catch (IOException ioe) {
- fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
- return list;
- }
-
-
- private static ActivityFeed createMockActivityFeed(
- int numBefore,
- int numAfter,
- int numInRange,
- DateTime after,
- DateTime before,
- boolean page) {
- ActivityFeed feed = new ActivityFeed();
- List<Activity> list = new LinkedList<>();
- for (int i = 0; i < numAfter; ++i) {
- DateTime published = before.plus(randomIntBetween(0, Integer.MAX_VALUE));
- Activity activity = createActivityWithPublishedDate(published);
- list.add(activity);
- }
- for (int i = 0; i < numInRange; ++i) {
- DateTime published;
- if ((before == null && after == null) || before == null) {
- published = DateTime.now(); // no date range or end time date range so just make the time now.
- } else if (after == null) {
- published = before.minusMillis(randomIntBetween(1, Integer.MAX_VALUE)); //no beginning to range
- } else { // has to be in range
- long range = before.getMillis() - after.getMillis();
- published = after.plus(range / 2); //in the middle
- }
- Activity activity = createActivityWithPublishedDate(published);
- activity.setTitle(IN_RANGE_IDENTIFIER);
- list.add(activity);
- }
- for (int i = 0; i < numBefore; ++i) {
- DateTime published = after.minusMillis(randomIntBetween(1, Integer.MAX_VALUE));
- Activity activity = createActivityWithPublishedDate(published);
- list.add(activity);
- }
- if (page) {
- feed.setNextPageToken("A");
- } else {
- feed.setNextPageToken(null);
- }
- feed.setItems(list);
- return feed;
- }
-
- private static Activity createActivityWithPublishedDate(DateTime dateTime) {
- Activity activity = new Activity();
- activity.setPublished(new com.google.api.client.util.DateTime(dateTime.getMillis()));
- activity.setId("a");
- return activity;
- }
-
- private static class ActivityFeedAnswer implements Answer<ActivityFeed> {
- private int afterCount = 0;
- private int beforeCount = 0;
- private int inCount = 0;
- private int maxBatch = 100;
-
- private int numAfter;
- private int numInRange;
- private int numBefore;
- private DateTime after;
- private DateTime before;
-
- private ActivityFeedAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- this.numBefore = numBefore;
- this.numAfter = numAfter;
- this.numInRange = numInRange;
- this.after = after;
- this.before = before;
- }
-
-
-
-
- @Override
- public ActivityFeed answer(InvocationOnMock invocationOnMock) throws Throwable {
- int totalCount = 0;
- int batchAfter = 0;
- int batchBefore = 0;
- int batchIn = 0;
- if (afterCount != numAfter) {
- if (numAfter - afterCount >= maxBatch) {
- afterCount += maxBatch;
- batchAfter += maxBatch;
- totalCount += batchAfter;
- } else {
- batchAfter += numAfter - afterCount;
- totalCount += numAfter - afterCount;
- afterCount = numAfter;
- }
- }
- if (totalCount < maxBatch && inCount != numInRange) {
- if (numInRange - inCount >= maxBatch - totalCount) {
- inCount += maxBatch - totalCount;
- batchIn += maxBatch - totalCount;
- totalCount += batchIn;
- } else {
- batchIn += numInRange - inCount;
- totalCount += numInRange - inCount;
- inCount = numInRange;
- }
- }
- if (totalCount < maxBatch && beforeCount != numBefore) {
- if (numBefore - batchBefore >= maxBatch - totalCount) {
- batchBefore += maxBatch - totalCount;
- totalCount = maxBatch;
- beforeCount += batchBefore;
- } else {
- batchBefore += numBefore - beforeCount;
- totalCount += numBefore - beforeCount;
- beforeCount = numBefore;
- }
- }
-
- return createMockActivityFeed(
- batchBefore,
- batchAfter,
- batchIn,
- after,
- before,
- numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
- }
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
deleted file mode 100644
index 8d30314..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/provider/TestGPlusUserDataCollector.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Person;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Basic Units for {@link com.google.gplus.provider.GPlusUserDataCollector}.
- */
-public class TestGPlusUserDataCollector {
-
- private static final String NO_ERROR = "no error";
-
- /**
- * Test that on success a datum will be added to the queue.
- * @throws Exception Exception
- */
- @Test
- public void testSucessfullPull() throws Exception {
- Plus plus = createMockPlus(0, null);
- BackOffStrategy backOff = new ConstantTimeBackOffStrategy(1);
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- UserInfo user = new UserInfo();
- user.setUserId("A");
-
- GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOff, datums, user);
- collector.run();
-
- assertEquals(1, datums.size());
- StreamsDatum datum = datums.take();
- assertNotNull(datum);
- assertEquals(NO_ERROR, datum.getId());
- assertNotNull(datum.getDocument());
- assertTrue(datum.getDocument() instanceof String);
- }
-
- /**
- * Test that on failure, no datums are output.
- * @throws Exception Exception
- */
- @Test
- public void testFail() throws Exception {
- Plus plus = createMockPlus(3, mock(GoogleJsonResponseException.class));
- UserInfo user = new UserInfo();
- user.setUserId("A");
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- BackOffStrategy backOffStrategy = new ConstantTimeBackOffStrategy(1);
-
- GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOffStrategy, datums, user);
- collector.run();
-
- assertEquals(0, datums.size());
- }
-
- private Plus createMockPlus(final int succedOnTry, final Throwable throwable) {
- Plus plus = mock(Plus.class);
- doAnswer(invocationOnMock -> createMockPeople(succedOnTry, throwable)).when(plus).people();
- return plus;
- }
-
- private Plus.People createMockPeople(final int succedOnTry, final Throwable throwable) {
- Plus.People people = mock(Plus.People.class);
- try {
- when(people.get(anyString())).thenAnswer(invocationOnMock -> createMockGetNoError(succedOnTry, throwable));
- } catch (IOException ioe) {
- fail("No Excpetion should have been thrown while creating mocks");
- }
- return people;
- }
-
- private Plus.People.Get createMockGetNoError(final int succedOnTry, final Throwable throwable) {
- Plus.People.Get get = mock(Plus.People.Get.class);
- try {
- doAnswer(new Answer() {
- private int counter = 0;
-
- @Override
- public Person answer(InvocationOnMock invocationOnMock) throws Throwable {
- if (counter == succedOnTry) {
- Person person = new Person();
- person.setId(NO_ERROR);
- return person;
- } else {
- ++counter;
- throw throwable;
- }
- }
- }).when(get).execute();
- } catch (IOException ioe) {
- fail("No Excpetion should have been thrown while creating mocks");
- }
- return get;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/serializer/util/GPlusEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/serializer/util/GPlusEventClassifierTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/serializer/util/GPlusEventClassifierTest.java
deleted file mode 100644
index 96a9d89..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/serializer/util/GPlusEventClassifierTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.Person;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * GPlusEventClassifierTest tests GPlusEventClassifier.
- */
-public class GPlusEventClassifierTest {
-
- private static StreamsJacksonMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void classifyActivityTest() {
- try {
- Activity activity = new Activity();
- activity.setKind("plus#activity");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(activity));
-
- assertEquals(retClass, Activity.class);
- } catch (Exception ex) {
- //
- }
- }
-
- @Test
- public void classifyPersonTest() {
- try {
- Person person = new Person();
- person.setKind("plus#person");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
-
- assertEquals(retClass, Person.class);
- } catch (Exception ex) {
- //
- }
- }
-
- @Test
- public void classifyObjectNodeTest() {
- try {
- Person person = new Person();
- person.setKind("fake");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
-
- assertEquals(retClass, ObjectNode.class);
- } catch (Exception ex) {
- //
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java
new file mode 100644
index 0000000..1fb79ca
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus;
+
+import org.apache.streams.gplus.serializer.util.GPlusCommentDeserializer;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.model.Comment;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+/**
+ * Tests conversion of gplus inputs to Activity.
+ */
+public class GooglePlusCommentSerDeIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeIT.class);
+ private ObjectMapper objectMapper;
+
+ /**
+ * setup.
+ */
+ @BeforeClass
+ public void setupTestCommentObjects() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ @Test
+ public void testCommentObjects() {
+ InputStream is = GooglePlusCommentSerDeIT.class.getResourceAsStream("/google_plus_comments_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ Activity activity = new Activity();
+ List<Comment> comments = new ArrayList<>();
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Comment comment = objectMapper.readValue(line, Comment.class);
+
+ LOGGER.info("comment: {}", comment);
+
+ assertNotNull(comment);
+ assertNotNull(comment.getEtag());
+ assertNotNull(comment.getId());
+ assertNotNull(comment.getInReplyTo());
+ assertNotNull(comment.getObject());
+ assertNotNull(comment.getPlusoners());
+ assertNotNull(comment.getPublished());
+ assertNotNull(comment.getUpdated());
+ assertNotNull(comment.getSelfLink());
+ assertEquals(comment.getVerb(), "post");
+
+ comments.add(comment);
+ }
+ }
+
+ assertEquals(comments.size(), 3);
+
+ GooglePlusActivityUtil.updateActivity(comments, activity);
+ assertNotNull(activity);
+ assertNotNull(activity.getObject());
+ assertEquals(activity.getObject().getAttachments().size(), 3);
+ } catch (Exception ex) {
+ LOGGER.error("Exception while testing serializability: {}", ex);
+ }
+ }
+
+ @Test
+ public void testEmptyComments() {
+ Activity activity = new Activity();
+ GooglePlusActivityUtil.updateActivity(new ArrayList<>(), activity);
+ assertNull(activity.getObject());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java
new file mode 100644
index 0000000..ce53bf3
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus;
+
+import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Provider;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.model.Person;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Tests conversion of gplus inputs to Activity.
+ */
+public class GooglePlusPersonSerDeIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusPersonSerDeIT.class);
+ private ObjectMapper objectMapper;
+
+ /**
+ * setup.
+ */
+ @BeforeClass
+ public void setup() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ @Test
+ public void testPersonObjects() {
+ InputStream is = GooglePlusPersonSerDeIT.class.getResourceAsStream("/google_plus_person_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ Person person = objectMapper.readValue(line, Person.class);
+
+ GooglePlusActivityUtil.updateActivity(person, activity);
+ LOGGER.info("activity: {}", activity);
+
+ assertNotNull(activity);
+ assertTrue(activity.getId().contains("id:googleplus:update"));
+ assertEquals(activity.getVerb(), "update");
+
+ Provider provider = activity.getProvider();
+ assertEquals(provider.getId(), "id:providers:googleplus");
+ assertEquals(provider.getDisplayName(), "GooglePlus");
+
+ ActivityObject actor = activity.getActor();
+ assertNotNull(actor.getImage());
+ assertTrue(actor.getId().contains("id:googleplus:"));
+ assertNotNull(actor.getUrl());
+
+ }
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while testing serializability: {}", ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java
new file mode 100644
index 0000000..b7bb3bf
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.processors;
+
+import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.extensions.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Provider;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+/**
+ * Tests conversion of gplus inputs to Activity.
+ */
+public class GooglePlusActivitySerDeIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivitySerDeIT.class);
+ private ObjectMapper objectMapper;
+
+ /**
+ * setup.
+ */
+ @BeforeClass
+ public void setup() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testActivityObjects() {
+ InputStream is = GooglePlusActivitySerDeIT.class.getResourceAsStream("/google_plus_activity_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ com.google.api.services.plus.model.Activity googlePlusActivity =
+ objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
+
+ GooglePlusActivityUtil.updateActivity(googlePlusActivity, activity);
+ LOGGER.info("activity: {}", activity);
+
+ assertNotNull(activity);
+ assert (activity.getId().contains("id:googleplus:post"));
+ assertEquals(activity.getVerb(), "post");
+
+ Provider provider = activity.getProvider();
+ assertEquals(provider.getId(), "id:providers:googleplus");
+ assertEquals(provider.getDisplayName(), "GooglePlus");
+
+ ActivityObject actor = activity.getActor();
+ assertNotNull(actor.getImage());
+ assert (actor.getId().contains("id:googleplus:"));
+ assertNotNull(actor.getUrl());
+
+ assertNotNull(activity.getPublished());
+ assertNotNull(activity.getTitle());
+ assertNotNull(activity.getUrl());
+
+ Map<String, Object> extensions = ExtensionUtil.getInstance().getExtensions(activity);
+ assertNotNull(extensions);
+
+ if (activity.getContent() != null) {
+ assertNotNull(extensions.get("rebroadcasts"));
+ assertNotNull(extensions.get("keywords"));
+ assertNotNull(extensions.get("likes"));
+ assert (((Map<String, Object>) extensions.get("rebroadcasts")).containsKey("count"));
+ assert (((Map<String, Object>) extensions.get("likes")).containsKey("count"));
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while testing serializability: {}", ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java
new file mode 100644
index 0000000..d3472c4
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.processors;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.gplus.processor.GooglePlusTypeConverter;
+import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.model.Person;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Tests conversion of gplus inputs to Activity
+ */
+public class GooglePlusTypeConverterIT {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverterIT.class);
+ private GooglePlusTypeConverter googlePlusTypeConverter;
+ private ObjectMapper objectMapper;
+
+ @BeforeClass
+ public void setup() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ googlePlusTypeConverter = new GooglePlusTypeConverter();
+ googlePlusTypeConverter.prepare(null);
+ }
+
+ @Test(dependsOnGroups = {"testGPlusUserDataProvider"})
+ public void testProcessPerson() throws IOException, ActivitySerializerException {
+
+ File file = new File("target/test-classes/GPlusUserDataProviderIT.stdout.txt");
+ InputStream is = new FileInputStream(file);
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ Person person = objectMapper.readValue(line, Person.class);
+ StreamsDatum streamsDatum = new StreamsDatum(person);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
+ GooglePlusActivityUtil.updateActivity(person, activity);
+
+ assertEquals(retList.size(), 1);
+ assert (retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ }
+ }
+ }
+
+ @Test(dependsOnGroups = {"testGPlusUserActivityProvider"})
+ public void testProcessActivity() throws IOException, ActivitySerializerException {
+
+ File file = new File("target/test-classes/GPlusUserActivityProviderIT.stdout.txt");
+ InputStream is = new FileInputStream(file);
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ while (br.ready()) {
+ String line = br.readLine();
+ if (!StringUtils.isEmpty(line)) {
+ LOGGER.info("raw: {}", line);
+ Activity activity = new Activity();
+
+ com.google.api.services.plus.model.Activity gPlusActivity = objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
+ StreamsDatum streamsDatum = new StreamsDatum(gPlusActivity);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
+ GooglePlusActivityUtil.updateActivity(gPlusActivity, activity);
+
+ assertEquals(retList.size(), 1);
+ assertTrue(retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java
new file mode 100644
index 0000000..9f8524d
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.providers;
+
+import org.apache.streams.gplus.provider.GPlusUserActivityProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.LineNumberReader;
+
+public class GPlusUserActivityProviderIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityProviderIT.class);
+
+ @Test(groups={"testGPlusUserActivityProvider"})
+ public void testGPlusUserActivityProvider() throws Exception {
+
+ String configfile = "./target/test-classes/GPlusUserActivityProviderIT.conf";
+ String outfile = "./target/test-classes/GPlusUserActivityProviderIT.stdout.txt";
+
+ String[] args = new String[2];
+ args[0] = configfile;
+ args[1] = outfile;
+
+ Thread testThread = new Thread(() -> {
+ try {
+ GPlusUserActivityProvider.main(args);
+ } catch ( Exception ex ) {
+ LOGGER.error("Test Exception!", ex);
+ }
+ });
+ testThread.start();
+ testThread.join(60000);
+
+ File out = new File(outfile);
+ assert (out.exists());
+ assert (out.canRead());
+ assert (out.isFile());
+
+ FileReader outReader = new FileReader(out);
+ LineNumberReader outCounter = new LineNumberReader(outReader);
+
+ while (outCounter.readLine() != null) {}
+
+ assert (outCounter.getLineNumber() >= 1);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java
new file mode 100644
index 0000000..012cdc2
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.providers;
+
+import org.apache.streams.gplus.provider.GPlusUserDataProvider;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.LineNumberReader;
+
+public class GPlusUserDataProviderIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataProviderIT.class);
+
+ @Test(groups={"testGPlusUserDataProvider"})
+ public void testGPlusUserDataProvider() throws Exception {
+
+ String configfile = "./target/test-classes/GPlusUserDataProviderIT.conf";
+ String outfile = "./target/test-classes/GPlusUserDataProviderIT.stdout.txt";
+
+ String[] args = new String[2];
+ args[0] = configfile;
+ args[1] = outfile;
+
+ Thread testThread = new Thread(() -> {
+ try {
+ GPlusUserDataProvider.main(args);
+ } catch ( Exception ex ) {
+ LOGGER.error("Test Exception!", ex);
+ }
+ });
+ testThread.start();
+ testThread.join(60000);
+
+ GPlusUserDataProvider.main(new String[]{configfile, outfile});
+
+ File out = new File(outfile);
+ assert (out.exists());
+ assert (out.canRead());
+ assert (out.isFile());
+
+ FileReader outReader = new FileReader(out);
+ LineNumberReader outCounter = new LineNumberReader(outReader);
+
+ while (outCounter.readLine() != null) {}
+
+ assert (outCounter.getLineNumber() >= 1);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java
new file mode 100644
index 0000000..afd70fa
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.providers;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.GPlusOAuthConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.gplus.provider.AbstractGPlusProvider;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.google.api.services.plus.Plus;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests for {@link org.apache.streams.gplus.provider.AbstractGPlusProvider}
+ */
+public class TestAbstractGPlusProvider extends RandomizedTest {
+
+ /**
+ * Test that every collector will be run and that data queued from the collectors will be processed.
+ */
+ @Test
+ @Repeat(iterations = 3)
+ public void testDataCollectorRunsPerUser() {
+ int numUsers = randomIntBetween(1, 1000);
+ List<UserInfo> userList = new LinkedList<>();
+ for (int i = 0; i < numUsers; ++i) {
+ userList.add(new UserInfo());
+ }
+ GPlusConfiguration config = new GPlusConfiguration();
+ GPlusOAuthConfiguration oauth = new GPlusOAuthConfiguration();
+ oauth.setAppName("a");
+ oauth.setPathToP12KeyFile("a");
+ oauth.setServiceAccountEmailAddress("a");
+ config.setOauth(oauth);
+ config.setGooglePlusUsers(userList);
+ AbstractGPlusProvider provider = new AbstractGPlusProvider(config) {
+
+ @Override
+ protected Plus createPlusClient() throws IOException {
+ return mock(Plus.class);
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ final BlockingQueue<StreamsDatum> q = queue;
+ return () -> {
+ try {
+ q.put(new StreamsDatum(null));
+ } catch (InterruptedException ie) {
+ fail("Test was interrupted");
+ }
+ };
+ }
+ };
+
+ try {
+ provider.prepare(null);
+ provider.startStream();
+ int datumCount = 0;
+ while (provider.isRunning()) {
+ datumCount += provider.readCurrent().size();
+ }
+ assertEquals(numUsers, datumCount);
+ } finally {
+ provider.cleanUp();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java
new file mode 100644
index 0000000..ae0c4f7
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.providers;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.gplus.provider.GPlusUserActivityCollector;
+import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.ActivityFeed;
+import org.joda.time.DateTime;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link org.apache.streams.gplus.provider.GPlusUserActivityCollector}
+ */
+public class TestGPlusUserActivityCollector extends RandomizedTest {
+
+ private static final String ACTIVITY_TEMPLATE = "{ \"kind\": \"plus#activity\", \"etag\": \"\\\"Vea_b94Y77GDGgRK7gFNPnolKQw/v1-6aVSBGT4qiStMoz7f2_AN2fM\\\"\", \"title\": \"\", \"published\": \"%s\", \"updated\": \"2014-10-27T06:26:33.927Z\", \"id\": \"z13twrlznpvtzz52w22mdt1y0k3of1djw04\", \"url\": \"https://plus.google.com/116771159471120611293/posts/GR7CGR8N5VL\", \"actor\": { \"id\": \"116771159471120611293\", \"displayName\": \"displayName\", \"url\": \"https://plus.google.com/116771159471120611293\", \"image\": { \"url\": \"https://lh6.googleusercontent.com/-C0fiZBxdvw0/AAAAAAAAAAI/AAAAAAAAJ5k/K4pgR3_-_ms/photo.jpg?sz=50\" } }, \"verb\": \"share\", \"object\": { \"objectType\": \"activity\", \"id\": \"z13zgvtiurjgfti1v234iflghvq2c1dge04\", \"actor\": { \"id\": \"104954254300557350002\", \"displayName\": \"displayName\", \"url\": \"https://plus.google.com/104954254300557350002\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-SO1scj4p2LA/AAAAAAAAAAI/AAAAAAAAI-s/efA
9LBVe144/photo.jpg?sz=50\" } }, \"content\": \"\", \"url\": \"https://plus.google.com/104954254300557350002/posts/AwewXhtn7ws\", \"replies\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/comments\" }, \"plusoners\": { \"totalItems\": 9, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/plusoners\" }, \"resharers\": { \"totalItems\": 0, \"selfLink\": \"https://content.googleapis.com/plus/v1/activities/z13twrlznpvtzz52w22mdt1y0k3of1djw04/people/resharers\" }, \"attachments\": [ { \"objectType\": \"photo\", \"id\": \"104954254300557350002.6074732746360957410\", \"content\": \"26/10/2014 - 1\", \"url\": \"https://plus.google.com/photos/104954254300557350002/albums/6074732747132702225/6074732746360957410\", \"image\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w506-h750/2014%2B-%2B1\", \"type\": \"ima
ge/jpeg\" }, \"fullImage\": { \"url\": \"https://lh4.googleusercontent.com/-oO3fnARlDm0/VE3JP1xHKeI/AAAAAAAAeCY/-X2jzc6HruA/w600-h1141/2014%2B-%2B1\", \"type\": \"image/jpeg\", \"height\": 1141, \"width\": 600 } } ] }, \"annotation\": \"Truth \U0001f61c\", \"provider\": { \"title\": \"Reshared Post\" }, \"access\": { \"kind\": \"plus#acl\", \"description\": \"Public\", \"items\": [ { \"type\": \"public\" } ] } }";
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private static final String IN_RANGE_IDENTIFIER = "data in range";
+
+ static {
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private static ActivityFeed createMockActivityFeed(
+ int numBefore,
+ int numAfter,
+ int numInRange,
+ DateTime after,
+ DateTime before,
+ boolean page) {
+ ActivityFeed feed = new ActivityFeed();
+ List<Activity> list = new LinkedList<>();
+ for (int i = 0; i < numAfter; ++i) {
+ DateTime published = before.plus(randomIntBetween(0, Integer.MAX_VALUE));
+ Activity activity = createActivityWithPublishedDate(published);
+ list.add(activity);
+ }
+ for (int i = 0; i < numInRange; ++i) {
+ DateTime published;
+ if ((before == null && after == null) || before == null) {
+ published = DateTime.now(); // no date range or end time date range so just make the time now.
+ } else if (after == null) {
+ published = before.minusMillis(randomIntBetween(1, Integer.MAX_VALUE)); //no beginning to range
+ } else { // has to be in range
+ long range = before.getMillis() - after.getMillis();
+ published = after.plus(range / 2); //in the middle
+ }
+ Activity activity = createActivityWithPublishedDate(published);
+ activity.setTitle(IN_RANGE_IDENTIFIER);
+ list.add(activity);
+ }
+ for (int i = 0; i < numBefore; ++i) {
+ DateTime published = after.minusMillis(randomIntBetween(1, Integer.MAX_VALUE));
+ Activity activity = createActivityWithPublishedDate(published);
+ list.add(activity);
+ }
+ if (page) {
+ feed.setNextPageToken("A");
+ } else {
+ feed.setNextPageToken(null);
+ }
+ feed.setItems(list);
+ return feed;
+ }
+
+ private static Activity createActivityWithPublishedDate(DateTime dateTime) {
+ Activity activity = new Activity();
+ activity.setPublished(new com.google.api.client.util.DateTime(dateTime.getMillis()));
+ activity.setId("a");
+ return activity;
+ }
+
+ /**
+ * Creates a randomized activity and randomized date range.
+ * <p/>
+ * The activity feed is separated into three chunks,
+ * |. . . data too recent to be in date range . . .||. . . data in date range. . .||. . . data too old to be in date range|
+ * [index 0, ............................................................................................., index length-1]
+ * <p/>
+ * Inside of those chunks data has no order, but the list is ordered by those three chunks.
+ * <p/>
+ * The test will check to see if the num of data in the date range make onto the output queue.
+ */
+ @Test
+ @Repeat(iterations = 3)
+ public void testWithBeforeAndAfterDates() throws InterruptedException {
+ //initialize counts assuming no date ranges will be used
+ int numActivities = randomIntBetween(0, 1000);
+ int numActivitiesInDateRange = numActivities;
+ int numberOutOfRange = 0;
+ int numBeforeRange = 0;
+ int numAfterRange = 0;
+ //determine if date ranges will be used
+ DateTime beforeDate = null;
+ DateTime afterDate = null;
+ if (randomInt() % 2 == 0) {
+ beforeDate = DateTime.now().minusDays(randomIntBetween(1, 5));
+ }
+ if (randomInt() % 2 == 0) {
+ if (beforeDate == null) {
+ afterDate = DateTime.now().minusDays(randomIntBetween(1, 10));
+ } else {
+ afterDate = beforeDate.minusDays(randomIntBetween(1, 10));
+ }
+ }
+ //update counts if date ranges are going to be used.
+ if (beforeDate != null || afterDate != null) { //assign amount to be in range
+ numActivitiesInDateRange = randomIntBetween(0, numActivities);
+ numberOutOfRange = numActivities - numActivitiesInDateRange;
+ }
+ if (beforeDate == null && afterDate != null) { //assign all out of range to be before the start of the range
+ numBeforeRange = numberOutOfRange;
+ } else if (beforeDate != null && afterDate == null) { //assign all out of range to be after the start of the range
+ numAfterRange = numberOutOfRange;
+ } else if (beforeDate != null && afterDate != null) { //assign half before range and half after the range
+ numAfterRange = (numberOutOfRange / 2) + (numberOutOfRange % 2);
+ numBeforeRange = numberOutOfRange / 2;
+ }
+
+ Plus plus = createMockPlus(numBeforeRange, numAfterRange, numActivitiesInDateRange, afterDate, beforeDate);
+ BackOffStrategy strategy = new ConstantTimeBackOffStrategy(1);
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ UserInfo userInfo = new UserInfo();
+ userInfo.setUserId("A");
+ userInfo.setAfterDate(afterDate);
+ userInfo.setBeforeDate(beforeDate);
+ GPlusUserActivityCollector collector = new GPlusUserActivityCollector(plus, datums, strategy, userInfo);
+ collector.run();
+
+ assertEquals(numActivitiesInDateRange, datums.size());
+ while (!datums.isEmpty()) {
+ StreamsDatum datum = datums.take();
+ assertNotNull(datum);
+ assertNotNull(datum.getDocument());
+ assertTrue(datum.getDocument() instanceof String);
+ assertTrue(((String)datum.getDocument()).contains(IN_RANGE_IDENTIFIER)); //only in range documents are on the out going queue.
+ }
+ }
+
+ private Plus createMockPlus(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
+ Plus plus = mock(Plus.class);
+ final Plus.Activities activities = createMockPlusActivities(numBefore, numAfter, numInRange, after, before);
+ doAnswer(invocationOnMock -> activities).when(plus).activities();
+ return plus;
+ }
+
+ private Plus.Activities createMockPlusActivities(
+ final int numBefore,
+ final int numAfter,
+ final int numInRange,
+ final DateTime after,
+ final DateTime before) {
+ Plus.Activities activities = mock(Plus.Activities.class);
+ try {
+ Plus.Activities.List list = createMockPlusActivitiesList(numBefore, numAfter, numInRange, after, before);
+ when(activities.list(anyString(), anyString())).thenReturn(list);
+ } catch (IOException ioe) {
+ fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
+ }
+ return activities;
+ }
+
+ private Plus.Activities.List createMockPlusActivitiesList(
+ final int numBefore,
+ final int numAfter,
+ final int numInRange,
+ final DateTime after,
+ final DateTime before) {
+ Plus.Activities.List list = mock(Plus.Activities.List.class);
+ when(list.setMaxResults(anyLong())).thenReturn(list);
+ when(list.setPageToken(anyString())).thenReturn(list);
+ ActivityFeedAnswer answer = new ActivityFeedAnswer(numBefore, numAfter, numInRange, after, before);
+ try {
+ doAnswer(answer).when(list).execute();
+ } catch (IOException ioe) {
+ fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
+ }
+ return list;
+ }
+
+ private static class ActivityFeedAnswer implements Answer<ActivityFeed> {
+ private int afterCount = 0;
+ private int beforeCount = 0;
+ private int inCount = 0;
+ private int maxBatch = 100;
+
+ private int numAfter;
+ private int numInRange;
+ private int numBefore;
+ private DateTime after;
+ private DateTime before;
+
+ private ActivityFeedAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
+ this.numBefore = numBefore;
+ this.numAfter = numAfter;
+ this.numInRange = numInRange;
+ this.after = after;
+ this.before = before;
+ }
+
+ @Override
+ public ActivityFeed answer(InvocationOnMock invocationOnMock) throws Throwable {
+ int totalCount = 0;
+ int batchAfter = 0;
+ int batchBefore = 0;
+ int batchIn = 0;
+ if (afterCount != numAfter) {
+ if (numAfter - afterCount >= maxBatch) {
+ afterCount += maxBatch;
+ batchAfter += maxBatch;
+ totalCount += batchAfter;
+ } else {
+ batchAfter += numAfter - afterCount;
+ totalCount += numAfter - afterCount;
+ afterCount = numAfter;
+ }
+ }
+ if (totalCount < maxBatch && inCount != numInRange) {
+ if (numInRange - inCount >= maxBatch - totalCount) {
+ inCount += maxBatch - totalCount;
+ batchIn += maxBatch - totalCount;
+ totalCount += batchIn;
+ } else {
+ batchIn += numInRange - inCount;
+ totalCount += numInRange - inCount;
+ inCount = numInRange;
+ }
+ }
+ if (totalCount < maxBatch && beforeCount != numBefore) {
+ if (numBefore - batchBefore >= maxBatch - totalCount) {
+ batchBefore += maxBatch - totalCount;
+ totalCount = maxBatch;
+ beforeCount += batchBefore;
+ } else {
+ batchBefore += numBefore - beforeCount;
+ totalCount += numBefore - beforeCount;
+ beforeCount = numBefore;
+ }
+ }
+
+ return createMockActivityFeed(
+ batchBefore,
+ batchAfter,
+ batchIn,
+ after,
+ before,
+ numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
+ }
+ }
+
+
+}
[9/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
STREAMS-463: Move every class in all repos underneath org.apache.streams, this closes apache/incubator-streams#356
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b71cce83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b71cce83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b71cce83
Branch: refs/heads/master
Commit: b71cce83de01f9c07db59e0af9915e75366c86db
Parents: 571d406
Author: smarthi <sm...@apache.org>
Authored: Tue Feb 7 23:31:32 2017 -0500
Committer: smarthi <sm...@apache.org>
Committed: Tue Feb 7 23:31:32 2017 -0500
----------------------------------------------------------------------
README.md | 2 +-
src/site/markdown/release.md | 4 +-
.../streams-persist-cassandra/pom.xml | 5 -
streams-contrib/streams-persist-graph/pom.xml | 7 -
.../hdfs/test/HdfsPersistConfigTest.java | 243 ++++++-------
.../streams/hdfs/test/TestHdfsPersist.java | 111 +++---
streams-contrib/streams-persist-mongo/pom.xml | 5 -
streams-contrib/streams-persist-neo4j/pom.xml | 5 -
.../gmail/provider/GMailImapProviderTask.java | 62 ----
.../GMailMessageActivitySerializer.java | 189 ----------
.../google/gmail/provider/GMailProvider.java | 191 ----------
.../gmail/provider/GMailImapProviderTask.java | 62 ++++
.../GMailMessageActivitySerializer.java | 193 ++++++++++
.../streams/gmail/provider/GMailProvider.java | 183 ++++++++++
.../com/google/gmail/GMailConfiguration.json | 17 -
.../google/gmail/GMailConfiguration.json | 17 +
.../gmail/test/GMailMessageSerDeTest.java | 67 ----
.../gmail/test/GMailMessageSerDeTest.java | 67 ++++
.../processor/GooglePlusCommentProcessor.java | 98 -----
.../processor/GooglePlusTypeConverter.java | 140 --------
.../gplus/provider/AbstractGPlusProvider.java | 260 --------------
.../gplus/provider/GPlusActivitySerializer.java | 70 ----
.../gplus/provider/GPlusDataCollector.java | 68 ----
.../provider/GPlusUserActivityCollector.java | 152 --------
.../provider/GPlusUserActivityProvider.java | 131 -------
.../gplus/provider/GPlusUserDataCollector.java | 106 ------
.../gplus/provider/GPlusUserDataProvider.java | 134 -------
.../util/GPlusActivityDeserializer.java | 172 ---------
.../util/GPlusCommentDeserializer.java | 104 ------
.../serializer/util/GPlusEventClassifier.java | 68 ----
.../util/GPlusPersonDeserializer.java | 122 -------
.../serializer/util/GooglePlusActivityUtil.java | 298 ----------------
.../processor/GooglePlusCommentProcessor.java | 96 +++++
.../processor/GooglePlusTypeConverter.java | 140 ++++++++
.../gplus/provider/AbstractGPlusProvider.java | 260 ++++++++++++++
.../gplus/provider/GPlusActivitySerializer.java | 70 ++++
.../gplus/provider/GPlusDataCollector.java | 68 ++++
.../provider/GPlusUserActivityCollector.java | 152 ++++++++
.../provider/GPlusUserActivityProvider.java | 131 +++++++
.../gplus/provider/GPlusUserDataCollector.java | 106 ++++++
.../gplus/provider/GPlusUserDataProvider.java | 133 +++++++
.../util/GPlusActivityDeserializer.java | 172 +++++++++
.../util/GPlusCommentDeserializer.java | 103 ++++++
.../serializer/util/GPlusEventClassifier.java | 68 ++++
.../util/GPlusPersonDeserializer.java | 122 +++++++
.../serializer/util/GooglePlusActivityUtil.java | 298 ++++++++++++++++
.../com/google/gplus/GPlusConfiguration.json | 101 ------
.../google/gplus/GPlusConfiguration.json | 101 ++++++
.../google/gplus/GooglePlusCommentSerDeIT.java | 122 -------
.../google/gplus/GooglePlusPersonSerDeIT.java | 106 ------
.../processor/GooglePlusActivitySerDeIT.java | 119 -------
.../provider/TestAbstractGPlusProvider.java | 97 -----
.../TestGPlusUserActivityCollector.java | 308 ----------------
.../provider/TestGPlusUserDataCollector.java | 137 -------
.../util/GPlusEventClassifierTest.java | 75 ----
.../streams/gplus/GooglePlusCommentSerDeIT.java | 116 ++++++
.../streams/gplus/GooglePlusPersonSerDeIT.java | 103 ++++++
.../processors/GooglePlusActivitySerDeIT.java | 119 +++++++
.../processors/GooglePlusTypeConverterIT.java | 132 +++++++
.../providers/GPlusUserActivityProviderIT.java | 67 ++++
.../providers/GPlusUserDataProviderIT.java | 70 ++++
.../providers/TestAbstractGPlusProvider.java | 98 +++++
.../TestGPlusUserActivityCollector.java | 299 ++++++++++++++++
.../providers/TestGPlusUserDataCollector.java | 135 +++++++
.../util/GPlusEventClassifierTest.java | 71 ++++
.../processors/GooglePlusTypeConverterIT.java | 132 -------
.../providers/GPlusUserActivityProviderIT.java | 67 ----
.../test/providers/GPlusUserDataProviderIT.java | 69 ----
.../com/instagram/InstagramConfiguration.json | 71 ----
.../InstagramUserInformationConfiguration.json | 20 --
.../instagram/InstagramConfiguration.json | 71 ++++
.../InstagramUserInformationConfiguration.json | 20 ++
.../main/jsonschema/com/moreover/Moreover.json | 337 ------------------
.../com/moreover/MoreoverConfiguration.json | 38 --
.../org/apache/streams/moreover/Moreover.json | 337 ++++++++++++++++++
.../streams/moreover/MoreoverConfiguration.json | 38 ++
.../perpetual/RssFeedSchedulerTest.java | 20 +-
.../sysomos/provider/SysomosProvider.java | 128 +++----
.../main/jsonschema/com/sysomos/Sysomos.json | 61 ----
.../com/sysomos/SysomosConfiguration.json | 43 ---
.../org/apache/streams/sysomos/Sysomos.json | 61 ++++
.../streams/sysomos/SysomosConfiguration.json | 43 +++
.../com/sysomos/test/SysomosJsonSerDeIT.java | 68 ----
.../com/sysomos/test/SysomosXmlSerDeIT.java | 102 ------
.../sysomos/test/SysomosJsonSerDeIT.java | 69 ++++
.../streams/sysomos/test/SysomosXmlSerDeIT.java | 102 ++++++
.../src/main/jsonschema/com/twitter/Delete.json | 37 --
.../src/main/jsonschema/com/twitter/Follow.json | 18 -
.../main/jsonschema/com/twitter/FriendList.json | 24 --
.../main/jsonschema/com/twitter/Retweet.json | 19 -
.../com/twitter/TwitterConfiguration.json | 87 -----
.../twitter/TwitterFollowingConfiguration.json | 33 --
.../com/twitter/TwitterStreamConfiguration.json | 45 ---
.../TwitterTimelineProviderConfiguration.json | 23 --
.../TwitterUserInformationConfiguration.json | 25 --
.../src/main/jsonschema/com/twitter/User.json | 129 -------
.../jsonschema/com/twitter/UserstreamEvent.json | 52 ---
.../src/main/jsonschema/com/twitter/tweet.json | 219 ------------
.../streams/twitter/TwitterConfiguration.json | 87 +++++
.../twitter/TwitterFollowingConfiguration.json | 33 ++
.../twitter/TwitterStreamConfiguration.json | 45 +++
.../TwitterTimelineProviderConfiguration.json | 23 ++
.../TwitterUserInformationConfiguration.json | 25 ++
.../org/apache/streams/twitter/pojo/Delete.json | 37 ++
.../org/apache/streams/twitter/pojo/Follow.json | 18 +
.../apache/streams/twitter/pojo/FriendList.json | 24 ++
.../apache/streams/twitter/pojo/Retweet.json | 19 +
.../org/apache/streams/twitter/pojo/User.json | 129 +++++++
.../streams/twitter/pojo/UserstreamEvent.json | 52 +++
.../org/apache/streams/twitter/pojo/tweet.json | 219 ++++++++++++
.../youtube/processor/YoutubeTypeConverter.java | 129 -------
.../provider/YoutubeChannelDataCollector.java | 114 ------
.../provider/YoutubeChannelProvider.java | 132 -------
.../youtube/provider/YoutubeDataCollector.java | 70 ----
.../com/youtube/provider/YoutubeProvider.java | 277 ---------------
.../provider/YoutubeUserActivityCollector.java | 229 ------------
.../provider/YoutubeUserActivityProvider.java | 132 -------
.../youtube/serializer/YoutubeActivityUtil.java | 200 -----------
.../serializer/YoutubeChannelDeserializer.java | 152 --------
.../serializer/YoutubeEventClassifier.java | 62 ----
.../serializer/YoutubeVideoDeserializer.java | 118 -------
.../youtube/processor/YoutubeTypeConverter.java | 129 +++++++
.../provider/YoutubeChannelDataCollector.java | 114 ++++++
.../provider/YoutubeChannelProvider.java | 124 +++++++
.../youtube/provider/YoutubeDataCollector.java | 70 ++++
.../youtube/provider/YoutubeProvider.java | 270 ++++++++++++++
.../provider/YoutubeUserActivityCollector.java | 228 ++++++++++++
.../provider/YoutubeUserActivityProvider.java | 132 +++++++
.../youtube/serializer/YoutubeActivityUtil.java | 200 +++++++++++
.../serializer/YoutubeChannelDeserializer.java | 152 ++++++++
.../serializer/YoutubeEventClassifier.java | 62 ++++
.../serializer/YoutubeVideoDeserializer.java | 117 ++++++
.../com/youtube/YoutubeConfiguration.json | 105 ------
.../streams/youtube/YoutubeConfiguration.json | 105 ++++++
.../processor/YoutubeTypeConverterTest.java | 111 ------
.../YoutubeChannelDataCollectorTest.java | 102 ------
.../youtube/provider/YoutubeProviderTest.java | 166 ---------
.../YoutubeUserActivityCollectorTest.java | 354 -------------------
.../serializer/YoutubeEventClassifierTest.java | 50 ---
.../serializer/YoutubeVideoSerDeTest.java | 123 -------
.../processor/YoutubeTypeConverterTest.java | 111 ++++++
.../YoutubeChannelDataCollectorTest.java | 102 ++++++
.../youtube/provider/YoutubeProviderTest.java | 166 +++++++++
.../YoutubeUserActivityCollectorTest.java | 354 +++++++++++++++++++
.../serializer/YoutubeEventClassifierTest.java | 50 +++
.../serializer/YoutubeVideoSerDeTest.java | 120 +++++++
.../providers/YoutubeChannelProviderIT.java | 3 +-
.../YoutubeUserActivityProviderIT.java | 2 +-
.../streams-plugin-cassandra/pom.xml | 5 -
.../streams-plugin-elasticsearch/pom.xml | 5 -
streams-plugins/streams-plugin-hbase/pom.xml | 5 -
streams-plugins/streams-plugin-hive/pom.xml | 5 -
streams-plugins/streams-plugin-pig/pom.xml | 5 -
streams-plugins/streams-plugin-scala/pom.xml | 6 -
.../src/test/resources/log4j.properties | 2 +-
.../src/test/resources/logback-test.xml | 2 +-
156 files changed, 8060 insertions(+), 8179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index b51df17..1d1a4b2 100644
--- a/README.md
+++ b/README.md
@@ -19,4 +19,4 @@ Apache Streams is an effort undergoing incubation at [The Apache Software Founda
## License
-Copyright (c) 2016. Apache Streams is licensed under the [Apache License 2.0](LICENSE).
+Copyright (c) 2017. Apache Streams is licensed under the [Apache License 2.0](LICENSE).
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/src/site/markdown/release.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/release.md b/src/site/markdown/release.md
index 7998b4f..44730be 100644
--- a/src/site/markdown/release.md
+++ b/src/site/markdown/release.md
@@ -24,8 +24,8 @@ As an alternative to releasing separately, the projects MAY be released together
export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m"
- 2. Use the latest Sun 1.7.0 JDK
- 3. Use Maven 3.2.1 or later
+ 2. Use the latest Sun 1.8.x JDK
+ 3. Use Maven 3.3.9 or later
4. Make sure the [Release Setup](release-setup.html) steps have been performed.
2. Prepare the source for release:
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-cassandra/pom.xml b/streams-contrib/streams-persist-cassandra/pom.xml
index 75785dd..498548a 100644
--- a/streams-contrib/streams-persist-cassandra/pom.xml
+++ b/streams-contrib/streams-persist-cassandra/pom.xml
@@ -84,11 +84,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-testing</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-graph/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-graph/pom.xml b/streams-contrib/streams-persist-graph/pom.xml
index 996c706..bda1943 100644
--- a/streams-contrib/streams-persist-graph/pom.xml
+++ b/streams-contrib/streams-persist-graph/pom.xml
@@ -76,12 +76,6 @@
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-testing</artifactId>
<version>${project.version}</version>
@@ -91,7 +85,6 @@
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
index 6e5f351..6d7aaf6 100644
--- a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
+++ b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
@@ -25,152 +25,143 @@ import org.apache.streams.hdfs.HdfsWriterConfiguration;
import org.apache.streams.hdfs.WebHdfsPersistReader;
import org.apache.streams.hdfs.WebHdfsPersistWriter;
+import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URISyntaxException;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
/**
* Test for checking that strings append to FS paths as expected
*/
public class HdfsPersistConfigTest {
- private static final Logger LOGGER = LoggerFactory.getLogger(HdfsPersistConfigTest.class);
-
- @Test
- public void getWriterFileUriTest()
- {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
- webHdfsPersistWriter.prepare(null);
- assertTrue(webHdfsPersistWriter.isConnected());
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(HdfsPersistConfigTest.class);
- @Test
- public void getWriterHdfsUriTest()
- {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
- writerConfiguration.setHost("localhost");
- writerConfiguration.setPort(9000L);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
+ @Test
+ public void getWriterFileUriTest() {
+ HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
+ writerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
+ writerConfiguration.setPath("path");
+ writerConfiguration.setWriterPath("writerPath");
+ writerConfiguration.setUser("cloudera");
- }
-
- @Test
- public void getWriterWebHdfsUriTest()
- {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
- writerConfiguration.setHost("localhost");
- writerConfiguration.setPort(57000L);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
+ WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
+ String uri = null;
+ try {
+ uri = webHdfsPersistWriter.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
}
-
- @Test
- public void getReaderFileUriTest()
- {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
+ Assert.assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
+ webHdfsPersistWriter.prepare(null);
+ Assert.assertTrue(webHdfsPersistWriter.isConnected());
+ }
+
+ @Test
+ public void getWriterHdfsUriTest() {
+ HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
+ writerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
+ writerConfiguration.setHost("localhost");
+ writerConfiguration.setPort(9000L);
+ writerConfiguration.setPath("path");
+ writerConfiguration.setWriterPath("writerPath");
+ writerConfiguration.setUser("cloudera");
+
+ WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
+
+ String uri = null;
+ try {
+ uri = webHdfsPersistWriter.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
}
+ Assert.assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
+
+ }
+
+ @Test
+ public void getWriterWebHdfsUriTest() {
+ HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
+ writerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
+ writerConfiguration.setHost("localhost");
+ writerConfiguration.setPort(57000L);
+ writerConfiguration.setPath("path");
+ writerConfiguration.setWriterPath("writerPath");
+ writerConfiguration.setUser("cloudera");
+
+ WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
+
+ String uri = null;
+ try {
+ uri = webHdfsPersistWriter.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
+ }
+ Assert.assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
- @Test
- public void getReaderHdfsUriTest()
- {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
- readerConfiguration.setHost("localhost");
- readerConfiguration.setPort(9000L);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
+ }
- }
+ @Test
+ public void getReaderFileUriTest() {
+ HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
+ readerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
+ readerConfiguration.setPath("path");
+ readerConfiguration.setReaderPath("readerPath");
- @Test
- public void getReaderWebHdfsUriTest()
- {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
- readerConfiguration.setHost("localhost");
- readerConfiguration.setPort(57000L);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- fail("URI Syntax");
- }
- assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
+ WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
+ String uri = null;
+ try {
+ uri = webHdfsPersistReader.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
+ }
+ Assert.assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
+ }
+
+ @Test
+ public void getReaderHdfsUriTest() {
+ HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
+ readerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
+ readerConfiguration.setHost("localhost");
+ readerConfiguration.setPort(9000L);
+ readerConfiguration.setPath("path");
+ readerConfiguration.setReaderPath("readerPath");
+
+ WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
+
+ String uri = null;
+ try {
+ uri = webHdfsPersistReader.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
}
+ Assert.assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
+
+ }
+
+ @Test
+ public void getReaderWebHdfsUriTest() {
+ HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
+ readerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
+ readerConfiguration.setHost("localhost");
+ readerConfiguration.setPort(57000L);
+ readerConfiguration.setPath("path");
+ readerConfiguration.setReaderPath("readerPath");
+
+ WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
+
+ String uri = null;
+ try {
+ uri = webHdfsPersistReader.getURI().toString();
+ } catch (URISyntaxException e) {
+ Assert.fail("URI Syntax");
+ }
+ Assert.assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
index 1ee1e39..798a8b8 100644
--- a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
+++ b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
@@ -49,75 +49,78 @@ import java.util.List;
*/
public class TestHdfsPersist {
- private static final Logger LOGGER = LoggerFactory.getLogger(TestHdfsPersist.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestHdfsPersist.class);
- private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- @Before
- public void setup() {
- File file = new File("/target/TestHdfsPersist/");
- if( file.exists())
- file.delete();
+ @Before
+ public void setup() {
+ File file = new File("/target/TestHdfsPersist/");
+ if (file.exists()) {
+ file.delete();
}
+ }
- @Test
- public void TestHdfsPersist() throws Exception {
+ @Test
+ public void TestHdfsPersist() throws Exception {
- List<List<String>> fieldArrays = new ArrayList<>();
- fieldArrays.add(new ArrayList<>());
- fieldArrays.add(Collections.singletonList("ID"));
- fieldArrays.add(Arrays.asList("ID", "DOC"));
- fieldArrays.add(Arrays.asList("ID", "TS", "DOC"));
- fieldArrays.add(Arrays.asList("ID", "TS", "META", "DOC"));
-
- for( List<String> fields : fieldArrays )
- TestHdfsPersistCase(fields);
+ List<List<String>> fieldArrays = new ArrayList<>();
+ fieldArrays.add(new ArrayList<>());
+ fieldArrays.add(Collections.singletonList("ID"));
+ fieldArrays.add(Arrays.asList("ID", "DOC"));
+ fieldArrays.add(Arrays.asList("ID", "TS", "DOC"));
+ fieldArrays.add(Arrays.asList("ID", "TS", "META", "DOC"));
+ for (List<String> fields : fieldArrays) {
+ TestHdfsPersistCase(fields);
}
- public void TestHdfsPersistCase(List<String> fields) throws Exception {
-
- HdfsConfiguration hdfsConfiguration = new HdfsConfiguration().withScheme(HdfsConfiguration.Scheme.FILE).withHost("localhost").withUser("cloudera").withPath("target/TestHdfsPersist");
- hdfsConfiguration.setFields(fields);
- HdfsWriterConfiguration hdfsWriterConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsWriterConfiguration.class);
- if( fields.size() % 2 == 1 )
- hdfsWriterConfiguration.setCompression(HdfsWriterConfiguration.Compression.GZIP);
- hdfsWriterConfiguration.setWriterFilePrefix("activities");
- hdfsWriterConfiguration.setWriterPath(Integer.toString(fields.size()));
- WebHdfsPersistWriter writer = new WebHdfsPersistWriter(hdfsWriterConfiguration);
-
- writer.prepare(null);
-
- InputStream testActivityFolderStream = TestHdfsPersist.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
+ }
- int count = 0;
+ public void TestHdfsPersistCase(List<String> fields) throws Exception {
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = TestHdfsPersist.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.getAdditionalProperties().remove("$license");
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- writer.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
+ HdfsConfiguration hdfsConfiguration = new HdfsConfiguration().withScheme(HdfsConfiguration.Scheme.FILE).withHost("localhost").withUser("cloudera").withPath("target/TestHdfsPersist");
+ hdfsConfiguration.setFields(fields);
+ HdfsWriterConfiguration hdfsWriterConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsWriterConfiguration.class);
+ if (fields.size() % 2 == 1) {
+ hdfsWriterConfiguration.setCompression(HdfsWriterConfiguration.Compression.GZIP);
+ }
+ hdfsWriterConfiguration.setWriterFilePrefix("activities");
+ hdfsWriterConfiguration.setWriterPath(Integer.toString(fields.size()));
+ WebHdfsPersistWriter writer = new WebHdfsPersistWriter(hdfsWriterConfiguration);
+
+ writer.prepare(null);
+
+ InputStream testActivityFolderStream = TestHdfsPersist.class.getClassLoader()
+ .getResourceAsStream("activities");
+ List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
+
+ int count = 0;
+
+ for (String file : files) {
+ LOGGER.info("File: " + file);
+ InputStream testActivityFileStream = TestHdfsPersist.class.getClassLoader()
+ .getResourceAsStream("activities/" + file);
+ Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
+ activity.getAdditionalProperties().remove("$license");
+ StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
+ writer.write(datum);
+ LOGGER.info("Wrote: " + activity.getVerb());
+ count++;
+ }
- writer.cleanUp();
+ writer.cleanUp();
- HdfsReaderConfiguration hdfsReaderConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsReaderConfiguration.class);
+ HdfsReaderConfiguration hdfsReaderConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsReaderConfiguration.class);
- WebHdfsPersistReader reader = new WebHdfsPersistReader(hdfsReaderConfiguration);
- hdfsReaderConfiguration.setReaderPath(Integer.toString(fields.size()));
+ WebHdfsPersistReader reader = new WebHdfsPersistReader(hdfsReaderConfiguration);
+ hdfsReaderConfiguration.setReaderPath(Integer.toString(fields.size()));
- reader.prepare(null);
+ reader.prepare(null);
- StreamsResultSet resultSet = reader.readAll();
+ StreamsResultSet resultSet = reader.readAll();
- Assert.assertEquals(resultSet.size(), count);
+ Assert.assertEquals(resultSet.size(), count);
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-mongo/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-mongo/pom.xml b/streams-contrib/streams-persist-mongo/pom.xml
index 3183455..877821d 100644
--- a/streams-contrib/streams-persist-mongo/pom.xml
+++ b/streams-contrib/streams-persist-mongo/pom.xml
@@ -73,11 +73,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.streams</groupId>
<artifactId>streams-testing</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-persist-neo4j/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-neo4j/pom.xml b/streams-contrib/streams-persist-neo4j/pom.xml
index d117558..ab8d71a 100644
--- a/streams-contrib/streams-persist-neo4j/pom.xml
+++ b/streams-contrib/streams-persist-neo4j/pom.xml
@@ -92,11 +92,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.streams</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailImapProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailImapProviderTask.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailImapProviderTask.java
deleted file mode 100644
index 168aed7..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailImapProviderTask.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gmail.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.util.ComponentUtils;
-
-import com.googlecode.gmail4j.GmailMessage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * GMailImapProviderTask collects Gmail via IMAP driver.
- */
-public class GMailImapProviderTask implements Runnable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailImapProviderTask.class);
-
- private GMailProvider provider;
-
- public GMailImapProviderTask(GMailProvider provider) {
- this.provider = provider;
- }
-
- @Override
- public void run() {
-
- final List<GmailMessage> messages = this.provider.imapClient.getUnreadMessages();
-
- for (GmailMessage message : messages) {
-
- Activity activity;
- GMailMessageActivitySerializer serializer = new GMailMessageActivitySerializer( this.provider );
- activity = serializer.deserialize(message);
- StreamsDatum entry = new StreamsDatum(activity);
- ComponentUtils.offerUntilSuccess(entry, this.provider.providerQueue);
-
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailMessageActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailMessageActivitySerializer.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailMessageActivitySerializer.java
deleted file mode 100644
index bdc22db..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailMessageActivitySerializer.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gmail.provider;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Generator;
-import org.apache.streams.pojo.json.Icon;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.googlecode.gmail4j.GmailException;
-import com.googlecode.gmail4j.GmailMessage;
-import com.googlecode.gmail4j.javamail.JavaMailGmailMessage;
-import com.sun.mail.imap.IMAPFolder;
-import com.sun.mail.imap.IMAPMessage;
-import com.sun.mail.imap.IMAPSSLStore;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import javax.mail.internet.MimeMultipart;
-
-/**
- * GMailMessageActivitySerializer converts a GMail message to Activity.
- */
-public class GMailMessageActivitySerializer implements ActivitySerializer<GmailMessage> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GMailMessageActivitySerializer.class);
-
- private GMailProvider provider;
-
- public GMailMessageActivitySerializer(GMailProvider provider) {
-
- this.provider = provider;
-
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, Boolean.FALSE);
-
- mapper.addMixInAnnotations(IMAPSSLStore.class, MessageMixIn.class);
- mapper.addMixInAnnotations(IMAPFolder.class, MessageMixIn.class);
- mapper.addMixInAnnotations(IMAPMessage.class, MessageMixIn.class);
- mapper.addMixInAnnotations(MimeMultipart.class, MessageMixIn.class);
- mapper.addMixInAnnotations(JavaMailGmailMessage.class, MessageMixIn.class);
-
- }
-
- public GMailMessageActivitySerializer() {
- }
-
- @Override
- public String serializationFormat() {
- return "gmail.v1";
- }
-
- @Override
- public GmailMessage serialize(Activity activity) {
- return null;
- }
-
- @Override
- public Activity deserialize(GmailMessage gmailMessage) {
-
- Activity activity = new Activity();
- activity.setId(formatId(this.provider.getConfig().getUserName(), String.valueOf(gmailMessage.getMessageNumber())));
- activity.setPublished(new DateTime(gmailMessage.getSendDate()));
- Provider provider = new Provider();
- provider.setId("http://gmail.com");
- provider.setDisplayName("GMail");
- activity.setProvider(provider);
- ActivityObject actor = new ActivityObject();
- actor.setId(gmailMessage.getFrom().getEmail());
- actor.setDisplayName(gmailMessage.getFrom().getName());
- activity.setActor(actor);
- activity.setVerb("email");
- ActivityObject object = new ActivityObject();
- try {
- object.setId(gmailMessage.getTo().get(0).getEmail());
- object.setDisplayName(gmailMessage.getTo().get(0).getName());
- } catch( GmailException e ) {
- LOGGER.warn(e.getMessage());
- }
- activity.setTitle(gmailMessage.getSubject());
- try {
- activity.setContent(gmailMessage.getContentText());
- } catch( GmailException e ) {
- LOGGER.warn(e.getMessage());
- }
- activity.setObject(object);
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<GmailMessage> serializedList) {
- throw new NotImplementedException("Not currently implemented");
- }
-
- public Activity convert(ObjectNode event) {
- return null;
- }
-
- public static Generator buildGenerator(ObjectNode event) {
- return null;
- }
-
- public static Icon getIcon(ObjectNode event) {
- return null;
- }
-
- public static Provider buildProvider(ObjectNode event) {
- Provider provider = new Provider();
- provider.setId("id:providers:gmail");
- return provider;
- }
-
- public static List<Object> getLinks(ObjectNode event) {
- return null;
- }
-
- public static String getUrls(ObjectNode event) {
- return null;
- }
-
- public static void addGMailExtension(Activity activity, GmailMessage gmailMessage) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("gmail", gmailMessage);
- }
-
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-
- interface MessageMixIn {
- @JsonManagedReference
- @JsonIgnore
- IMAPSSLStore getDefaultFolder(); // we don't need it!
- @JsonManagedReference
- @JsonIgnore
- IMAPSSLStore getPersonalNamespaces(); // we don't need it!
- @JsonManagedReference
- @JsonIgnore
- IMAPFolder getStore(); // we don't need it!
- // @JsonManagedReference
-// @JsonIgnore
-// @JsonBackReference
- //IMAPFolder getParent(); // we don't need it!
- @JsonManagedReference
- @JsonIgnore
- @JsonBackReference
- IMAPMessage getFolder(); // we don't need it!
- @JsonManagedReference
- @JsonIgnore
- @JsonProperty("parent")
- @JsonBackReference
- MimeMultipart getParent();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailProvider.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailProvider.java
deleted file mode 100644
index bd21acc..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/com/google/gmail/provider/GMailProvider.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gmail.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-
-import com.google.gmail.GMailConfiguration;
-import com.googlecode.gmail4j.GmailClient;
-import com.googlecode.gmail4j.GmailConnection;
-import com.googlecode.gmail4j.http.HttpGmailConnection;
-import com.googlecode.gmail4j.javamail.ImapGmailClient;
-import com.googlecode.gmail4j.javamail.ImapGmailConnection;
-import com.googlecode.gmail4j.rss.RssGmailClient;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * GMailProvider collects messages from GMail.
- */
-public class GMailProvider implements StreamsProvider, Serializable {
-
- public final static String STREAMS_ID = "GMailProvider";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailProvider.class);
-
- private GMailConfiguration config;
-
- private Class klass;
-
- public GMailConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(GMailConfiguration config) {
- this.config = config;
- }
-
- protected BlockingQueue inQueue = new LinkedBlockingQueue<>(10000);
-
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
- protected Future task;
-
- public BlockingQueue<Object> getInQueue() {
- return inQueue;
- }
-
- protected GmailClient rssClient;
- protected ImapGmailClient imapClient;
-
- private ExecutorService executor;
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
- return new ThreadPoolExecutor(nThreads, nThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- public GMailProvider() {
- this.config = new ComponentConfigurator<>(GMailConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
- }
-
- public GMailProvider(GMailConfiguration config) {
- this.config = config;
- }
-
- public GMailProvider(Class klass) {
- this.config = new ComponentConfigurator<>(GMailConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
- this.klass = klass;
- }
-
- public GMailProvider(GMailConfiguration config, Class klass) {
- this.config = config;
- this.klass = klass;
- }
-
- protected DatumStatusCounter countersTotal = new DatumStatusCounter();
- protected DatumStatusCounter countersCurrent = new DatumStatusCounter();
-
- @Override
- public String getId() {
- return "GMailProvider";
- }
-
- @Override
- public void startStream() {
-
- task = executor.submit(new GMailImapProviderTask(this));
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized( GMailProvider.class ) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- providerQueue.clear();
- }
-
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !task.isDone() && !task.isCancelled();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(this.klass);
-
- Objects.requireNonNull(config.getUserName());
- Objects.requireNonNull(config.getPassword());
-
- rssClient = new RssGmailClient();
- GmailConnection rssConnection = new HttpGmailConnection(config.getUserName(), config.getPassword().toCharArray());
- rssClient.setConnection(rssConnection);
-
- imapClient = new ImapGmailClient();
- GmailConnection imapConnection = new ImapGmailConnection();
- imapConnection.setLoginCredentials(config.getUserName(), config.getPassword().toCharArray());
- imapClient.setConnection(imapConnection);
-
- executor = Executors.newSingleThreadExecutor();
-
- startStream();
- }
-
- @Override
- public void cleanUp() {
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java
new file mode 100644
index 0000000..bfebe33
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gmail.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.util.ComponentUtils;
+
+import com.googlecode.gmail4j.GmailMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * GMailImapProviderTask collects Gmail via IMAP driver.
+ */
+public class GMailImapProviderTask implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(GMailImapProviderTask.class);
+
+ private GMailProvider provider;
+
+ public GMailImapProviderTask(GMailProvider provider) {
+ this.provider = provider;
+ }
+
+ @Override
+ public void run() {
+
+ final List<GmailMessage> messages = this.provider.imapClient.getUnreadMessages();
+
+ for (GmailMessage message : messages) {
+
+ Activity activity;
+ GMailMessageActivitySerializer serializer = new GMailMessageActivitySerializer( this.provider );
+ activity = serializer.deserialize(message);
+ StreamsDatum entry = new StreamsDatum(activity);
+ ComponentUtils.offerUntilSuccess(entry, this.provider.providerQueue);
+
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java
new file mode 100644
index 0000000..68b8e35
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gmail.provider;
+
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.pojo.extensions.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Generator;
+import org.apache.streams.pojo.json.Icon;
+import org.apache.streams.pojo.json.Provider;
+
+import com.fasterxml.jackson.annotation.JsonBackReference;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonManagedReference;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.googlecode.gmail4j.GmailException;
+import com.googlecode.gmail4j.GmailMessage;
+import com.googlecode.gmail4j.javamail.JavaMailGmailMessage;
+import com.sun.mail.imap.IMAPFolder;
+import com.sun.mail.imap.IMAPMessage;
+import com.sun.mail.imap.IMAPSSLStore;
+import org.apache.commons.lang.NotImplementedException;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.mail.internet.MimeMultipart;
+
+/**
+ * GMailMessageActivitySerializer converts a GMail message to Activity.
+ */
+public class GMailMessageActivitySerializer implements ActivitySerializer<GmailMessage> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GMailMessageActivitySerializer.class);
+
+ private GMailProvider provider;
+
+ public GMailMessageActivitySerializer(GMailProvider provider) {
+
+ this.provider = provider;
+
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, Boolean.FALSE);
+
+ mapper.addMixInAnnotations(IMAPSSLStore.class, MessageMixIn.class);
+ mapper.addMixInAnnotations(IMAPFolder.class, MessageMixIn.class);
+ mapper.addMixInAnnotations(IMAPMessage.class, MessageMixIn.class);
+ mapper.addMixInAnnotations(MimeMultipart.class, MessageMixIn.class);
+ mapper.addMixInAnnotations(JavaMailGmailMessage.class, MessageMixIn.class);
+
+ }
+
+ public GMailMessageActivitySerializer() {
+ }
+
+ public static Generator buildGenerator(ObjectNode event) {
+ return null;
+ }
+
+ public static Icon getIcon(ObjectNode event) {
+ return null;
+ }
+
+ public static Provider buildProvider(ObjectNode event) {
+ Provider provider = new Provider();
+ provider.setId("id:providers:gmail");
+ return provider;
+ }
+
+ public static List<Object> getLinks(ObjectNode event) {
+ return null;
+ }
+
+ public static String getUrls(ObjectNode event) {
+ return null;
+ }
+
+ public static void addGMailExtension(Activity activity, GmailMessage gmailMessage) {
+ Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
+ extensions.put("gmail", gmailMessage);
+ }
+
+ public static String formatId(String... idparts) {
+ return String.join(":",
+ Stream.concat(Arrays.stream(new String[] {"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
+ }
+
+ @Override
+ public String serializationFormat() {
+ return "gmail.v1";
+ }
+
+ @Override
+ public GmailMessage serialize(Activity activity) {
+ return null;
+ }
+
+ @Override
+ public Activity deserialize(GmailMessage gmailMessage) {
+
+ Activity activity = new Activity();
+ activity.setId(formatId(this.provider.getConfig().getUserName(), String.valueOf(gmailMessage.getMessageNumber())));
+ activity.setPublished(new DateTime(gmailMessage.getSendDate()));
+ Provider provider = new Provider();
+ provider.setId("http://gmail.com");
+ provider.setDisplayName("GMail");
+ activity.setProvider(provider);
+ ActivityObject actor = new ActivityObject();
+ actor.setId(gmailMessage.getFrom().getEmail());
+ actor.setDisplayName(gmailMessage.getFrom().getName());
+ activity.setActor(actor);
+ activity.setVerb("email");
+ ActivityObject object = new ActivityObject();
+ try {
+ object.setId(gmailMessage.getTo().get(0).getEmail());
+ object.setDisplayName(gmailMessage.getTo().get(0).getName());
+ } catch (GmailException e) {
+ LOGGER.warn(e.getMessage());
+ }
+ activity.setTitle(gmailMessage.getSubject());
+ try {
+ activity.setContent(gmailMessage.getContentText());
+ } catch (GmailException e) {
+ LOGGER.warn(e.getMessage());
+ }
+ activity.setObject(object);
+ return activity;
+ }
+
+ @Override
+ public List<Activity> deserializeAll(List<GmailMessage> serializedList) {
+ throw new NotImplementedException("Not currently implemented");
+ }
+
+ public Activity convert(ObjectNode event) {
+ return null;
+ }
+
+ interface MessageMixIn {
+ @JsonManagedReference
+ @JsonIgnore
+ IMAPSSLStore getDefaultFolder(); // we don't need it!
+
+ @JsonManagedReference
+ @JsonIgnore
+ IMAPSSLStore getPersonalNamespaces(); // we don't need it!
+
+ @JsonManagedReference
+ @JsonIgnore
+ IMAPFolder getStore(); // we don't need it!
+
+// @JsonManagedReference
+// @JsonIgnore
+// @JsonBackReference
+ //IMAPFolder getParent(); // we don't need it!
+ @JsonManagedReference
+ @JsonIgnore
+ @JsonBackReference
+ IMAPMessage getFolder(); // we don't need it!
+
+ @JsonManagedReference
+ @JsonIgnore
+ @JsonProperty("parent")
+ @JsonBackReference
+ MimeMultipart getParent();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java
new file mode 100644
index 0000000..06c73fc
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gmail.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.DatumStatusCounter;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.gmail.GMailConfiguration;
+
+import com.googlecode.gmail4j.GmailClient;
+import com.googlecode.gmail4j.GmailConnection;
+import com.googlecode.gmail4j.http.HttpGmailConnection;
+import com.googlecode.gmail4j.javamail.ImapGmailClient;
+import com.googlecode.gmail4j.javamail.ImapGmailConnection;
+import com.googlecode.gmail4j.rss.RssGmailClient;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.util.Objects;
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * GMailProvider collects messages from GMail.
+ */
+public class GMailProvider implements StreamsProvider, Serializable {
+
+ public final static String STREAMS_ID = "GMailProvider";
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(GMailProvider.class);
+ protected BlockingQueue inQueue = new LinkedBlockingQueue<>(10000);
+ protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
+ protected Future task;
+ protected GmailClient rssClient;
+ protected ImapGmailClient imapClient;
+ protected DatumStatusCounter countersTotal = new DatumStatusCounter();
+ protected DatumStatusCounter countersCurrent = new DatumStatusCounter();
+ private GMailConfiguration config;
+ private Class klass;
+ private ExecutorService executor;
+
+ public GMailProvider() {
+ this.config = new ComponentConfigurator<>(GMailConfiguration.class)
+ .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
+ }
+
+ public GMailProvider(GMailConfiguration config) {
+ this.config = config;
+ }
+
+ public GMailProvider(Class klass) {
+ this.config = new ComponentConfigurator<>(GMailConfiguration.class)
+ .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
+ this.klass = klass;
+ }
+
+ public GMailProvider(GMailConfiguration config, Class klass) {
+ this.config = config;
+ this.klass = klass;
+ }
+
+ private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
+ return new ThreadPoolExecutor(nThreads, nThreads,
+ 5000L, TimeUnit.MILLISECONDS,
+ new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
+ }
+
+ public GMailConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(GMailConfiguration config) {
+ this.config = config;
+ }
+
+ public BlockingQueue<Object> getInQueue() {
+ return inQueue;
+ }
+
+ @Override
+ public String getId() {
+ return "GMailProvider";
+ }
+
+ @Override
+ public void startStream() {
+
+ task = executor.submit(new GMailImapProviderTask(this));
+
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+
+ StreamsResultSet current;
+
+ synchronized ( GMailProvider.class ) {
+ current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
+ current.setCounter(new DatumStatusCounter());
+ current.getCounter().add(countersCurrent);
+ countersTotal.add(countersCurrent);
+ countersCurrent = new DatumStatusCounter();
+ providerQueue.clear();
+ }
+
+ return current;
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return !task.isDone() && !task.isCancelled();
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ Objects.requireNonNull(this.klass);
+ Objects.requireNonNull(config.getUserName());
+ Objects.requireNonNull(config.getPassword());
+
+ rssClient = new RssGmailClient();
+ GmailConnection rssConnection = new HttpGmailConnection(config.getUserName(), config.getPassword().toCharArray());
+ rssClient.setConnection(rssConnection);
+
+ imapClient = new ImapGmailClient();
+ GmailConnection imapConnection = new ImapGmailConnection();
+ imapConnection.setLoginCredentials(config.getUserName(), config.getPassword().toCharArray());
+ imapClient.setConnection(imapConnection);
+
+ executor = Executors.newSingleThreadExecutor();
+
+ startStream();
+ }
+
+ @Override
+ public void cleanUp() {
+ try {
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/com/google/gmail/GMailConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/com/google/gmail/GMailConfiguration.json b/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/com/google/gmail/GMailConfiguration.json
deleted file mode 100644
index 2a6a468..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/com/google/gmail/GMailConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "com.google.gmail.GMailConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "userName": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json b/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json
new file mode 100644
index 0000000..614b290
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.gmail.GMailConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "userName": {
+ "type": "string"
+ },
+ "password": {
+ "type": "string"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/test/java/com/google/gmail/test/GMailMessageSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/test/java/com/google/gmail/test/GMailMessageSerDeTest.java b/streams-contrib/streams-provider-google/google-gmail/src/test/java/com/google/gmail/test/GMailMessageSerDeTest.java
deleted file mode 100644
index 3a611f7..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/test/java/com/google/gmail/test/GMailMessageSerDeTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gmail.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of gmail inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class GMailMessageSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailMessageSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Ignore
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = GMailMessageSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- // implement
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java b/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java
new file mode 100644
index 0000000..8d3da9e
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gmail.test;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Tests conversion of gmail inputs to Activity
+ */
+@Ignore("ignore until test resources are available.")
+public class GMailMessageSerDeTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(GMailMessageSerDeTest.class);
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ @Ignore
+ @Test
+ public void Tests()
+ {
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
+
+ InputStream is = GMailMessageSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ LOGGER.debug(line);
+
+ // implement
+ }
+ } catch( Exception e ) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
deleted file mode 100644
index 0b84932..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusCommentProcessor.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.api.services.plus.model.Comment;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusCommentProcessor collects comments about a google plus activity.
- */
-public class GooglePlusCommentProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "GooglePlusCommentProcessor";
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentProcessor.class);
- private GooglePlusActivityUtil googlePlusActivityUtil;
- private int count;
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
-
- //Get G+ activity ID from our own activity ID
- if (item instanceof Activity) {
- Activity activity = (Activity) item;
- String activityId = getGPlusID(activity.getId());
-
- //Call Google Plus API to get list of comments for this activity ID
- /* TODO: FILL ME OUT WITH THE API CALL **/
- List<Comment> comments = new ArrayList<>();
-
- GooglePlusActivityUtil.updateActivity(comments, activity);
- result = new StreamsDatum(activity);
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.error("Exception while converting Comment to Activity: {}", ex.getMessage());
- }
-
- if ( result != null ) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- googlePlusActivityUtil = new GooglePlusActivityUtil();
- count = 0;
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- private String getGPlusID(String activityId) {
- String[] activityParts = activityId.split(":");
- return (activityParts.length > 0) ? activityParts[activityParts.length - 1] : "";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
deleted file mode 100644
index 51a6d51..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/processor/GooglePlusTypeConverter.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import com.google.gplus.serializer.util.GPlusActivityDeserializer;
-import com.google.gplus.serializer.util.GPlusEventClassifier;
-import com.google.gplus.serializer.util.GPlusPersonDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusTypeConverter is a StreamsProcessor that converts gplus activities to activitystreams activities.
- */
-public class GooglePlusTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "GooglePlusTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverter.class);
- private StreamsJacksonMapper mapper;
- private Queue<Person> inQueue;
- private Queue<StreamsDatum> outQueue;
- private int count = 0;
-
- public GooglePlusTypeConverter() {}
-
- public Queue<StreamsDatum> getProcessorOutputQueue() {
- return outQueue;
- }
-
- public void setProcessorInputQueue(Queue<Person> inputQueue) {
- inQueue = inputQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- Activity activity = null;
-
- if (item instanceof String) {
- item = deserializeItem(item);
- }
-
- if (item instanceof Person) {
- activity = new Activity();
- GooglePlusActivityUtil.updateActivity((Person)item, activity);
- } else if (item instanceof com.google.api.services.plus.model.Activity) {
- activity = new Activity();
- GooglePlusActivityUtil.updateActivity((com.google.api.services.plus.model.Activity)item, activity);
- }
-
- if (activity != null) {
- result = new StreamsDatum(activity);
- count++;
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.error("Exception while converting Person to Activity: {}", ex.getMessage());
- }
-
- if ( result != null ) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- private Object deserializeItem(Object item) {
- try {
- Class klass = GPlusEventClassifier.detectClass((String) item);
-
- if (klass.equals(Person.class)) {
- item = mapper.readValue((String) item, Person.class);
- } else if (klass.equals(com.google.api.services.plus.model.Activity.class)) {
- item = mapper.readValue((String) item, com.google.api.services.plus.model.Activity.class);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserializeItem: {}", ex);
- }
-
- return item;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- mapper.registerModule(simpleModule);
-
- simpleModule = new SimpleModule();
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- mapper.registerModule(simpleModule);
- }
-
- @Override
- public void cleanUp() {
- //No-op
- }
-}
[2/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java
new file mode 100644
index 0000000..d4719ce
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.services.youtube.YouTube;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Retrieve recent activity from a list of user ids or names.
+ */
+public class YoutubeUserActivityProvider extends YoutubeProvider {
+
+ public YoutubeUserActivityProvider() {
+ super();
+ }
+
+ public YoutubeUserActivityProvider(YoutubeConfiguration config) {
+ super(config);
+ }
+
+ /**
+ * To use from command line:
+ * <p>
+ * <p/>
+ * Supply (at least) the following required configuration in application.conf:
+ * <p>
+ * <p/>
+ * youtube.oauth.pathToP12KeyFile
+ * youtube.oauth.serviceAccountEmailAddress
+ * youtube.apiKey
+ * youtube.youtubeUsers
+ * <p>
+ * <p/>
+ * Launch using:
+ * <p>
+ * <p/>
+ * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
+ *
+ * @param args args
+ * @throws Exception Exception
+ */
+ public static void main(String[] args) throws Exception {
+
+ Preconditions.checkArgument(args.length >= 2);
+
+ String configfile = args[0];
+ String outfile = args[1];
+
+ Config reference = ConfigFactory.load();
+ File file = new File(configfile);
+ assert (file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
+
+ Config typesafe = testResourceConfig.withFallback(reference).resolve();
+
+ StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
+ YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
+ YoutubeUserActivityProvider provider = new YoutubeUserActivityProvider(config);
+
+ ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
+ provider.prepare(config);
+ provider.startStream();
+ do {
+ Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
+ Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
+ while (iterator.hasNext()) {
+ StreamsDatum datum = iterator.next();
+ String json;
+ try {
+ if (datum.getDocument() instanceof String) {
+ json = (String) datum.getDocument();
+ } else {
+ json = mapper.writeValueAsString(datum.getDocument());
+ }
+ outStream.println(json);
+ } catch (JsonProcessingException ex) {
+ System.err.println(ex.getMessage());
+ }
+ }
+ }
+ while (provider.isRunning());
+ provider.cleanUp();
+ outStream.flush();
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
+ return new YoutubeUserActivityCollector(youtube, queue, strategy, userInfo, config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java
new file mode 100644
index 0000000..4b985f6
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.pojo.extensions.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Image;
+import org.apache.streams.pojo.json.Provider;
+
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.Thumbnail;
+import com.google.api.services.youtube.model.ThumbnailDetails;
+import com.google.api.services.youtube.model.Video;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class YoutubeActivityUtil {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeActivityUtil.class);
+
+ /**
+ * Given a {@link YouTube.Videos} object and an
+ * {@link Activity} object, fill out the appropriate details
+ *
+ * @param video Video
+ * @param activity Activity
+ * @throws ActivitySerializerException ActivitySerializerException
+ */
+ public static void updateActivity(Video video, Activity activity, String channelId) throws ActivitySerializerException {
+ activity.setActor(buildActor(video, video.getSnippet().getChannelId()));
+ activity.setVerb("post");
+
+ activity.setId(formatId(activity.getVerb(), Optional.ofNullable(video.getId()).orElse(null)));
+
+ activity.setPublished(new DateTime(video.getSnippet().getPublishedAt().getValue()));
+ activity.setTitle(video.getSnippet().getTitle());
+ activity.setContent(video.getSnippet().getDescription());
+ activity.setUrl("https://www.youtube.com/watch?v=" + video.getId());
+
+ activity.setProvider(getProvider());
+
+ activity.setObject(buildActivityObject(video));
+
+ addYoutubeExtensions(activity, video);
+ }
+
+
+ /**
+ * Given a {@link Channel} object and an
+ * {@link Activity} object, fill out the appropriate details
+ *
+ * @param channel Channel
+ * @param activity Activity
+ * @throws ActivitySerializerException ActivitySerializerException
+ */
+ public static void updateActivity(Channel channel, Activity activity, String channelId) throws ActivitySerializerException {
+ try {
+ activity.setProvider(getProvider());
+ activity.setVerb("post");
+ activity.setActor(createActorForChannel(channel));
+ Map<String, Object> extensions = new HashMap<>();
+ extensions.put("youtube", channel);
+ activity.setAdditionalProperty("extensions", extensions);
+ } catch (Throwable throwable) {
+ throw new ActivitySerializerException(throwable);
+ }
+ }
+
+ /**
+ * createActorForChannel.
+ * @param channel Channel
+ * @return $.actor
+ */
+ public static ActivityObject createActorForChannel(Channel channel) {
+ ActivityObject actor = new ActivityObject();
+ // TODO: use generic provider id concatenator
+ actor.setId("id:youtube:" + channel.getId());
+ actor.setSummary(channel.getSnippet().getDescription());
+ actor.setDisplayName(channel.getSnippet().getTitle());
+ Image image = new Image();
+ image.setUrl(channel.getSnippet().getThumbnails().getHigh().getUrl());
+ actor.setImage(image);
+ actor.setUrl("https://youtube.com/user/" + channel.getId());
+ Map<String, Object> actorExtensions = new HashMap<>();
+ actorExtensions.put("followers", channel.getStatistics().getSubscriberCount());
+ actorExtensions.put("posts", channel.getStatistics().getVideoCount());
+ actor.setAdditionalProperty("extensions", actorExtensions);
+ return actor;
+ }
+
+ /**
+ * Given a video object, create the appropriate activity object with a valid image
+ * (thumbnail) and video URL.
+ * @param video Video
+ * @return Activity Object with Video URL and a thumbnail image
+ */
+ private static ActivityObject buildActivityObject(Video video) {
+ ActivityObject activityObject = new ActivityObject();
+
+ ThumbnailDetails thumbnailDetails = video.getSnippet().getThumbnails();
+ Thumbnail thumbnail = thumbnailDetails.getDefault();
+
+ if (thumbnail != null) {
+ Image image = new Image();
+ image.setUrl(thumbnail.getUrl());
+ image.setHeight(thumbnail.getHeight());
+ image.setWidth(thumbnail.getWidth());
+
+ activityObject.setImage(image);
+ }
+
+ activityObject.setUrl("https://www.youtube.com/watch?v=" + video.getId());
+ activityObject.setObjectType("video");
+
+ return activityObject;
+ }
+
+ /**
+ * Add the Youtube extensions to the Activity object that we're building.
+ * @param activity Activity
+ * @param video Video
+ */
+ private static void addYoutubeExtensions(Activity activity, Video video) {
+ Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
+
+ extensions.put("youtube", video);
+
+ if (video.getStatistics() != null) {
+ Map<String, Object> likes = new HashMap<>();
+ likes.put("count", video.getStatistics().getCommentCount());
+ extensions.put("likes", likes);
+ }
+ }
+
+ /**
+ * Build an {@link ActivityObject} actor given the video object
+ * @param video Video
+ * @param id id
+ * @return Actor object
+ */
+ private static ActivityObject buildActor(Video video, String id) {
+ ActivityObject actor = new ActivityObject();
+
+ actor.setId("id:youtube:" + id);
+ actor.setDisplayName(video.getSnippet().getChannelTitle());
+ actor.setSummary(video.getSnippet().getDescription());
+ actor.setAdditionalProperty("handle", video.getSnippet().getChannelTitle());
+
+ return actor;
+ }
+
+ /**
+ * Gets the common youtube {@link Provider} object
+ * @return a provider object representing YouTube
+ */
+ public static Provider getProvider() {
+ Provider provider = new Provider();
+ provider.setId("id:providers:youtube");
+ provider.setDisplayName("YouTube");
+ return provider;
+ }
+
+ /**
+ * Formats the ID to conform with the Apache Streams activity ID convention
+ * @param idparts the parts of the ID to join
+ * @return a valid Activity ID in format "id:youtube:part1:part2:...partN"
+ */
+ public static String formatId(String... idparts) {
+ return String.join(":",
+ Stream.concat(Arrays.stream(new String[] {"id:youtube"}), Arrays.stream(idparts)).collect(Collectors.toList()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java
new file mode 100644
index 0000000..e5d0ed0
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.api.client.util.DateTime;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.ChannelContentDetails;
+import com.google.api.services.youtube.model.ChannelLocalization;
+import com.google.api.services.youtube.model.ChannelSnippet;
+import com.google.api.services.youtube.model.ChannelStatistics;
+import com.google.api.services.youtube.model.ChannelTopicDetails;
+import com.google.api.services.youtube.model.Thumbnail;
+import com.google.api.services.youtube.model.ThumbnailDetails;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * YoutubeChannelDeserializer is a JsonDeserializer for Channel.
+ */
+public class YoutubeChannelDeserializer extends JsonDeserializer<Channel> {
+
+ @Override
+ public Channel deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
+ JsonNode node = jp.getCodec().readTree(jp);
+ try {
+ Channel channel = new Channel();
+ if (node.findPath("etag") != null) {
+ channel.setEtag(node.get("etag").asText());
+ }
+ if (node.findPath("kind") != null) {
+ channel.setKind(node.get("kind").asText());
+ }
+ channel.setId(node.get("id").asText());
+ channel.setTopicDetails(setTopicDetails(node.findValue("topicDetails")));
+ channel.setStatistics(setChannelStatistics(node.findValue("statistics")));
+ channel.setContentDetails(setContentDetails(node.findValue("contentDetails")));
+ channel.setSnippet(setChannelSnippet(node.findValue("snippet")));
+ return channel;
+ } catch (Throwable throwable) {
+ throw new IOException(throwable);
+ }
+ }
+
+ protected ChannelSnippet setChannelSnippet(JsonNode node) {
+ ChannelSnippet snippet = new ChannelSnippet();
+ snippet.setTitle(node.get("title").asText());
+ snippet.setDescription(node.get("description").asText());
+ snippet.setPublishedAt(new DateTime(node.get("publishedAt").get("value").longValue()));
+ snippet.setLocalized(setLocalized(node.findValue("localized")));
+ snippet.setThumbnails(setThumbnails(node.findValue("thumbnails")));
+ return snippet;
+ }
+
+ protected ThumbnailDetails setThumbnails(JsonNode node) {
+ ThumbnailDetails details = new ThumbnailDetails();
+ if (node == null) {
+ return details;
+ }
+ details.setDefault(new Thumbnail().setUrl(node.get("default").get("url").asText()));
+ details.setHigh(new Thumbnail().setUrl(node.get("high").get("url").asText()));
+ details.setMedium(new Thumbnail().setUrl(node.get("medium").get("url").asText()));
+ return details;
+ }
+
+ protected ChannelLocalization setLocalized(JsonNode node) {
+ if (node == null) {
+ return new ChannelLocalization();
+ }
+ ChannelLocalization localization = new ChannelLocalization();
+ localization.setDescription(node.get("description").asText());
+ localization.setTitle(node.get("title").asText());
+ return localization;
+ }
+
+ protected ChannelContentDetails setContentDetails(JsonNode node) {
+ ChannelContentDetails contentDetails = new ChannelContentDetails();
+ if (node == null) {
+ return contentDetails;
+ }
+ if (node.findValue("googlePlusUserId") != null) {
+ contentDetails.setGooglePlusUserId(node.get("googlePlusUserId").asText());
+ }
+ contentDetails.setRelatedPlaylists(setRelatedPlaylists(node.findValue("relatedPlaylists")));
+ return contentDetails;
+ }
+
+ protected ChannelContentDetails.RelatedPlaylists setRelatedPlaylists(JsonNode node) {
+ ChannelContentDetails.RelatedPlaylists playlists = new ChannelContentDetails.RelatedPlaylists();
+ if (node == null) {
+ return playlists;
+ }
+ if (node.findValue("favorites") != null) {
+ playlists.setFavorites(node.get("favorites").asText());
+ }
+ if (node.findValue("likes") != null) {
+ playlists.setLikes(node.get("likes").asText());
+ }
+ if (node.findValue("uploads") != null) {
+ playlists.setUploads(node.get("uploads").asText());
+ }
+ return playlists;
+ }
+
+ protected ChannelStatistics setChannelStatistics(JsonNode node) {
+ ChannelStatistics stats = new ChannelStatistics();
+ if (node == null) {
+ return stats;
+ }
+ stats.setCommentCount(node.get("commentCount").bigIntegerValue());
+ stats.setHiddenSubscriberCount(node.get("hiddenSubscriberCount").asBoolean());
+ stats.setSubscriberCount(node.get("subscriberCount").bigIntegerValue());
+ stats.setVideoCount(node.get("videoCount").bigIntegerValue());
+ stats.setViewCount(node.get("viewCount").bigIntegerValue());
+ return stats;
+ }
+
+ protected ChannelTopicDetails setTopicDetails(JsonNode node) {
+ ChannelTopicDetails details = new ChannelTopicDetails();
+ if (node == null) {
+ return details;
+ }
+ List<String> topicIds = new LinkedList<>();
+ for (JsonNode jsonNode : node.get("topicIds")) {
+ topicIds.add(jsonNode.asText());
+ }
+ details.setTopicIds(topicIds);
+ return details;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java
new file mode 100644
index 0000000..964d6ca
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.api.services.youtube.model.Video;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class YoutubeEventClassifier {
+ private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+ private static final String VIDEO_IDENTIFIER = "\"youtube#video\"";
+ private static final String CHANNEL_IDENTIFIER = "youtube#channel";
+
+ /**
+ * detect probable Class of a json String from YouTube.
+ * @param json json
+ * @return Class
+ */
+ public static Class detectClass(String json) {
+ Objects.requireNonNull(json);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(json));
+
+ ObjectNode objectNode;
+ try {
+ objectNode = (ObjectNode) mapper.readTree(json);
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ return null;
+ }
+
+ if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(VIDEO_IDENTIFIER)) {
+ return Video.class;
+ } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().contains(CHANNEL_IDENTIFIER)) {
+ return com.google.api.services.youtube.model.Channel.class;
+ } else {
+ return ObjectNode.class;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java
new file mode 100644
index 0000000..24f8e51
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.serializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.api.client.util.DateTime;
+import com.google.api.services.youtube.model.Thumbnail;
+import com.google.api.services.youtube.model.ThumbnailDetails;
+import com.google.api.services.youtube.model.Video;
+import com.google.api.services.youtube.model.VideoSnippet;
+import com.google.api.services.youtube.model.VideoStatistics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class YoutubeVideoDeserializer extends JsonDeserializer<Video> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoDeserializer.class);
+
+ /**
+ * Because the Youtube Video object contains complex objects within its hierarchy, we have to use
+ * a custom deserializer
+ *
+ * @param jsonParser jsonParser
+ * @param deserializationContext deserializationContext
+ * @return The deserialized {@link com.google.api.services.youtube.YouTube.Videos} object
+ * @throws java.io.IOException IOException
+ * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
+ */
+ @Override
+ public Video deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Video video = new Video();
+
+ try {
+ video.setId(node.get("id").asText());
+ video.setEtag(node.get("etag").asText());
+ video.setKind(node.get("kind").asText());
+
+ video.setSnippet(buildSnippet(node));
+ video.setStatistics(buildStatistics(node));
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserialize a Video object: {}", ex);
+ }
+
+ return video;
+ }
+
+ /**
+ * Given the raw JsonNode, construct a video snippet object.
+ * @param node JsonNode
+ * @return VideoSnippet
+ */
+ private VideoSnippet buildSnippet(JsonNode node) {
+ VideoSnippet snippet = new VideoSnippet();
+ JsonNode snippetNode = node.get("snippet");
+
+ snippet.setChannelId(snippetNode.get("channelId").asText());
+ snippet.setChannelTitle(snippetNode.get("channelTitle").asText());
+ snippet.setDescription(snippetNode.get("description").asText());
+ snippet.setTitle(snippetNode.get("title").asText());
+ snippet.setPublishedAt(new DateTime(snippetNode.get("publishedAt").get("value").asLong()));
+
+ ThumbnailDetails thumbnailDetails = new ThumbnailDetails();
+ for (JsonNode t : snippetNode.get("thumbnails")) {
+ Thumbnail thumbnail = new Thumbnail();
+
+ thumbnail.setHeight(t.get("height").asLong());
+ thumbnail.setUrl(t.get("url").asText());
+ thumbnail.setWidth(t.get("width").asLong());
+
+ thumbnailDetails.setDefault(thumbnail);
+ }
+
+ snippet.setThumbnails(thumbnailDetails);
+
+ return snippet;
+ }
+
+ /**
+ * Given the raw JsonNode, construct a statistics object.
+ * @param node JsonNode
+ * @return VideoStatistics
+ */
+ private VideoStatistics buildStatistics(JsonNode node) {
+ VideoStatistics statistics = new VideoStatistics();
+ JsonNode statisticsNode = node.get("statistics");
+
+ statistics.setCommentCount(statisticsNode.get("commentCount").bigIntegerValue());
+ statistics.setDislikeCount(statisticsNode.get("dislikeCount").bigIntegerValue());
+ statistics.setFavoriteCount(statisticsNode.get("favoriteCount").bigIntegerValue());
+ statistics.setLikeCount(statisticsNode.get("likeCount").bigIntegerValue());
+ statistics.setViewCount(statisticsNode.get("viewCount").bigIntegerValue());
+
+ return statistics;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/jsonschema/com/youtube/YoutubeConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/jsonschema/com/youtube/YoutubeConfiguration.json b/streams-contrib/streams-provider-youtube/src/main/jsonschema/com/youtube/YoutubeConfiguration.json
deleted file mode 100644
index 9b57394..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/jsonschema/com/youtube/YoutubeConfiguration.json
+++ /dev/null
@@ -1,105 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.youtube.pojo.YoutubeConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "apiKey": {
- "type": "string",
- "description": "API key to allow for authenticated (but not owned) requests"
- },
- "follow": {
- "type": "array",
- "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "youtubeUsers": {
- "type": "array",
- "description": "A list of user user ids and optional date parameters for the GPlus provider",
- "items": {
- "type": "object",
- "$ref": "#/definitions/userInfo"
- }
- },
- "defaultAfterDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
- },
- "defaultBeforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "DEPRICATED",
- "properties": {
- "appName": {
- "type": "string"
- },
- "pathToP12KeyFile": {
- "type": "string",
- "description": "Absolute Path to key file"
- },
- "serviceAccountEmailAddress": {
- "type": "string",
- "description": "Service Account email address for your app"
- }
- }
- }
- },
- "definitions": {
- "userInfo": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "dynamic": "true",
- "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
- "properties": {
- "userId": {
- "type": "string",
- "description": "Google+ user id"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json b/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json
new file mode 100644
index 0000000..7e80731
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json
@@ -0,0 +1,105 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.youtube.YoutubeConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "The protocol"
+ },
+ "host": {
+ "type": "string",
+ "description": "The host"
+ },
+ "port": {
+ "type": "integer",
+ "description": "The port"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version"
+ },
+ "endpoint": {
+ "type": "string",
+ "description": "The endpoint"
+ },
+ "apiKey": {
+ "type": "string",
+ "description": "API key to allow for authenticated (but not owned) requests"
+ },
+ "follow": {
+ "type": "array",
+ "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ },
+ "youtubeUsers": {
+ "type": "array",
+ "description": "A list of user user ids and optional date parameters for the GPlus provider",
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/userInfo"
+ }
+ },
+ "defaultAfterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
+ },
+ "defaultBeforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
+ },
+ "oauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "description": "DEPRICATED",
+ "properties": {
+ "appName": {
+ "type": "string"
+ },
+ "pathToP12KeyFile": {
+ "type": "string",
+ "description": "Absolute Path to key file"
+ },
+ "serviceAccountEmailAddress": {
+ "type": "string",
+ "description": "Service Account email address for your app"
+ }
+ }
+ }
+ },
+ "definitions": {
+ "userInfo": {
+ "type": "object",
+ "javaInterfaces" : ["java.io.Serializable"],
+ "dynamic": "true",
+ "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
+ "properties": {
+ "userId": {
+ "type": "string",
+ "description": "Google+ user id"
+ },
+ "afterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
+ },
+ "beforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/processor/YoutubeTypeConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/processor/YoutubeTypeConverterTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/processor/YoutubeTypeConverterTest.java
deleted file mode 100644
index 469b8d0..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/processor/YoutubeTypeConverterTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Video;
-import com.youtube.serializer.YoutubeActivityUtil;
-import com.youtube.serializer.YoutubeVideoDeserializer;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for YoutubeTypeConverter.
- */
-public class YoutubeTypeConverterTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverterTest.class);
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
-
- private YoutubeTypeConverter youtubeTypeConverter;
- private ObjectMapper objectMapper;
-
- /**
- * setup for test.
- */
- @Before
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- youtubeTypeConverter = new YoutubeTypeConverter();
- youtubeTypeConverter.prepare(null);
- }
-
- @Test
- public void testVideoConversion() {
- try {
- LOGGER.info("raw: {}", testVideo);
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
- StreamsDatum streamsDatum = new StreamsDatum(video);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
- YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
-
- assertEquals(retList.size(), 1);
- assert (retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to convert video to activity: {}", ex);
- }
- }
-
- @Test
- public void testStringVideoConversion() {
- try {
- LOGGER.info("raw: {}", testVideo);
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
- StreamsDatum streamsDatum = new StreamsDatum(testVideo);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
- YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
-
- assertEquals(retList.size(), 1);
- assert (retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to convert video to activity: {}", ex);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeChannelDataCollectorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeChannelDataCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeChannelDataCollectorTest.java
deleted file mode 100644
index 0ae4822..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeChannelDataCollectorTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.LinearTimeBackOffStrategy;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.ChannelListResponse;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.junit.Test;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * YoutubeChannelDataCollectorTest tests YoutubeChannelDataCollector.
- */
-public class YoutubeChannelDataCollectorTest {
-
- private static final String ID = "12345";
-
- @Test
- public void testDataCollector() throws Exception {
- YouTube youTube = createMockYoutube();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- BackOffStrategy strategy = new LinearTimeBackOffStrategy(1);
- UserInfo userInfo = new UserInfo();
- userInfo.setUserId(ID);
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey(ID);
- YoutubeChannelDataCollector collector = new YoutubeChannelDataCollector(youTube, queue, strategy, userInfo, config);
- collector.run();
- assertEquals(1, queue.size());
- StreamsDatum datum = queue.take();
- assertNotNull(datum);
- String document = (String) datum.getDocument();
- assertNotNull(document);
- }
-
- private YouTube createMockYoutube() throws Exception {
- YouTube mockYouTube = mock(YouTube.class);
- YouTube.Channels channels = createMockChannels();
- when(mockYouTube.channels()).thenReturn(channels);
- return mockYouTube;
- }
-
- private YouTube.Channels createMockChannels() throws Exception {
- YouTube.Channels mockChannels = mock(YouTube.Channels.class);
- YouTube.Channels.List channelLists = createMockChannelsList();
- when(mockChannels.list(anyString())).thenReturn(channelLists);
- return mockChannels;
- }
-
- private YouTube.Channels.List createMockChannelsList() throws Exception {
- YouTube.Channels.List mockList = mock(YouTube.Channels.List.class);
- when(mockList.setId(anyString())).thenReturn(mockList);
- when(mockList.setKey(anyString())).thenReturn(mockList);
- ChannelListResponse response = createMockResponse();
- when(mockList.execute()).thenReturn(response);
- return mockList;
- }
-
- private ChannelListResponse createMockResponse() {
- ChannelListResponse response = new ChannelListResponse();
- List<Channel> channelList = new LinkedList<>();
- response.setItems(channelList);
- Channel channel = new Channel();
- channel.setId(ID);
- channelList.add(channel);
- return response;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeProviderTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeProviderTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeProviderTest.java
deleted file mode 100644
index 121c3d5..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeProviderTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.services.youtube.YouTube;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.joda.time.DateTime;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-
-/**
- * Test for YoutubeProvider.
- */
-public class YoutubeProviderTest {
-
- /**
- * Test that every collector will be run and that data queued from the collectors will be processed.
- */
- @Test
- public void testDataCollectorRunsPerUser() {
- Random random = new Random(System.currentTimeMillis());
- int numUsers = random.nextInt(1000);
- List<UserInfo> userList = new LinkedList<>();
-
- for ( int i = 0; i < numUsers; ++i ) {
- userList.add(new UserInfo());
- }
-
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setYoutubeUsers(userList);
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- try {
- provider.prepare(null);
- provider.startStream();
- int datumCount = 0;
- while (provider.isRunning()) {
- datumCount += provider.readCurrent().size();
- }
- assertEquals(numUsers, datumCount);
- } finally {
- provider.cleanUp();
- }
- }
-
- @Test
- public void testConfigSetterGetter() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- config.setVersion("fake_version_1");
- YoutubeConfiguration newConfig = new YoutubeConfiguration();
- newConfig.setApiKey("API_KEY");
- config.setVersion("fake_version_2");
-
- YoutubeProvider provider = buildProvider(config);
-
- assertEquals(provider.getConfig(), config);
-
- provider.setConfig(newConfig);
- assertEquals(provider.getConfig(), newConfig);
- }
-
- @Test
- public void testUserInfoWithDefaultDates() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- DateTime afterDate = new DateTime(System.currentTimeMillis());
- DateTime beforeDate = afterDate.minus(10000);
-
- provider.setDefaultAfterDate(afterDate);
- provider.setDefaultBeforeDate(beforeDate);
-
- Set<String> users = new HashSet<>();
- users.add("test_user_1");
- users.add("test_user_2");
- users.add("test_user_3");
-
- provider.setUserInfoWithDefaultDates(users);
-
- List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
-
- for (UserInfo user : youtubeUsers) {
- assert (user.getAfterDate().equals(afterDate));
- assert (user.getBeforeDate().equals(beforeDate));
- }
- }
-
- @Test
- public void testUserInfoWithAfterDate() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- Map<String, DateTime> users = new HashMap<>();
- users.put("user1", new DateTime(System.currentTimeMillis()));
- users.put("user3", new DateTime(System.currentTimeMillis()));
- users.put("user4", new DateTime(System.currentTimeMillis()));
-
- provider.setUserInfoWithAfterDate(users);
-
- List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
-
- for (UserInfo user : youtubeUsers) {
- assert (user.getAfterDate().equals(users.get(user.getUserId())));
- }
- }
-
- private YoutubeProvider buildProvider(YoutubeConfiguration config) {
- return new YoutubeProvider(config) {
-
- @Override
- protected YouTube createYouTubeClient() throws IOException {
- return mock(YouTube.class);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- final BlockingQueue<StreamsDatum> q = queue;
- return () -> {
- try {
- q.put(new StreamsDatum(null));
- } catch (InterruptedException ie) {
- fail("Test was interrupted");
- }
- };
- }
- };
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeUserActivityCollectorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeUserActivityCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeUserActivityCollectorTest.java
deleted file mode 100644
index d6a540e..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/provider/YoutubeUserActivityCollectorTest.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.local.queues.ThroughputQueue;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Activity;
-import com.google.api.services.youtube.model.ActivityContentDetails;
-import com.google.api.services.youtube.model.ActivityContentDetailsUpload;
-import com.google.api.services.youtube.model.ActivityListResponse;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoListResponse;
-import com.google.api.services.youtube.model.VideoSnippet;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.joda.time.DateTime;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.anyLong;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test for YoutubeUserActivityCollector.
- */
-public class YoutubeUserActivityCollectorTest {
- private static final String USER_ID = "fake_user_id";
- private static final String IN_RANGE_IDENTIFIER = "data in range";
- private YoutubeConfiguration config;
-
- @Before
- public void setup() {
- this.config = new YoutubeConfiguration();
- this.config.setApiKey("API_KEY");
- }
-
- @Test
- public void testGetVideos() throws IOException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 1, 0, now, now.minus(10000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- List<Video> video = collector.getVideoList("fake_video_id");
-
- assertNotNull(video.get(0));
- }
-
- @Test
- public void testGetVideosNull() throws IOException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 0, 0, now.plus(10000), now.minus(10000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- List<Video> video = collector.getVideoList("fake_video_id");
-
- assertEquals(video.size(), 0);
- }
-
- @Test
- public void testProcessActivityFeed() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 0, 5, now.plus(3000000), now.minus(1000000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedBefore() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 0, 0, now, now);
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
-
- assertEquals(collector.getDatumQueue().size(), 0);
- }
-
- @Test
- public void testProcessActivityFeedAfter() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 5, 0, now, now);
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now.getMillis() - 100000), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedMismatchCount() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedMismatchCountInRange() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now), new DateTime(now).minus(100000));
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- private ActivityListResponse buildActivityListResponse(int num) {
- ActivityListResponse activityListResponse = new ActivityListResponse();
- List<Activity> items = new ArrayList<>();
-
- for ( int x = 0; x < num; x++ ) {
- Activity activity = new Activity();
-
- ActivityContentDetails contentDetails = new ActivityContentDetails();
- ActivityContentDetailsUpload upload = new ActivityContentDetailsUpload();
- upload.setVideoId("video_id_" + x);
- contentDetails.setUpload(upload);
-
- activity.setId("id_" + x);
- activity.setContentDetails(contentDetails);
-
- items.add(activity);
- }
-
- activityListResponse.setItems(items);
-
- return activityListResponse;
- }
-
- private YouTube buildYouTube(int numBeforeRange, int numAfterRange, int numInRange, DateTime afterDate, DateTime beforeDate) {
-
- return createYoutubeMock(numBeforeRange, numAfterRange, numInRange, afterDate, beforeDate);
-
- }
-
- private YouTube createYoutubeMock(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube youtube = mock(YouTube.class);
-
- final YouTube.Videos videos = createMockVideos(numBefore, numAfter, numInRange, after, before);
- doAnswer(invocationOnMock -> videos).when(youtube).videos();
-
- return youtube;
- }
-
- private YouTube.Videos createMockVideos(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube.Videos videos = mock(YouTube.Videos.class);
-
- try {
- YouTube.Videos.List list = createMockVideosList(numBefore, numAfter, numInRange, after, before);
- when(videos.list(anyString())).thenReturn(list);
- } catch (IOException ex) {
- fail("Exception thrown while creating mock");
- }
-
- return videos;
- }
-
- private YouTube.Videos.List createMockVideosList(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube.Videos.List list = mock(YouTube.Videos.List.class);
-
- when(list.setMaxResults(anyLong())).thenReturn(list);
- when(list.setPageToken(anyString())).thenReturn(list);
- when(list.setId(anyString())).thenReturn(list);
- when(list.setKey(anyString())).thenReturn(list);
-
- VideoListResponseAnswer answer = new VideoListResponseAnswer(numBefore, numAfter, numInRange, after, before);
- try {
- doAnswer(answer).when(list).execute();
- } catch (IOException ioe) {
- fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
-
- return list;
- }
-
- private static VideoListResponse createMockVideoListResponse(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before, boolean page) {
- VideoListResponse feed = new VideoListResponse();
- List<Video> list = new LinkedList<>();
-
- for (int i = 0; i < numAfter; ++i) {
- com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime(after.getMillis() + 1000000);
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(published);
- list.add(video);
- }
- for (int i = 0; i < numInRange; ++i) {
- DateTime published;
- if ((before == null && after == null) || before == null) {
- published = DateTime.now(); // no date range or end time date range so just make the time now.
- } else if (after == null) {
- published = before.minusMillis(100000); //no beginning to range
- } else { // has to be in range
- long range = before.getMillis() - after.getMillis();
- published = after.plus(range / 2); //in the middle
- }
- com.google.api.client.util.DateTime ytPublished = new com.google.api.client.util.DateTime(published.getMillis());
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(ytPublished);
- video.getSnippet().setTitle(IN_RANGE_IDENTIFIER);
- list.add(video);
- }
- for (int i = 0; i < numBefore; ++i) {
- com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime((after.minusMillis(100000)).getMillis());
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(published);
- list.add(video);
- }
- if (page) {
- feed.setNextPageToken("A");
- } else {
- feed.setNextPageToken(null);
- }
-
- feed.setItems(list);
-
- return feed;
- }
-
- private static class VideoListResponseAnswer implements Answer<VideoListResponse> {
- private int afterCount = 0;
- private int beforeCount = 0;
- private int inCount = 0;
- private int maxBatch = 100;
-
- private int numAfter;
- private int numInRange;
- private int numBefore;
- private DateTime after;
- private DateTime before;
-
- private VideoListResponseAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- this.numBefore = numBefore;
- this.numAfter = numAfter;
- this.numInRange = numInRange;
- this.after = after;
- this.before = before;
- }
-
- @Override
- public VideoListResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
- int totalCount = 0;
- int batchAfter = 0;
- int batchBefore = 0;
- int batchIn = 0;
- inCount = 0;
- afterCount = 0;
- beforeCount = 0;
-
- if (afterCount != numAfter) {
- if (numAfter - afterCount >= maxBatch) {
- afterCount += maxBatch;
- batchAfter += maxBatch;
- totalCount += batchAfter;
- } else {
- batchAfter += numAfter - afterCount;
- totalCount += numAfter - afterCount;
- afterCount = numAfter;
- }
- }
- if (totalCount < maxBatch && inCount != numInRange) {
- if (numInRange - inCount >= maxBatch - totalCount) {
- inCount += maxBatch - totalCount;
- batchIn += maxBatch - totalCount;
- totalCount += batchIn;
- } else {
- batchIn += numInRange - inCount;
- totalCount += numInRange - inCount;
- inCount = numInRange;
- }
- }
- if (totalCount < maxBatch && beforeCount != numBefore) {
- if (numBefore - batchBefore >= maxBatch - totalCount) {
- batchBefore += maxBatch - totalCount;
- totalCount = maxBatch;
- beforeCount += batchBefore;
- } else {
- batchBefore += numBefore - beforeCount;
- totalCount += numBefore - beforeCount;
- beforeCount = numBefore;
- }
- }
-
- return createMockVideoListResponse(batchBefore, batchAfter, batchIn, after, before, numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeEventClassifierTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeEventClassifierTest.java
deleted file mode 100644
index 16565bb..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeEventClassifierTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.youtube.model.Video;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class YoutubeEventClassifierTest {
-
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
- private final String testObjectNode = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#somethingElse\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
-
- @Test
- public void testVideoClassification() {
- Class klass = YoutubeEventClassifier.detectClass(testVideo);
-
- assertEquals(klass, Video.class);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testExceptionClassification() {
- YoutubeEventClassifier.detectClass("");
- }
-
- @Test
- public void testObjectNodeClassification() {
- Class klass = YoutubeEventClassifier.detectClass(testObjectNode);
-
- assertEquals(klass, ObjectNode.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeVideoSerDeTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeVideoSerDeTest.java b/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeVideoSerDeTest.java
deleted file mode 100644
index 29afd19..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/com/youtube/serializer/YoutubeVideoSerDeTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Video;
-import org.joda.time.DateTime;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for YoutubeVideoSerDe.
- */
-public class YoutubeVideoSerDeTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoSerDeTest.class);
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
- private ObjectMapper objectMapper;
- private YoutubeActivityUtil youtubeActivityUtil;
-
- /**
- * setup for test.
- */
- @Before
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- youtubeActivityUtil = new YoutubeActivityUtil();
- }
-
- @Test
- public void testVideoObject() {
- LOGGER.info("raw: {}", testVideo);
-
- try {
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
-
- youtubeActivityUtil.updateActivity(video, activity, "testChannelId");
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assert (activity.getId().contains("id:youtube:post"));
- assertEquals(activity.getVerb(), "post");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:youtube");
- assertEquals(provider.getDisplayName(), "YouTube");
-
- ActivityObject actor = activity.getActor();
- assert (actor.getId().contains("id:youtube:"));
- assertNotNull(actor.getDisplayName());
- assertNotNull(actor.getSummary());
-
- assertNotNull(activity.getTitle());
- assertNotNull(activity.getUrl());
- assertNotNull(activity.getContent());
-
- assertEquals(activity.getPublished().getClass(), DateTime.class);
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- assertNotNull(extensions.get("youtube"));
- assertNotNull(extensions.get("likes"));
-
- assertTrue(testActivityObject(activity));
- } catch (Exception ex) {
- LOGGER.error("Exception while testing the Ser/De functionality of the Video deserializer: {}", ex);
- }
- }
-
- private boolean testActivityObject(Activity activity) {
- boolean valid = false;
-
- ActivityObject obj = activity.getObject();
-
- if ( obj.getObjectType().equals("video")
- && !obj.getImage().equals(null)
- && !obj.getUrl().equals("null")
- && obj.getUrl().contains("https://www.youtube.com/watch?v=")) {
- valid = true;
- }
-
- return valid;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java
new file mode 100644
index 0000000..fc5a309
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.processor;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.youtube.serializer.YoutubeActivityUtil;
+import org.apache.streams.youtube.serializer.YoutubeVideoDeserializer;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.youtube.model.Video;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Test for YoutubeTypeConverter.
+ */
+public class YoutubeTypeConverterTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverterTest.class);
+ private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/events/cj074q9r6csgv6i2kqhi2isc6k0\",\"publishedAt\":{\"value\":1422977409000,\"dateOnly\":false,\"timeZoneShift\":-360},\"thumbnails\":{\"default\":{\"height\":480,\"url\":\"https://i.ytimg.com/vi/sUOepRctwVE/sddefault.jpg\",\"width\":640}},\"title\":\"Be Heart Smart: Congestive Heart Failure LIVE Event\"},\"statistics\":{\"commentCount\":1,\"dislikeCount\":0,\"favoriteCount\":0,\"likeCount\":0,\"viewCount\":9}}";
+
+ private YoutubeTypeConverter youtubeTypeConverter;
+ private ObjectMapper objectMapper;
+
+ /**
+ * setup for test.
+ */
+ @Before
+ public void setup() {
+ objectMapper = StreamsJacksonMapper.getInstance();
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
+ objectMapper.registerModule(simpleModule);
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ youtubeTypeConverter = new YoutubeTypeConverter();
+ youtubeTypeConverter.prepare(null);
+ }
+
+ @Test
+ public void testVideoConversion() {
+ try {
+ LOGGER.info("raw: {}", testVideo);
+ Activity activity = new Activity();
+
+ Video video = objectMapper.readValue(testVideo, Video.class);
+ StreamsDatum streamsDatum = new StreamsDatum(video);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
+ YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
+
+ assertEquals(retList.size(), 1);
+ assert (retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to convert video to activity: {}", ex);
+ }
+ }
+
+ @Test
+ public void testStringVideoConversion() {
+ try {
+ LOGGER.info("raw: {}", testVideo);
+ Activity activity = new Activity();
+
+ Video video = objectMapper.readValue(testVideo, Video.class);
+ StreamsDatum streamsDatum = new StreamsDatum(testVideo);
+
+ assertNotNull(streamsDatum.getDocument());
+
+ List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
+ YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
+
+ assertEquals(retList.size(), 1);
+ assert (retList.get(0).getDocument() instanceof Activity);
+ assertEquals(activity, retList.get(0).getDocument());
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to convert video to activity: {}", ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java
new file mode 100644
index 0000000..b291760
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.LinearTimeBackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.ChannelListResponse;
+import org.junit.Test;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * YoutubeChannelDataCollectorTest tests YoutubeChannelDataCollector.
+ */
+public class YoutubeChannelDataCollectorTest {
+
+ private static final String ID = "12345";
+
+ @Test
+ public void testDataCollector() throws Exception {
+ YouTube youTube = createMockYoutube();
+ BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
+ BackOffStrategy strategy = new LinearTimeBackOffStrategy(1);
+ UserInfo userInfo = new UserInfo();
+ userInfo.setUserId(ID);
+ YoutubeConfiguration config = new YoutubeConfiguration();
+ config.setApiKey(ID);
+ YoutubeChannelDataCollector collector = new YoutubeChannelDataCollector(youTube, queue, strategy, userInfo, config);
+ collector.run();
+ assertEquals(1, queue.size());
+ StreamsDatum datum = queue.take();
+ assertNotNull(datum);
+ String document = (String) datum.getDocument();
+ assertNotNull(document);
+ }
+
+ private YouTube createMockYoutube() throws Exception {
+ YouTube mockYouTube = mock(YouTube.class);
+ YouTube.Channels channels = createMockChannels();
+ when(mockYouTube.channels()).thenReturn(channels);
+ return mockYouTube;
+ }
+
+ private YouTube.Channels createMockChannels() throws Exception {
+ YouTube.Channels mockChannels = mock(YouTube.Channels.class);
+ YouTube.Channels.List channelLists = createMockChannelsList();
+ when(mockChannels.list(anyString())).thenReturn(channelLists);
+ return mockChannels;
+ }
+
+ private YouTube.Channels.List createMockChannelsList() throws Exception {
+ YouTube.Channels.List mockList = mock(YouTube.Channels.List.class);
+ when(mockList.setId(anyString())).thenReturn(mockList);
+ when(mockList.setKey(anyString())).thenReturn(mockList);
+ ChannelListResponse response = createMockResponse();
+ when(mockList.execute()).thenReturn(response);
+ return mockList;
+ }
+
+ private ChannelListResponse createMockResponse() {
+ ChannelListResponse response = new ChannelListResponse();
+ List<Channel> channelList = new LinkedList<>();
+ response.setItems(channelList);
+ Channel channel = new Channel();
+ channel.setId(ID);
+ channelList.add(channel);
+ return response;
+ }
+
+}
[3/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeProvider.java
deleted file mode 100644
index 401b836..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeProvider.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.client.repackaged.com.google.common.base.Strings;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public abstract class YoutubeProvider implements StreamsProvider {
-
- private static final String STREAMS_ID = "YoutubeProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeProvider.class);
- private static final int MAX_BATCH_SIZE = 1000;
-
- // This OAuth 2.0 access scope allows for full read/write access to the
- // authenticated user's account.
- private List<String> scopes = Collections.singletonList("https://www.googleapis.com/auth/youtube");
-
- /**
- * Define a global instance of the HTTP transport.
- */
- private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
-
- /**
- * Define a global instance of the JSON factory.
- */
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
-
- private static final int DEFAULT_THREAD_POOL_SIZE = 5;
-
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- private ListeningExecutorService executor;
- private BlockingQueue<StreamsDatum> datumQueue;
- private AtomicBoolean isComplete;
- private boolean previousPullWasEmpty;
-
- protected YouTube youtube;
- protected YoutubeConfiguration config;
-
- /**
- * YoutubeProvider constructor.
- * Resolves config from JVM 'youtube'.
- */
- public YoutubeProvider() {
- this.config = new ComponentConfigurator<>(YoutubeConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("youtube"));
-
- Objects.requireNonNull(this.config.getApiKey());
- }
-
- /**
- * YoutubeProvider constructor - uses supplied YoutubeConfiguration.
- * @param config YoutubeConfiguration
- */
- public YoutubeProvider(YoutubeConfiguration config) {
- this.config = config;
-
- Objects.requireNonNull(this.config.getApiKey());
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- try {
- this.youtube = createYouTubeClient();
- } catch (IOException | GeneralSecurityException ex) {
- LOGGER.error("Failed to created oauth for YouTube : {}", ex);
- throw new RuntimeException(ex);
- }
-
- this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEFAULT_THREAD_POOL_SIZE));
- this.datumQueue = new LinkedBlockingQueue<>(1000);
- this.isComplete = new AtomicBoolean(false);
- this.previousPullWasEmpty = false;
- }
-
- @Override
- public void startStream() {
- BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
-
- for (UserInfo user : this.config.getYoutubeUsers()) {
- if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
- user.setAfterDate(this.config.getDefaultAfterDate());
- }
- if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- }
-
- ListenableFuture future = executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.youtube, user));
- futures.add(future);
- }
-
- this.executor.shutdown();
- }
-
- protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo);
-
- @Override
- public StreamsResultSet readCurrent() {
- BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- int batchCount = 0;
- while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
- StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
- if (datum != null) {
- ++batchCount;
- ComponentUtils.offerUntilSuccess(datum, batch);
- }
- }
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @VisibleForTesting
- protected YouTube createYouTubeClient() throws IOException, GeneralSecurityException {
- GoogleCredential.Builder credentialBuilder = new GoogleCredential.Builder()
- .setTransport(HTTP_TRANSPORT)
- .setJsonFactory(JSON_FACTORY)
- .setServiceAccountId(getConfig().getOauth().getServiceAccountEmailAddress())
- .setServiceAccountScopes(scopes);
-
- if ( !Strings.isNullOrEmpty(getConfig().getOauth().getPathToP12KeyFile())) {
- File p12KeyFile = new File(getConfig().getOauth().getPathToP12KeyFile());
- if ( p12KeyFile.exists() && p12KeyFile.isFile() && p12KeyFile.canRead()) {
- credentialBuilder = credentialBuilder.setServiceAccountPrivateKeyFromP12File(p12KeyFile);
- }
- }
- Credential credential = credentialBuilder.build();
- return new YouTube.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setApplicationName("Streams Application").build();
- }
-
- @Override
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(this.executor, 10, 10);
- this.executor = null;
- }
-
- public YoutubeConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(YoutubeConfiguration config) {
- this.config = config;
- }
-
- /**
- * Set and overwrite the default before date that was read from the configuration file.
- * @param defaultBeforeDate defaultBeforeDate
- */
- public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
- this.config.setDefaultBeforeDate(defaultBeforeDate);
- }
-
- /**
- * Set and overwrite the default after date that was read from teh configuration file.
- * @param defaultAfterDate defaultAfterDate
- */
- public void setDefaultAfterDate(DateTime defaultAfterDate) {
- this.config.setDefaultAfterDate(defaultAfterDate);
- }
-
- /**
- * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
- * @param userIds Set of String userIds
- */
- public void setUserInfoWithDefaultDates(Set<String> userIds) {
- List<UserInfo> youtubeUsers = new LinkedList<>();
-
- for (String userId : userIds) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(this.config.getDefaultAfterDate());
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- youtubeUsers.add(user);
- }
-
- this.config.setYoutubeUsers(youtubeUsers);
- }
-
- /**
- * Set and overwrite user into from teh configuration file. Only sets after dater.
- * @param usersAndAfterDates usersAndAfterDates
- */
- public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
- List<UserInfo> youtubeUsers = new LinkedList<>();
-
- for (String userId : usersAndAfterDates.keySet()) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(usersAndAfterDates.get(userId));
- youtubeUsers.add(user);
- }
-
- this.config.setYoutubeUsers(youtubeUsers);
- }
-
- @Override
- public boolean isRunning() {
- if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isComplete.set(true);
- LOGGER.info("Exiting");
- }
- return !isComplete.get();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityCollector.java
deleted file mode 100644
index 518a762..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityCollector.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.ActivityListResponse;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoListResponse;
-import com.google.gson.Gson;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * YoutubeDataCollector for YoutubeUserActivityProvider.
- */
-public class YoutubeUserActivityCollector extends YoutubeDataCollector {
-
- /**
- * Max results allowed per request
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final long MAX_RESULTS = 50;
- private static final int MAX_ATTEMPTS = 5;
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeUserActivityCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- static { //set up mapper for Google Activity Object
- SimpleModule simpleModule = new SimpleModule();
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private BlockingQueue<StreamsDatum> datumQueue;
- private BackOffStrategy backOff;
- private YouTube youtube;
- private UserInfo userInfo;
- private YoutubeConfiguration config;
-
- Gson gson = new Gson();
-
- /**
- * YoutubeUserActivityCollector constructor.
- * @param youtube YouTube
- * @param datumQueue BlockingQueue of StreamsDatum
- * @param backOff BackOffStrategy
- * @param userInfo UserInfo
- * @param config YoutubeConfiguration
- */
- public YoutubeUserActivityCollector(
- YouTube youtube,
- BlockingQueue<StreamsDatum> datumQueue,
- BackOffStrategy backOff,
- UserInfo userInfo,
- YoutubeConfiguration config) {
- this.youtube = youtube;
- this.datumQueue = datumQueue;
- this.backOff = backOff;
- this.userInfo = userInfo;
- this.config = config;
- }
-
- @Override
- public void run() {
- collectActivityData();
- }
-
- /**
- * Iterate through all users in the Youtube configuration and collect all videos
- * associated with their accounts.
- */
- protected void collectActivityData() {
- try {
- YouTube.Activities.List request = null;
- ActivityListResponse feed = null;
-
- boolean tryAgain = false;
- int attempt = 0;
- DateTime afterDate = userInfo.getAfterDate();
- DateTime beforeDate = userInfo.getBeforeDate();
-
- do {
- try {
- if (request == null) {
- request = this.youtube.activities().list("contentDetails")
- .setChannelId(userInfo.getUserId())
- .setMaxResults(MAX_RESULTS)
- .setKey(config.getApiKey());
- feed = request.execute();
- } else {
- request = this.youtube.activities().list("contentDetails")
- .setChannelId(userInfo.getUserId())
- .setMaxResults(MAX_RESULTS)
- .setPageToken(feed.getNextPageToken())
- .setKey(config.getApiKey());
- feed = request.execute();
- }
- this.backOff.reset(); //successful pull reset api.
-
- processActivityFeed(feed, afterDate, beforeDate);
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
- ++attempt;
- }
- }
- while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
- } catch (Throwable throwable) {
- if (throwable instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throwable.printStackTrace();
- LOGGER.warn("Unable to pull Activities for user={} : {}",this.userInfo.getUserId(), throwable);
- }
- }
-
- /**
- * Given a feed and an after and before date, fetch all relevant user videos
- * and place them into the datumQueue for post-processing.
- * @param feed ActivityListResponse
- * @param afterDate DateTime
- * @param beforeDate DateTime
- * @throws IOException IOException
- * @throws InterruptedException InterruptedException
- */
- void processActivityFeed(ActivityListResponse feed, DateTime afterDate, DateTime beforeDate) throws IOException, InterruptedException {
- for (com.google.api.services.youtube.model.Activity activity : feed.getItems()) {
- try {
- List<Video> videos = new ArrayList<>();
-
- if (activity.getContentDetails().getUpload() != null) {
- videos.addAll(getVideoList(activity.getContentDetails().getUpload().getVideoId()));
- }
- if (activity.getContentDetails().getPlaylistItem() != null && activity.getContentDetails().getPlaylistItem().getResourceId() != null) {
- videos.addAll(getVideoList(activity.getContentDetails().getPlaylistItem().getResourceId().getVideoId()));
- }
-
- processVideos(videos, afterDate, beforeDate, activity, feed);
- } catch (Exception ex) {
- LOGGER.error("Error while trying to process activity: {}, {}", activity, ex);
- }
- }
- }
-
- /**
- * Process a list of Video objects.
- * @param videos List of Video
- * @param afterDate afterDate
- * @param beforeDate beforeDate
- * @param activity com.google.api.services.youtube.model.Activity
- * @param feed ActivityListResponse
- */
- void processVideos(List<Video> videos, DateTime afterDate, DateTime beforeDate, com.google.api.services.youtube.model.Activity activity, ActivityListResponse feed) {
- try {
- for (Video video : videos) {
- if (video != null) {
- org.joda.time.DateTime published = new org.joda.time.DateTime(video.getSnippet().getPublishedAt().getValue());
- if ((afterDate == null && beforeDate == null)
- || (beforeDate == null && afterDate.isBefore(published))
- || (afterDate == null && beforeDate.isAfter(published))
- || ((afterDate != null && beforeDate != null) && (afterDate.isAfter(published) && beforeDate.isBefore(published)))) {
- LOGGER.debug("Providing Youtube Activity: {}", MAPPER.writeValueAsString(video));
- this.datumQueue.put(new StreamsDatum(gson.toJson(video), activity.getId()));
- } else if (afterDate != null && afterDate.isAfter(published)) {
- feed.setNextPageToken(null); // do not fetch next page
- break;
- }
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to process video list: {}, {}", videos, ex);
- }
- }
-
- /**
- * Given a Youtube videoId, return the relevant Youtube Video object.
- * @param videoId videoId
- * @return List of Video
- * @throws IOException
- */
- List<Video> getVideoList(String videoId) throws IOException {
- VideoListResponse videosListResponse = this.youtube.videos().list("snippet,statistics")
- .setId(videoId)
- .setKey(config.getApiKey())
- .execute();
-
- if (videosListResponse.getItems().size() == 0) {
- LOGGER.debug("No Youtube videos found for videoId: {}", videoId);
- return new ArrayList<>();
- }
-
- return videosListResponse.getItems();
- }
-
- BlockingQueue<StreamsDatum> getDatumQueue() {
- return this.datumQueue;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityProvider.java
deleted file mode 100644
index 934a0e5..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeUserActivityProvider.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of user ids or names.
- */
-public class YoutubeUserActivityProvider extends YoutubeProvider {
-
- public YoutubeUserActivityProvider() {
- super();
- }
-
- public YoutubeUserActivityProvider(YoutubeConfiguration config) {
- super(config);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- return new YoutubeUserActivityCollector(youtube, queue, strategy, userInfo, config);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * youtube.oauth.pathToP12KeyFile
- * youtube.oauth.serviceAccountEmailAddress
- * youtube.apiKey
- * youtube.youtubeUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
- YoutubeUserActivityProvider provider = new YoutubeUserActivityProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- if ( datum.getDocument() instanceof String ) {
- json = (String) datum.getDocument();
- } else {
- json = mapper.writeValueAsString(datum.getDocument());
- }
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeActivityUtil.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeActivityUtil.java
deleted file mode 100644
index 32a011f..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeActivityUtil.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-import com.google.api.services.youtube.model.Video;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class YoutubeActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeActivityUtil.class);
-
- /**
- * Given a {@link YouTube.Videos} object and an
- * {@link Activity} object, fill out the appropriate details
- *
- * @param video Video
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Video video, Activity activity, String channelId) throws ActivitySerializerException {
- activity.setActor(buildActor(video, video.getSnippet().getChannelId()));
- activity.setVerb("post");
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(video.getId()).orElse(null)));
-
- activity.setPublished(new DateTime(video.getSnippet().getPublishedAt().getValue()));
- activity.setTitle(video.getSnippet().getTitle());
- activity.setContent(video.getSnippet().getDescription());
- activity.setUrl("https://www.youtube.com/watch?v=" + video.getId());
-
- activity.setProvider(getProvider());
-
- activity.setObject(buildActivityObject(video));
-
- addYoutubeExtensions(activity, video);
- }
-
-
- /**
- * Given a {@link Channel} object and an
- * {@link Activity} object, fill out the appropriate details
- *
- * @param channel Channel
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Channel channel, Activity activity, String channelId) throws ActivitySerializerException {
- try {
- activity.setProvider(getProvider());
- activity.setVerb("post");
- activity.setActor(createActorForChannel(channel));
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("youtube", channel);
- activity.setAdditionalProperty("extensions", extensions);
- } catch (Throwable throwable) {
- throw new ActivitySerializerException(throwable);
- }
- }
-
- /**
- * createActorForChannel.
- * @param channel Channel
- * @return $.actor
- */
- public static ActivityObject createActorForChannel(Channel channel) {
- ActivityObject actor = new ActivityObject();
- // TODO: use generic provider id concatenator
- actor.setId("id:youtube:" + channel.getId());
- actor.setSummary(channel.getSnippet().getDescription());
- actor.setDisplayName(channel.getSnippet().getTitle());
- Image image = new Image();
- image.setUrl(channel.getSnippet().getThumbnails().getHigh().getUrl());
- actor.setImage(image);
- actor.setUrl("https://youtube.com/user/" + channel.getId());
- Map<String, Object> actorExtensions = new HashMap<>();
- actorExtensions.put("followers", channel.getStatistics().getSubscriberCount());
- actorExtensions.put("posts", channel.getStatistics().getVideoCount());
- actor.setAdditionalProperty("extensions", actorExtensions);
- return actor;
- }
-
- /**
- * Given a video object, create the appropriate activity object with a valid image
- * (thumbnail) and video URL.
- * @param video Video
- * @return Activity Object with Video URL and a thumbnail image
- */
- private static ActivityObject buildActivityObject(Video video) {
- ActivityObject activityObject = new ActivityObject();
-
- ThumbnailDetails thumbnailDetails = video.getSnippet().getThumbnails();
- Thumbnail thumbnail = thumbnailDetails.getDefault();
-
- if (thumbnail != null) {
- Image image = new Image();
- image.setUrl(thumbnail.getUrl());
- image.setHeight(thumbnail.getHeight());
- image.setWidth(thumbnail.getWidth());
-
- activityObject.setImage(image);
- }
-
- activityObject.setUrl("https://www.youtube.com/watch?v=" + video.getId());
- activityObject.setObjectType("video");
-
- return activityObject;
- }
-
- /**
- * Add the Youtube extensions to the Activity object that we're building.
- * @param activity Activity
- * @param video Video
- */
- private static void addYoutubeExtensions(Activity activity, Video video) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- extensions.put("youtube", video);
-
- if (video.getStatistics() != null) {
- Map<String, Object> likes = new HashMap<>();
- likes.put("count", video.getStatistics().getCommentCount());
- extensions.put("likes", likes);
- }
- }
-
- /**
- * Build an {@link ActivityObject} actor given the video object
- * @param video Video
- * @param id id
- * @return Actor object
- */
- private static ActivityObject buildActor(Video video, String id) {
- ActivityObject actor = new ActivityObject();
-
- actor.setId("id:youtube:" + id);
- actor.setDisplayName(video.getSnippet().getChannelTitle());
- actor.setSummary(video.getSnippet().getDescription());
- actor.setAdditionalProperty("handle", video.getSnippet().getChannelTitle());
-
- return actor;
- }
-
- /**
- * Gets the common youtube {@link Provider} object
- * @return a provider object representing YouTube
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:youtube");
- provider.setDisplayName("YouTube");
- return provider;
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:youtube:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:youtube"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeChannelDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeChannelDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeChannelDeserializer.java
deleted file mode 100644
index e28b4a1..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeChannelDeserializer.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.ChannelContentDetails;
-import com.google.api.services.youtube.model.ChannelLocalization;
-import com.google.api.services.youtube.model.ChannelSnippet;
-import com.google.api.services.youtube.model.ChannelStatistics;
-import com.google.api.services.youtube.model.ChannelTopicDetails;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * YoutubeChannelDeserializer is a JsonDeserializer for Channel.
- */
-public class YoutubeChannelDeserializer extends JsonDeserializer<Channel> {
-
- @Override
- public Channel deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
- JsonNode node = jp.getCodec().readTree(jp);
- try {
- Channel channel = new Channel();
- if (node.findPath("etag") != null) {
- channel.setEtag(node.get("etag").asText());
- }
- if (node.findPath("kind") != null) {
- channel.setKind(node.get("kind").asText());
- }
- channel.setId(node.get("id").asText());
- channel.setTopicDetails(setTopicDetails(node.findValue("topicDetails")));
- channel.setStatistics(setChannelStatistics(node.findValue("statistics")));
- channel.setContentDetails(setContentDetails(node.findValue("contentDetails")));
- channel.setSnippet(setChannelSnippet(node.findValue("snippet")));
- return channel;
- } catch (Throwable throwable) {
- throw new IOException(throwable);
- }
- }
-
- protected ChannelSnippet setChannelSnippet(JsonNode node) {
- ChannelSnippet snippet = new ChannelSnippet();
- snippet.setTitle(node.get("title").asText());
- snippet.setDescription(node.get("description").asText());
- snippet.setPublishedAt(new DateTime(node.get("publishedAt").get("value").longValue()));
- snippet.setLocalized(setLocalized(node.findValue("localized")));
- snippet.setThumbnails(setThumbnails(node.findValue("thumbnails")));
- return snippet;
- }
-
- protected ThumbnailDetails setThumbnails(JsonNode node) {
- ThumbnailDetails details = new ThumbnailDetails();
- if (node == null) {
- return details;
- }
- details.setDefault(new Thumbnail().setUrl(node.get("default").get("url").asText()));
- details.setHigh(new Thumbnail().setUrl(node.get("high").get("url").asText()));
- details.setMedium(new Thumbnail().setUrl(node.get("medium").get("url").asText()));
- return details;
- }
-
- protected ChannelLocalization setLocalized(JsonNode node) {
- if (node == null) {
- return new ChannelLocalization();
- }
- ChannelLocalization localization = new ChannelLocalization();
- localization.setDescription(node.get("description").asText());
- localization.setTitle(node.get("title").asText());
- return localization;
- }
-
- protected ChannelContentDetails setContentDetails(JsonNode node) {
- ChannelContentDetails contentDetails = new ChannelContentDetails();
- if (node == null) {
- return contentDetails;
- }
- if (node.findValue("googlePlusUserId") != null) {
- contentDetails.setGooglePlusUserId(node.get("googlePlusUserId").asText());
- }
- contentDetails.setRelatedPlaylists(setRelatedPlaylists(node.findValue("relatedPlaylists")));
- return contentDetails;
- }
-
- protected ChannelContentDetails.RelatedPlaylists setRelatedPlaylists(JsonNode node) {
- ChannelContentDetails.RelatedPlaylists playlists = new ChannelContentDetails.RelatedPlaylists();
- if (node == null) {
- return playlists;
- }
- if (node.findValue("favorites") != null) {
- playlists.setFavorites(node.get("favorites").asText());
- }
- if (node.findValue("likes") != null) {
- playlists.setLikes(node.get("likes").asText());
- }
- if (node.findValue("uploads") != null) {
- playlists.setUploads(node.get("uploads").asText());
- }
- return playlists;
- }
-
- protected ChannelStatistics setChannelStatistics(JsonNode node) {
- ChannelStatistics stats = new ChannelStatistics();
- if (node == null) {
- return stats;
- }
- stats.setCommentCount(node.get("commentCount").bigIntegerValue());
- stats.setHiddenSubscriberCount(node.get("hiddenSubscriberCount").asBoolean());
- stats.setSubscriberCount(node.get("subscriberCount").bigIntegerValue());
- stats.setVideoCount(node.get("videoCount").bigIntegerValue());
- stats.setViewCount(node.get("viewCount").bigIntegerValue());
- return stats;
- }
-
- protected ChannelTopicDetails setTopicDetails(JsonNode node) {
- ChannelTopicDetails details = new ChannelTopicDetails();
- if (node == null) {
- return details;
- }
- List<String> topicIds = new LinkedList<>();
- for (JsonNode jsonNode : node.get("topicIds")) {
- topicIds.add(jsonNode.asText());
- }
- details.setTopicIds(topicIds);
- return details;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeEventClassifier.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeEventClassifier.java
deleted file mode 100644
index e7645bd..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeEventClassifier.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.youtube.model.Video;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class YoutubeEventClassifier {
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private static final String VIDEO_IDENTIFIER = "\"youtube#video\"";
- private static final String CHANNEL_IDENTIFIER = "youtube#channel";
-
- /**
- * detect probable Class of a json String from YouTube.
- * @param json json
- * @return Class
- */
- public static Class detectClass(String json) {
- Objects.requireNonNull(json);
- Preconditions.checkArgument(StringUtils.isNotEmpty(json));
-
- ObjectNode objectNode;
- try {
- objectNode = (ObjectNode) mapper.readTree(json);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
-
- if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(VIDEO_IDENTIFIER)) {
- return Video.class;
- } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().contains(CHANNEL_IDENTIFIER)) {
- return com.google.api.services.youtube.model.Channel.class;
- } else {
- return ObjectNode.class;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeVideoDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeVideoDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeVideoDeserializer.java
deleted file mode 100644
index 43fe8c6..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/serializer/YoutubeVideoDeserializer.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.serializer;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoSnippet;
-import com.google.api.services.youtube.model.VideoStatistics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class YoutubeVideoDeserializer extends JsonDeserializer<Video> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoDeserializer.class);
-
- /**
- * Because the Youtube Video object contains complex objects within its hierarchy, we have to use
- * a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.youtube.YouTube.Videos} object
- * @throws java.io.IOException IOException
- * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
- */
- @Override
- public Video deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Video video = new Video();
-
- try {
- video.setId(node.get("id").asText());
- video.setEtag(node.get("etag").asText());
- video.setKind(node.get("kind").asText());
-
- video.setSnippet(buildSnippet(node));
- video.setStatistics(buildStatistics(node));
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize a Video object: {}", ex);
- }
-
- return video;
- }
-
- /**
- * Given the raw JsonNode, construct a video snippet object.
- * @param node JsonNode
- * @return VideoSnippet
- */
- private VideoSnippet buildSnippet(JsonNode node) {
- VideoSnippet snippet = new VideoSnippet();
- JsonNode snippetNode = node.get("snippet");
-
- snippet.setChannelId(snippetNode.get("channelId").asText());
- snippet.setChannelTitle(snippetNode.get("channelTitle").asText());
- snippet.setDescription(snippetNode.get("description").asText());
- snippet.setTitle(snippetNode.get("title").asText());
- snippet.setPublishedAt(new DateTime(snippetNode.get("publishedAt").get("value").asLong()));
-
- ThumbnailDetails thumbnailDetails = new ThumbnailDetails();
- for (JsonNode t : snippetNode.get("thumbnails")) {
- Thumbnail thumbnail = new Thumbnail();
-
- thumbnail.setHeight(t.get("height").asLong());
- thumbnail.setUrl(t.get("url").asText());
- thumbnail.setWidth(t.get("width").asLong());
-
- thumbnailDetails.setDefault(thumbnail);
- }
-
- snippet.setThumbnails(thumbnailDetails);
-
- return snippet;
- }
-
- /**
- * Given the raw JsonNode, construct a statistics object.
- * @param node JsonNode
- * @return VideoStatistics
- */
- private VideoStatistics buildStatistics(JsonNode node) {
- VideoStatistics statistics = new VideoStatistics();
- JsonNode statisticsNode = node.get("statistics");
-
- statistics.setCommentCount(statisticsNode.get("commentCount").bigIntegerValue());
- statistics.setDislikeCount(statisticsNode.get("dislikeCount").bigIntegerValue());
- statistics.setFavoriteCount(statisticsNode.get("favoriteCount").bigIntegerValue());
- statistics.setLikeCount(statisticsNode.get("likeCount").bigIntegerValue());
- statistics.setViewCount(statisticsNode.get("viewCount").bigIntegerValue());
-
- return statistics;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java
new file mode 100644
index 0000000..ee9ca18
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.processor;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.youtube.serializer.YoutubeActivityUtil;
+import org.apache.streams.youtube.serializer.YoutubeChannelDeserializer;
+import org.apache.streams.youtube.serializer.YoutubeEventClassifier;
+import org.apache.streams.youtube.serializer.YoutubeVideoDeserializer;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.Video;
+import org.apache.commons.lang.NotImplementedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+
+public class YoutubeTypeConverter implements StreamsProcessor {
+
+ public static final String STREAMS_ID = "YoutubeTypeConverter";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverter.class);
+
+ private StreamsJacksonMapper mapper;
+ private Queue<Video> inQueue;
+ private Queue<StreamsDatum> outQueue;
+ private int count = 0;
+
+ public YoutubeTypeConverter() {}
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum streamsDatum) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = streamsDatum.getDocument();
+
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+ Activity activity;
+
+ if (item instanceof String) {
+ item = deserializeItem(item);
+ }
+
+ if (item instanceof Video) {
+ activity = new Activity();
+ YoutubeActivityUtil.updateActivity((Video)item, activity, streamsDatum.getId());
+ } else if (item instanceof Channel) {
+ activity = new Activity();
+ YoutubeActivityUtil.updateActivity((Channel)item, activity, null);
+ } else {
+ throw new NotImplementedException("Type conversion not implement for type : " + item.getClass().getName());
+ }
+
+ if (activity != null) {
+ result = new StreamsDatum(activity);
+ count++;
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while converting Video to Activity: {}", ex);
+ }
+
+ if (result != null) {
+ List<StreamsDatum> streamsDatumList = new ArrayList<>();
+ streamsDatumList.add(result);
+ return streamsDatumList;
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+ private Object deserializeItem(Object item) {
+ try {
+ Class klass = YoutubeEventClassifier.detectClass((String) item);
+ if (klass.equals(Video.class)) {
+ item = mapper.readValue((String) item, Video.class);
+ } else if (klass.equals(Channel.class)) {
+ item = mapper.readValue((String) item, Channel.class);
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserializeItem: {}", ex);
+ }
+
+ return item;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ mapper = StreamsJacksonMapper.getInstance();
+
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
+ mapper.registerModule(simpleModule);
+ simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Channel.class, new YoutubeChannelDeserializer());
+ mapper.registerModule(simpleModule);
+ }
+
+ @Override
+ public void cleanUp() {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java
new file mode 100644
index 0000000..332b1a7
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Channel;
+import com.google.gson.Gson;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Collects YoutubeChannelData on behalf of YoutubeChannelProvider.
+ */
+public class YoutubeChannelDataCollector extends YoutubeDataCollector {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelDataCollector.class);
+ private static final String CONTENT = "snippet,contentDetails,statistics,topicDetails";
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private static final int MAX_ATTEMPTS = 5;
+
+ private YouTube youTube;
+ private BlockingQueue<StreamsDatum> queue;
+ private BackOffStrategy strategy;
+ private UserInfo userInfo;
+ private YoutubeConfiguration youtubeConfig;
+
+ /**
+ * YoutubeChannelDataCollector constructor.
+ * @param youTube YouTube
+ * @param queue BlockingQueue of StreamsDatum
+ * @param strategy BackOffStrategy
+ * @param userInfo UserInfo
+ * @param youtubeConfig YoutubeConfiguration
+ */
+ public YoutubeChannelDataCollector(
+ YouTube youTube,
+ BlockingQueue<StreamsDatum> queue,
+ BackOffStrategy strategy,
+ UserInfo userInfo,
+ YoutubeConfiguration youtubeConfig) {
+ this.youTube = youTube;
+ this.queue = queue;
+ this.strategy = strategy;
+ this.userInfo = userInfo;
+ this.youtubeConfig = youtubeConfig;
+ }
+
+ @Override
+ public void run() {
+ Gson gson = new Gson();
+ try {
+ int attempt = 0;
+ YouTube.Channels.List channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setKey(this.youtubeConfig.getApiKey());
+ boolean tryAgain = false;
+ do {
+ try {
+ List<Channel> channels = channelLists.execute().getItems();
+ for (Channel channel : channels) {
+ String json = gson.toJson(channel);
+ this.queue.put(new StreamsDatum(json, channel.getId()));
+ }
+ if (StringUtils.isEmpty(channelLists.getPageToken())) {
+ channelLists = null;
+ } else {
+ channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setOauthToken(this.youtubeConfig.getApiKey())
+ .setPageToken(channelLists.getPageToken());
+ }
+ } catch (GoogleJsonResponseException gjre) {
+ LOGGER.warn("GoogleJsonResposneException caught : {}", gjre);
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.strategy);
+ ++attempt;
+ } catch (Throwable throwable) {
+ LOGGER.warn("Unable to get channel info for id : {}", this.userInfo.getUserId());
+ LOGGER.warn("Excpection thrown while trying to get channel info : {}", throwable);
+ }
+ }
+ while ((tryAgain && attempt < MAX_ATTEMPTS) || channelLists != null);
+
+ } catch (Throwable throwable) {
+ LOGGER.warn(throwable.getMessage());
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java
new file mode 100644
index 0000000..d8a09d5
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.services.youtube.YouTube;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Retrieve recent activity from a list of channels.
+ */
+public class YoutubeChannelProvider extends YoutubeProvider {
+
+ public YoutubeChannelProvider() {
+ super();
+ }
+
+ public YoutubeChannelProvider(YoutubeConfiguration config) {
+ super(config);
+ }
+
+ /**
+ * To use from command line:
+ * <p/>
+ * Supply (at least) the following required configuration in application.conf:
+ * <p/>
+ * youtube.oauth.pathToP12KeyFile
+ * youtube.oauth.serviceAccountEmailAddress
+ * youtube.apiKey
+ * youtube.youtubeUsers
+ * <p/>
+ * Launch using:
+ * <p/>
+ * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
+ * @param args args
+ * @throws Exception Exception
+ */
+ public static void main(String[] args) throws Exception {
+
+ Preconditions.checkArgument(args.length >= 2);
+
+ String configfile = args[0];
+ String outfile = args[1];
+
+ Config reference = ConfigFactory.load();
+ File file = new File(configfile);
+ assert (file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
+
+ Config typesafe = testResourceConfig.withFallback(reference).resolve();
+
+ StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
+ YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
+ YoutubeChannelProvider provider = new YoutubeChannelProvider(config);
+
+ ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
+ provider.prepare(config);
+ provider.startStream();
+ do {
+ Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
+ for (StreamsDatum datum : provider.readCurrent()) {
+ String json;
+ try {
+ if (datum.getDocument() instanceof String) {
+ json = (String) datum.getDocument();
+ } else {
+ json = mapper.writeValueAsString(datum.getDocument());
+ }
+ outStream.println(json);
+ } catch (JsonProcessingException ex) {
+ System.err.println(ex.getMessage());
+ }
+ }
+ }
+ while (provider.isRunning());
+ provider.cleanUp();
+ outStream.flush();
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
+ return new YoutubeChannelDataCollector(youtube, queue, strategy, userInfo, this.config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java
new file mode 100644
index 0000000..b05365c
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.util.api.requests.backoff.BackOffException;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base Collector for Youtube Data.
+ */
+public abstract class YoutubeDataCollector implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeDataCollector.class);
+
+ /**
+ * Looks at the status code of the expception. If the code indicates that the request should be retried,
+ * it executes the back off strategy and returns true.
+ * @param gjre
+ * @param backOff
+ * @return returns true if the error code of the exception indicates the request should be retried.
+ */
+ public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
+ boolean tryAgain = false;
+
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request : {}", gjre);
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ break;
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ backOff.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ backOff.backOff();
+ break;
+ }
+
+ return tryAgain;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java
new file mode 100644
index 0000000..da21722
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.ComponentUtils;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.youtube.YouTube;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.commons.lang3.StringUtils;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public abstract class YoutubeProvider implements StreamsProvider {
+
+ private static final String STREAMS_ID = "YoutubeProvider";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeProvider.class);
+ private static final int MAX_BATCH_SIZE = 1000;
+ /**
+ * Define a global instance of the HTTP transport.
+ */
+ private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
+ /**
+ * Define a global instance of the JSON factory.
+ */
+ private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private static final int DEFAULT_THREAD_POOL_SIZE = 5;
+ protected YouTube youtube;
+ protected YoutubeConfiguration config;
+ // This OAuth 2.0 access scope allows for full read/write access to the
+ // authenticated user's account.
+ private List<String> scopes = Collections.singletonList("https://www.googleapis.com/auth/youtube");
+ private List<ListenableFuture<Object>> futures = new ArrayList<>();
+ private ListeningExecutorService executor;
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private AtomicBoolean isComplete;
+ private boolean previousPullWasEmpty;
+
+ /**
+ * YoutubeProvider constructor.
+ * Resolves config from JVM 'youtube'.
+ */
+ public YoutubeProvider() {
+ this.config = new ComponentConfigurator<>(YoutubeConfiguration.class)
+ .detectConfiguration(StreamsConfigurator.getConfig().getConfig("youtube"));
+
+ Objects.requireNonNull(this.config.getApiKey());
+ }
+
+ /**
+ * YoutubeProvider constructor - uses supplied YoutubeConfiguration.
+ * @param config YoutubeConfiguration
+ */
+ public YoutubeProvider(YoutubeConfiguration config) {
+ this.config = config;
+
+ Objects.requireNonNull(this.config.getApiKey());
+ }
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ try {
+ this.youtube = createYouTubeClient();
+ } catch (IOException | GeneralSecurityException ex) {
+ LOGGER.error("Failed to created oauth for YouTube : {}", ex);
+ throw new RuntimeException(ex);
+ }
+
+ this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEFAULT_THREAD_POOL_SIZE));
+ this.datumQueue = new LinkedBlockingQueue<>(1000);
+ this.isComplete = new AtomicBoolean(false);
+ this.previousPullWasEmpty = false;
+ }
+
+ @Override
+ public void startStream() {
+ BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
+
+ for (UserInfo user : this.config.getYoutubeUsers()) {
+ if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ }
+ if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ }
+
+ ListenableFuture future = executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.youtube, user));
+ futures.add(future);
+ }
+
+ this.executor.shutdown();
+ }
+
+ protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo);
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
+ int batchCount = 0;
+ while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
+ StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
+ if (datum != null) {
+ ++batchCount;
+ ComponentUtils.offerUntilSuccess(datum, batch);
+ }
+ }
+ return new StreamsResultSet(batch);
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @VisibleForTesting
+ protected YouTube createYouTubeClient() throws IOException, GeneralSecurityException {
+ GoogleCredential.Builder credentialBuilder = new GoogleCredential.Builder()
+ .setTransport(HTTP_TRANSPORT)
+ .setJsonFactory(JSON_FACTORY)
+ .setServiceAccountId(getConfig().getOauth().getServiceAccountEmailAddress())
+ .setServiceAccountScopes(scopes);
+
+ if (StringUtils.isNotEmpty(getConfig().getOauth().getPathToP12KeyFile())) {
+ File p12KeyFile = new File(getConfig().getOauth().getPathToP12KeyFile());
+ if (p12KeyFile.exists() && p12KeyFile.isFile() && p12KeyFile.canRead()) {
+ credentialBuilder = credentialBuilder.setServiceAccountPrivateKeyFromP12File(p12KeyFile);
+ }
+ }
+ Credential credential = credentialBuilder.build();
+ return new YouTube.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setApplicationName("Streams Application").build();
+ }
+
+ @Override
+ public void cleanUp() {
+ ComponentUtils.shutdownExecutor(this.executor, 10, 10);
+ this.executor = null;
+ }
+
+ public YoutubeConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(YoutubeConfiguration config) {
+ this.config = config;
+ }
+
+ /**
+ * Set and overwrite the default before date that was read from the configuration file.
+ * @param defaultBeforeDate defaultBeforeDate
+ */
+ public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
+ this.config.setDefaultBeforeDate(defaultBeforeDate);
+ }
+
+ /**
+ * Set and overwrite the default after date that was read from teh configuration file.
+ * @param defaultAfterDate defaultAfterDate
+ */
+ public void setDefaultAfterDate(DateTime defaultAfterDate) {
+ this.config.setDefaultAfterDate(defaultAfterDate);
+ }
+
+ /**
+ * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
+ * @param userIds Set of String userIds
+ */
+ public void setUserInfoWithDefaultDates(Set<String> userIds) {
+ List<UserInfo> youtubeUsers = new LinkedList<>();
+
+ for (String userId : userIds) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ youtubeUsers.add(user);
+ }
+
+ this.config.setYoutubeUsers(youtubeUsers);
+ }
+
+ /**
+ * Set and overwrite user into from teh configuration file. Only sets after dater.
+ * @param usersAndAfterDates usersAndAfterDates
+ */
+ public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
+ List<UserInfo> youtubeUsers = new LinkedList<>();
+
+ for (String userId : usersAndAfterDates.keySet()) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(usersAndAfterDates.get(userId));
+ youtubeUsers.add(user);
+ }
+
+ this.config.setYoutubeUsers(youtubeUsers);
+ }
+
+ @Override
+ public boolean isRunning() {
+ if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
+ LOGGER.info("Completed");
+ isComplete.set(true);
+ LOGGER.info("Exiting");
+ }
+ return !isComplete.get();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java
new file mode 100644
index 0000000..ba3e10f
--- /dev/null
+++ b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.youtube.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.youtube.YoutubeConfiguration;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.ActivityListResponse;
+import com.google.api.services.youtube.model.Video;
+import com.google.api.services.youtube.model.VideoListResponse;
+import com.google.gson.Gson;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * YoutubeDataCollector for YoutubeUserActivityProvider.
+ */
+public class YoutubeUserActivityCollector extends YoutubeDataCollector {
+
+ /**
+ * Max results allowed per request
+ * https://developers.google.com/+/api/latest/activities/list
+ */
+ private static final long MAX_RESULTS = 50;
+ private static final int MAX_ATTEMPTS = 5;
+ private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeUserActivityCollector.class);
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+
+ static { //set up mapper for Google Activity Object
+ SimpleModule simpleModule = new SimpleModule();
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private Gson gson = new Gson();
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private BackOffStrategy backOff;
+ private YouTube youtube;
+ private UserInfo userInfo;
+ private YoutubeConfiguration config;
+
+ /**
+ * YoutubeUserActivityCollector constructor.
+ * @param youtube YouTube
+ * @param datumQueue BlockingQueue of StreamsDatum
+ * @param backOff BackOffStrategy
+ * @param userInfo UserInfo
+ * @param config YoutubeConfiguration
+ */
+ public YoutubeUserActivityCollector(
+ YouTube youtube,
+ BlockingQueue<StreamsDatum> datumQueue,
+ BackOffStrategy backOff,
+ UserInfo userInfo,
+ YoutubeConfiguration config) {
+ this.youtube = youtube;
+ this.datumQueue = datumQueue;
+ this.backOff = backOff;
+ this.userInfo = userInfo;
+ this.config = config;
+ }
+
+ @Override
+ public void run() {
+ collectActivityData();
+ }
+
+ /**
+ * Iterate through all users in the Youtube configuration and collect all videos
+ * associated with their accounts.
+ */
+ protected void collectActivityData() {
+ try {
+ YouTube.Activities.List request = null;
+ ActivityListResponse feed = null;
+
+ boolean tryAgain = false;
+ int attempt = 0;
+ DateTime afterDate = userInfo.getAfterDate();
+ DateTime beforeDate = userInfo.getBeforeDate();
+
+ do {
+ try {
+ if (request == null) {
+ request = this.youtube.activities().list("contentDetails")
+ .setChannelId(userInfo.getUserId())
+ .setMaxResults(MAX_RESULTS)
+ .setKey(config.getApiKey());
+ feed = request.execute();
+ } else {
+ request = this.youtube.activities().list("contentDetails")
+ .setChannelId(userInfo.getUserId())
+ .setMaxResults(MAX_RESULTS)
+ .setPageToken(feed.getNextPageToken())
+ .setKey(config.getApiKey());
+ feed = request.execute();
+ }
+ this.backOff.reset(); //successful pull reset api.
+
+ processActivityFeed(feed, afterDate, beforeDate);
+ } catch (GoogleJsonResponseException gjre) {
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
+ ++attempt;
+ }
+ }
+ while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
+ } catch (Throwable throwable) {
+ if (throwable instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throwable.printStackTrace();
+ LOGGER.warn("Unable to pull Activities for user={} : {}", this.userInfo.getUserId(), throwable);
+ }
+ }
+
+ /**
+ * Given a feed and an after and before date, fetch all relevant user videos
+ * and place them into the datumQueue for post-processing.
+ * @param feed ActivityListResponse
+ * @param afterDate DateTime
+ * @param beforeDate DateTime
+ * @throws IOException IOException
+ * @throws InterruptedException InterruptedException
+ */
+ void processActivityFeed(ActivityListResponse feed, DateTime afterDate, DateTime beforeDate) throws IOException, InterruptedException {
+ for (com.google.api.services.youtube.model.Activity activity : feed.getItems()) {
+ try {
+ List<Video> videos = new ArrayList<>();
+
+ if (activity.getContentDetails().getUpload() != null) {
+ videos.addAll(getVideoList(activity.getContentDetails().getUpload().getVideoId()));
+ }
+ if (activity.getContentDetails().getPlaylistItem() != null && activity.getContentDetails().getPlaylistItem().getResourceId() != null) {
+ videos.addAll(getVideoList(activity.getContentDetails().getPlaylistItem().getResourceId().getVideoId()));
+ }
+
+ processVideos(videos, afterDate, beforeDate, activity, feed);
+ } catch (Exception ex) {
+ LOGGER.error("Error while trying to process activity: {}, {}", activity, ex);
+ }
+ }
+ }
+
+ /**
+ * Process a list of Video objects.
+ * @param videos List of Video
+ * @param afterDate afterDate
+ * @param beforeDate beforeDate
+ * @param activity com.google.api.services.youtube.model.Activity
+ * @param feed ActivityListResponse
+ */
+ void processVideos(List<Video> videos, DateTime afterDate, DateTime beforeDate, com.google.api.services.youtube.model.Activity activity, ActivityListResponse feed) {
+ try {
+ for (Video video : videos) {
+ if (video != null) {
+ org.joda.time.DateTime published = new org.joda.time.DateTime(video.getSnippet().getPublishedAt().getValue());
+ if ((afterDate == null && beforeDate == null)
+ || (beforeDate == null && afterDate.isBefore(published))
+ || (afterDate == null && beforeDate.isAfter(published))
+ || ((afterDate != null && beforeDate != null) && (afterDate.isAfter(published) && beforeDate.isBefore(published)))) {
+ LOGGER.debug("Providing Youtube Activity: {}", MAPPER.writeValueAsString(video));
+ this.datumQueue.put(new StreamsDatum(gson.toJson(video), activity.getId()));
+ } else if (afterDate != null && afterDate.isAfter(published)) {
+ feed.setNextPageToken(null); // do not fetch next page
+ break;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to process video list: {}, {}", videos, ex);
+ }
+ }
+
+ /**
+ * Given a Youtube videoId, return the relevant Youtube Video object.
+ * @param videoId videoId
+ * @return List of Videos
+ * @throws IOException
+ */
+ List<Video> getVideoList(String videoId) throws IOException {
+ VideoListResponse videosListResponse = this.youtube.videos().list("snippet,statistics")
+ .setId(videoId)
+ .setKey(config.getApiKey())
+ .execute();
+
+ if (videosListResponse.getItems().size() == 0) {
+ LOGGER.debug("No Youtube videos found for videoId: {}", videoId);
+ return new ArrayList<>();
+ }
+
+ return videosListResponse.getItems();
+ }
+
+ BlockingQueue<StreamsDatum> getDatumQueue() {
+ return this.datumQueue;
+ }
+}
\ No newline at end of file
[7/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java
new file mode 100644
index 0000000..7c07c70
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.ComponentUtils;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.plus.Plus;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.gson.Gson;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Provider that creates a GPlus client and will run task that queue data to an outing queue.
+ */
+public abstract class AbstractGPlusProvider implements StreamsProvider {
+
+ public static final String STREAMS_ID = "AbstractGPlusProvider";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractGPlusProvider.class);
+ private static final Set<String> SCOPE = new HashSet<String>() {
+ {
+ add("https://www.googleapis.com/auth/plus.login");
+ }
+ };
+ private static final int MAX_BATCH_SIZE = 1000;
+
+ private static final HttpTransport TRANSPORT = new NetHttpTransport();
+ private static final JacksonFactory JSON_FACTORY = new JacksonFactory();
+ private static final Gson GSON = new Gson();
+
+ private GPlusConfiguration config;
+
+ List<ListenableFuture<Object>> futures = new ArrayList<>();
+
+ private ListeningExecutorService executor;
+
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private AtomicBoolean isComplete;
+ private boolean previousPullWasEmpty;
+
+ protected GoogleCredential credential;
+ protected Plus plus;
+
+ public AbstractGPlusProvider() {
+ this.config = new ComponentConfigurator<>(GPlusConfiguration.class)
+ .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gplus"));
+ }
+
+ public AbstractGPlusProvider(GPlusConfiguration config) {
+ this.config = config;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+
+ Objects.requireNonNull(config.getOauth().getPathToP12KeyFile());
+ Objects.requireNonNull(config.getOauth().getAppName());
+ Objects.requireNonNull(config.getOauth().getServiceAccountEmailAddress());
+
+ try {
+ this.plus = createPlusClient();
+ } catch (IOException | GeneralSecurityException ex) {
+ LOGGER.error("Failed to created oauth for GPlus : {}", ex);
+ throw new RuntimeException(ex);
+ }
+ // GPlus rate limits you to 5 calls per second, so there is not a need to execute more than one
+ // collector unless you have multiple oauth tokens
+ //TODO make this configurable based on the number of oauth tokens
+ this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
+ this.datumQueue = new LinkedBlockingQueue<>(1000);
+ this.isComplete = new AtomicBoolean(false);
+ this.previousPullWasEmpty = false;
+ }
+
+ @Override
+ public void startStream() {
+
+ BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
+ for (UserInfo user : this.config.getGooglePlusUsers()) {
+ if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ }
+ if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ }
+ this.executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.plus, user));
+ }
+ this.executor.shutdown();
+ }
+
+ protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo);
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ public StreamsResultSet readCurrent() {
+ BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
+ int batchCount = 0;
+ while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
+ StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
+ if (datum != null) {
+ ++batchCount;
+ ComponentUtils.offerUntilSuccess(datum, batch);
+ }
+ }
+ boolean pullIsEmpty = batch.isEmpty() && this.datumQueue.isEmpty() && this.executor.isTerminated();
+ this.isComplete.set(this.previousPullWasEmpty && pullIsEmpty);
+ this.previousPullWasEmpty = pullIsEmpty;
+ return new StreamsResultSet(batch);
+ }
+
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return null;
+ }
+
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return null;
+ }
+
+ @VisibleForTesting
+ protected Plus createPlusClient() throws IOException, GeneralSecurityException {
+ credential = new GoogleCredential.Builder()
+ .setJsonFactory(JSON_FACTORY)
+ .setTransport(TRANSPORT)
+ .setServiceAccountScopes(SCOPE)
+ .setServiceAccountId(this.config.getOauth().getServiceAccountEmailAddress())
+ .setServiceAccountPrivateKeyFromP12File(new File(this.config.getOauth().getPathToP12KeyFile()))
+ .build();
+ return new Plus.Builder(TRANSPORT,JSON_FACTORY, credential).setApplicationName(this.config.getOauth().getAppName()).build();
+ }
+
+ @Override
+ public void cleanUp() {
+ ComponentUtils.shutdownExecutor(this.executor, 10, 10);
+ this.executor = null;
+ }
+
+ public GPlusConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(GPlusConfiguration config) {
+ this.config = config;
+ }
+
+ /**
+ * Set and overwrite the default before date that was read from the configuration file.
+ * @param defaultBeforeDate defaultBeforeDate
+ */
+ public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
+ this.config.setDefaultBeforeDate(defaultBeforeDate);
+ }
+
+ /**
+ * Set and overwrite the default after date that was read from teh configuration file.
+ * @param defaultAfterDate defaultAfterDate
+ */
+ public void setDefaultAfterDate(DateTime defaultAfterDate) {
+ this.config.setDefaultAfterDate(defaultAfterDate);
+ }
+
+ /**
+ * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
+ * @param userIds userIds
+ */
+ public void setUserInfoWithDefaultDates(Set<String> userIds) {
+ List<UserInfo> gplusUsers = new LinkedList<>();
+ for (String userId : userIds) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(this.config.getDefaultAfterDate());
+ user.setBeforeDate(this.config.getDefaultBeforeDate());
+ gplusUsers.add(user);
+ }
+ this.config.setGooglePlusUsers(gplusUsers);
+ }
+
+ /**
+ * Set and overwrite user into from the configuration file. Only sets after date.
+ * @param usersAndAfterDates usersAndAfterDates
+ */
+ public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
+ List<UserInfo> gplusUsers = new LinkedList<>();
+ for (String userId : usersAndAfterDates.keySet()) {
+ UserInfo user = new UserInfo();
+ user.setUserId(userId);
+ user.setAfterDate(usersAndAfterDates.get(userId));
+ gplusUsers.add(user);
+ }
+ this.config.setGooglePlusUsers(gplusUsers);
+ }
+
+ @Override
+ public boolean isRunning() {
+ if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
+ LOGGER.info("Completed");
+ isComplete.set(true);
+ LOGGER.info("Exiting");
+ }
+ return !isComplete.get();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java
new file mode 100644
index 0000000..74b0d8a
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.pojo.json.Activity;
+
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.commons.lang.NotImplementedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * GPlusActivitySerializer converts gplus activities to as1 activities.
+ */
+public class GPlusActivitySerializer implements ActivitySerializer<com.google.api.services.plus.model.Activity> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
+
+ AbstractGPlusProvider provider;
+
+ public GPlusActivitySerializer(AbstractGPlusProvider provider) {
+
+ this.provider = provider;
+ }
+
+ public GPlusActivitySerializer() {
+ }
+
+ @Override
+ public String serializationFormat() {
+ return "gplus.v1";
+ }
+
+ @Override
+ public com.google.api.services.plus.model.Activity serialize(Activity deserialized) {
+ throw new NotImplementedException("Not currently implemented");
+ }
+
+ @Override
+ public Activity deserialize(com.google.api.services.plus.model.Activity gplusActivity) {
+ Activity activity = new Activity();
+
+ GooglePlusActivityUtil.updateActivity(gplusActivity, activity);
+ return activity;
+ }
+
+ @Override
+ public List<Activity> deserializeAll(List<com.google.api.services.plus.model.Activity> serializedList) {
+ throw new NotImplementedException("Not currently implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java
new file mode 100644
index 0000000..3726bc5
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.util.api.requests.backoff.BackOffException;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * GPlusDataCollector collects GPlus Data on behalf of providers.
+ */
+public abstract class GPlusDataCollector implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusDataCollector.class);
+
+ /**
+ * Looks at the status code of the exception. If the code indicates that the request should be retried,
+ * it executes the back off strategy and returns true.
+ * @param gjre GoogleJsonResponseException
+ * @param backOff BackOffStrategy
+ * @return returns true if the error code of the exception indicates the request should be retried.
+ */
+ public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
+ boolean tryAgain = false;
+ switch (gjre.getStatusCode()) {
+ case 400 :
+ LOGGER.warn("Bad Request : {}", gjre);
+ break;
+ case 401 :
+ LOGGER.warn("Invalid Credentials : {}", gjre);
+ break;
+ case 403 :
+ LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
+ backOff.backOff();
+ tryAgain = true;
+ break;
+ case 503 :
+ LOGGER.warn("Google Backend Service Error : {}", gjre);
+ break;
+ default:
+ LOGGER.warn("Google Service returned error : {}", gjre);
+ tryAgain = true;
+ backOff.backOff();
+ break;
+ }
+ return tryAgain;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java
new file mode 100644
index 0000000..e253a18
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.ActivityFeed;
+import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Collects the public activities of a GPlus user. Has ability to filter by date ranges.
+ */
+public class GPlusUserActivityCollector extends GPlusDataCollector {
+
+ /**
+ * Key for all public activities
+ * https://developers.google.com/+/api/latest/activities/list
+ */
+ private static final String PUBLIC_COLLECTION = "public";
+ /**
+ * Max results allowed per request
+ * https://developers.google.com/+/api/latest/activities/list
+ */
+ private static final long MAX_RESULTS = 100;
+ private static final int MAX_ATTEMPTS = 5;
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityCollector.class);
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+
+ static { //set up mapper for Google Activity Object
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
+ simpleModule.addSerializer(
+ com.google.api.client.util.DateTime.class,
+ new StdSerializer<com.google.api.client.util.DateTime>(com.google.api.client.util.DateTime.class) {
+ @Override
+ public void serialize(
+ com.google.api.client.util.DateTime dateTime,
+ JsonGenerator jsonGenerator,
+ SerializerProvider serializerProvider)
+ throws IOException {
+ jsonGenerator.writeString(dateTime.toStringRfc3339());
+ }
+ });
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private BackOffStrategy backOff;
+ private Plus plus;
+ private UserInfo userInfo;
+
+ /**
+ * GPlusUserActivityCollector constructor.
+ * @param plus Plus
+ * @param datumQueue BlockingQueue<StreamsDatum>
+ * @param backOff BackOffStrategy
+ * @param userInfo UserInfo
+ */
+ public GPlusUserActivityCollector(Plus plus, BlockingQueue<StreamsDatum> datumQueue, BackOffStrategy backOff, UserInfo userInfo) {
+ this.plus = plus;
+ this.datumQueue = datumQueue;
+ this.backOff = backOff;
+ this.userInfo = userInfo;
+ }
+
+ @Override
+ public void run() {
+ collectActivityData();
+ }
+
+ protected void collectActivityData() {
+ try {
+ ActivityFeed feed = null;
+ boolean tryAgain = false;
+ int attempt = 0;
+ DateTime afterDate = userInfo.getAfterDate();
+ DateTime beforeDate = userInfo.getBeforeDate();
+ do {
+ try {
+ if (feed == null) {
+ feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
+ .setMaxResults(MAX_RESULTS).execute();
+ } else {
+ feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
+ .setMaxResults(MAX_RESULTS)
+ .setPageToken(feed.getNextPageToken()).execute();
+ }
+ this.backOff.reset(); //successful pull reset api.
+ for (com.google.api.services.plus.model.Activity activity : feed.getItems()) {
+ DateTime published = new DateTime(activity.getPublished().getValue());
+ if ((afterDate == null && beforeDate == null)
+ || (beforeDate == null && afterDate.isBefore(published))
+ || (afterDate == null && beforeDate.isAfter(published))
+ || ((afterDate != null && beforeDate != null) && (afterDate.isBefore(published) && beforeDate.isAfter(published)))) {
+ String json = MAPPER.writeValueAsString(activity);
+ this.datumQueue.put(new StreamsDatum(json, activity.getId()));
+ } else if (afterDate != null && afterDate.isAfter(published)) {
+ feed.setNextPageToken(null); // do not fetch next page
+ break;
+ }
+ }
+ } catch (GoogleJsonResponseException gjre) {
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
+ ++attempt;
+ }
+ }
+ while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
+ } catch (Throwable th) {
+ if (th instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ th.printStackTrace();
+ LOGGER.warn("Unable to pull Activities for user={} : {}",this.userInfo.getUserId(), th);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java
new file mode 100644
index 0000000..679660e
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.google.api.services.plus.Plus;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Retrieve recent activity from a list of accounts.
+ *
+ * <p/>
+ * To use from command line:
+ *
+ * <p/>
+ * Supply (at least) the following required configuration in application.conf:
+ *
+ * <p/>
+ * gplus.oauth.pathToP12KeyFile
+ * gplus.oauth.serviceAccountEmailAddress
+ * gplus.apiKey
+ * gplus.googlePlusUsers
+ *
+ * <p/>
+ * Launch using:
+ *
+ * <p/>
+ * mvn exec:java -Dexec.mainClass=org.apache.streams.gplus.provider.GPlusUserActivityProvider -Dexec.args="application.conf activity.json"
+ */
+public class GPlusUserActivityProvider extends AbstractGPlusProvider {
+
+ private static final String STREAMS_ID = "GPlusUserActivityProvider";
+
+ public GPlusUserActivityProvider() {
+ super();
+ }
+
+ public GPlusUserActivityProvider(GPlusConfiguration config) {
+ super(config);
+ }
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ return new GPlusUserActivityCollector(plus, queue, strategy, userInfo);
+ }
+
+ /**
+ * Retrieve recent activity from a list of accounts.
+ * @param args args
+ * @throws Exception Exception
+ */
+ public static void main(String[] args) throws Exception {
+
+ Preconditions.checkArgument(args.length >= 2);
+
+ String configfile = args[0];
+ String outfile = args[1];
+
+ Config reference = ConfigFactory.load();
+ File file = new File(configfile);
+ assert (file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
+
+ Config typesafe = testResourceConfig.withFallback(reference).resolve();
+
+ StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
+ GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
+ GPlusUserActivityProvider provider = new GPlusUserActivityProvider(config);
+
+ Gson gson = new Gson();
+
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
+ provider.prepare(config);
+ provider.startStream();
+ do {
+ Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
+ for (StreamsDatum datum : provider.readCurrent()) {
+ String json;
+ if (datum.getDocument() instanceof String) {
+ json = (String) datum.getDocument();
+ } else {
+ json = gson.toJson(datum.getDocument());
+ }
+ outStream.println(json);
+ }
+ }
+ while ( provider.isRunning());
+ provider.cleanUp();
+ outStream.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java
new file mode 100644
index 0000000..2cbbc8c
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Person;
+import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Collects user profile information for a specific GPlus user.
+ */
+public class GPlusUserDataCollector extends GPlusDataCollector {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
+ private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+ private static final int MAX_ATTEMPTS = 5;
+
+ static { //set up Mapper for Person objects
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ MAPPER.registerModule(simpleModule);
+ MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ private BackOffStrategy backOffStrategy;
+ private Plus plus;
+ private BlockingQueue<StreamsDatum> datumQueue;
+ private UserInfo userInfo;
+
+ /**
+ * GPlusUserDataCollector constructor.
+ * @param plus Plus
+ * @param backOffStrategy BackOffStrategy
+ * @param datumQueue BlockingQueue of StreamsDatum
+ * @param userInfo UserInfo
+ */
+ public GPlusUserDataCollector(Plus plus, BackOffStrategy backOffStrategy, BlockingQueue<StreamsDatum> datumQueue, UserInfo userInfo) {
+ this.plus = plus;
+ this.backOffStrategy = backOffStrategy;
+ this.datumQueue = datumQueue;
+ this.userInfo = userInfo;
+ }
+
+ protected void queueUserHistory() {
+ try {
+ boolean tryAgain;
+ int attempts = 0;
+ com.google.api.services.plus.model.Person person = null;
+ do {
+ try {
+ person = this.plus.people().get(userInfo.getUserId()).execute();
+ this.backOffStrategy.reset();
+ tryAgain = person == null;
+ } catch (GoogleJsonResponseException gjre) {
+ tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOffStrategy);
+ }
+ ++attempts;
+ }
+ while (tryAgain && attempts < MAX_ATTEMPTS);
+ String json = MAPPER.writeValueAsString(person);
+ this.datumQueue.put(new StreamsDatum(json, person.getId()));
+ } catch (Throwable throwable) {
+ LOGGER.warn("Unable to pull user data for user={} : {}", userInfo.getUserId(), throwable);
+ if (throwable instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ queueUserHistory();
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java
new file mode 100644
index 0000000..14bf472
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.provider;
+
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.GPlusConfiguration;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+
+import com.google.api.services.plus.Plus;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * Retrieve current profile status for a list of accounts.
+ *
+ * <p/>
+ * To use from command line:
+ *
+ * <p/>
+ * Supply (at least) the following required configuration in application.conf:
+ *
+ * <p/>
+ * gplus.oauth.pathToP12KeyFile
+ * gplus.oauth.serviceAccountEmailAddress
+ * gplus.apiKey
+ * gplus.googlePlusUsers
+ *
+ * <p/>
+ * Launch using:
+ *
+ * <p/>
+ * mvn exec:java -Dexec.mainClass=org.apache.streams.gplus.provider.GPlusUserDataProvider -Dexec.args="application.conf profiles.json"
+ */
+public class GPlusUserDataProvider extends AbstractGPlusProvider {
+
+ public static final String STREAMS_ID = "GPlusUserDataProvider";
+
+ public GPlusUserDataProvider() {
+ super();
+ }
+
+ public GPlusUserDataProvider(GPlusConfiguration config) {
+ super(config);
+ }
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
+ return new GPlusUserDataCollector(plus, strategy, queue, userInfo);
+ }
+
+ /**
+ * Retrieve current profile status for a list of accounts.
+ * @param args args
+ * @throws Exception Exception
+ */
+ public static void main(String[] args) throws Exception {
+
+ MatcherAssert.assertThat(args.length, Matchers.greaterThanOrEqualTo(2));
+
+ String configfile = args[0];
+ String outfile = args[1];
+
+ Config reference = ConfigFactory.load();
+ File file = new File(configfile);
+ assert (file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
+
+ Config typesafe = testResourceConfig.withFallback(reference).resolve();
+
+ StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
+ GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
+ GPlusUserDataProvider provider = new GPlusUserDataProvider(config);
+
+ Gson gson = new Gson();
+
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
+ provider.prepare(config);
+ provider.startStream();
+ do {
+ Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
+ for (StreamsDatum datum : provider.readCurrent()) {
+ String json;
+ if (datum.getDocument() instanceof String) {
+ json = (String) datum.getDocument();
+ } else {
+ json = gson.toJson(datum.getDocument());
+ }
+ outStream.println(json);
+ }
+ }
+ while ( provider.isRunning());
+ provider.cleanUp();
+ outStream.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java
new file mode 100644
index 0000000..585e511
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.api.client.util.DateTime;
+import com.google.api.services.plus.model.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Custom deserializer for GooglePlus' Person model.
+ */
+public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
+
+ /**
+ * Because the GooglePlus Activity object {@link com.google.api.services.plus.model.Activity} contains complex objects
+ * within its hierarchy, we have to use a custom deserializer
+ *
+ * @param jsonParser jsonParser
+ * @param deserializationContext deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Activity} object
+ * @throws IOException IOException
+ * @throws JsonProcessingException JsonProcessingException
+ */
+ @Override
+ public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Activity activity = new Activity();
+
+ try {
+ activity.setUrl(node.get("url").asText());
+ activity.setEtag(node.get("etag").asText());
+ activity.setTitle(node.get("title").asText());
+ activity.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
+ activity.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
+ activity.setId(node.get("id").asText());
+ activity.setVerb(node.get("verb").asText());
+
+ activity.setActor(buildActor(node));
+
+ activity.setObject(buildPlusObject(node));
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
+ }
+
+ return activity;
+ }
+
+ /**
+ * Given a raw JsonNode, build out the G+ {@link com.google.api.services.plus.model.Activity.Actor} object
+ *
+ * @param node node
+ * @return {@link com.google.api.services.plus.model.Activity.Actor} object
+ */
+ private Activity.Actor buildActor(JsonNode node) {
+ Activity.Actor actor = new Activity.Actor();
+ JsonNode actorNode = node.get("actor");
+
+ actor.setId(actorNode.get("id").asText());
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Activity.Actor.Image image = new Activity.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+
+ return actor;
+ }
+
+ /**
+ * Given a JsonNode, build out all aspects of the {@link com.google.api.services.plus.model.Activity.PlusObject} object
+ *
+ * @param node node
+ * @return {@link com.google.api.services.plus.model.Activity.PlusObject} object
+ */
+ private Activity.PlusObject buildPlusObject(JsonNode node) {
+ Activity.PlusObject object = new Activity.PlusObject();
+ JsonNode objectNode = node.get("object");
+ object.setObjectType(objectNode.get("objectType").asText());
+ object.setContent(objectNode.get("content").asText());
+ object.setUrl(objectNode.get("url").asText());
+
+ Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
+ JsonNode repliesNode = objectNode.get("replies");
+ replies.setTotalItems(repliesNode.get("totalItems").asLong());
+ replies.setSelfLink(repliesNode.get("selfLink").asText());
+ object.setReplies(replies);
+
+ Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
+ JsonNode plusonersNode = objectNode.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
+ object.setPlusoners(plusoners);
+
+ Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
+ JsonNode resharersNode = objectNode.get("resharers");
+ resharers.setTotalItems(resharersNode.get("totalItems").asLong());
+ resharers.setSelfLink(resharersNode.get("selfLink").asText());
+ object.setResharers(resharers);
+
+ object.setAttachments(buildAttachments(objectNode));//attachments);
+
+ return object;
+ }
+
+ /**
+ * Given a raw JsonNode representation of an Activity's attachments, build out that
+ * list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
+ *
+ * @param objectNode objectNode
+ * @return list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
+ */
+ private List<Activity.PlusObject.Attachments> buildAttachments(JsonNode objectNode) {
+ List<Activity.PlusObject.Attachments> attachments = new ArrayList<>();
+ if ( objectNode.has("attachments") ) {
+ for (JsonNode attachmentNode : objectNode.get("attachments")) {
+ Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
+ attachments1.setObjectType(attachmentNode.get("objectType").asText());
+ if (attachmentNode.has("displayName")) {
+ attachments1.setDisplayName(attachmentNode.get("displayName").asText());
+ }
+ if (attachmentNode.has("content")) {
+ attachments1.setContent(attachmentNode.get("content").asText());
+ }
+ if (attachmentNode.has("url")) {
+ attachments1.setUrl(attachmentNode.get("url").asText());
+ }
+
+ if( attachmentNode.has("image")) {
+ Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
+ JsonNode imageNode1 = attachmentNode.get("image");
+ image1.setUrl(imageNode1.get("url").asText());
+ attachments1.setImage(image1);
+ }
+
+ attachments.add(attachments1);
+ }
+ }
+ return attachments;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java
new file mode 100644
index 0000000..5d6a982
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.util.DateTime;
+import com.google.api.services.plus.model.Comment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * GPlusCommentDeserializer converts gplus comments to as1 comments.
+ */
+public class GPlusCommentDeserializer extends JsonDeserializer<Comment> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
+
+ /**
+ * Because the GooglePlus Comment object {@link com.google.api.services.plus.model.Comment} contains complex objects
+ * within its hierarchy, we have to use a custom deserializer
+ *
+ * @param jsonParser jsonParser
+ * @param deserializationContext deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Comment} object
+ * @throws java.io.IOException IOException
+ * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
+ */
+ @Override
+ public Comment deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
+ throws IOException {
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
+ Comment comment = new Comment();
+
+ try {
+ comment.setEtag(node.get("etag").asText());
+ comment.setVerb(node.get("verb").asText());
+ comment.setId(node.get("id").asText());
+ comment.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
+ comment.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
+
+ Comment.Actor actor = new Comment.Actor();
+ JsonNode actorNode = node.get("actor");
+ actor.setDisplayName(actorNode.get("displayName").asText());
+ actor.setUrl(actorNode.get("url").asText());
+
+ Comment.Actor.Image image = new Comment.Actor.Image();
+ JsonNode imageNode = actorNode.get("image");
+ image.setUrl(imageNode.get("url").asText());
+
+ actor.setImage(image);
+
+ comment.setObject(objectMapper.readValue(objectMapper.writeValueAsString(node.get("object")), Comment.PlusObject.class));
+
+ comment.setSelfLink(node.get("selfLink").asText());
+
+ List<Comment.InReplyTo> replies = new ArrayList<>();
+ for (JsonNode reply : node.get("inReplyTo")) {
+ Comment.InReplyTo irt = objectMapper.readValue(objectMapper.writeValueAsString(reply), Comment.InReplyTo.class);
+ replies.add(irt);
+ }
+
+ comment.setInReplyTo(replies);
+
+ Comment.Plusoners plusoners = new Comment.Plusoners();
+ JsonNode plusonersNode = node.get("plusoners");
+ plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
+ comment.setPlusoners(plusoners);
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
+ }
+
+ return comment;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java
new file mode 100644
index 0000000..f6fa524
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.Person;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * GPlusEventClassifier classifies GPlus Events.
+ */
+public class GPlusEventClassifier implements Serializable {
+
+ private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+ private static final String ACTIVITY_IDENTIFIER = "\"plus#activity\"";
+ private static final String PERSON_IDENTIFIER = "\"plus#person\"";
+
+ /**
+ * Detect likely class of String json.
+ * @param json String json
+ * @return likely class
+ */
+ public static Class detectClass(String json) {
+ Objects.requireNonNull(json);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(json));
+
+ ObjectNode objectNode;
+ try {
+ objectNode = (ObjectNode) mapper.readTree(json);
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ return null;
+ }
+
+ if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(ACTIVITY_IDENTIFIER)) {
+ return Activity.class;
+ } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(PERSON_IDENTIFIER)) {
+ return Person.class;
+ } else {
+ return ObjectNode.class;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java
new file mode 100644
index 0000000..8f7ae72
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.services.plus.model.Person;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Custom deserializer for GooglePlus' Person model.
+ */
+public class GPlusPersonDeserializer extends JsonDeserializer<Person> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GPlusPersonDeserializer.class);
+
+ /**
+ * Because the GooglePlus Person object contains complex objects within its hierarchy, we have to use
+ * a custom deserializer
+ *
+ * @param jsonParser jsonParser
+ * @param deserializationContext deserializationContext
+ * @return The deserialized {@link com.google.api.services.plus.model.Person} object
+ * @throws IOException IOException
+ * @throws JsonProcessingException JsonProcessingException
+ */
+ @Override
+ public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
+ ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+ Person person = new Person();
+ try {
+ person.setId(node.get("id").asText());
+ person.setCircledByCount((Integer) (node.get("circledByCount")).numberValue());
+ person.setDisplayName(node.get("displayName").asText());
+ if( node.has("etag")) {
+ person.setEtag(node.get("etag").asText());
+ }
+ if( node.has("gender")) {
+ person.setGender(node.get("gender").asText());
+ }
+
+ Person.Image image = new Person.Image();
+ if( node.has("image") ) {
+ JsonNode imageNode = node.get("image");
+ image.setIsDefault(imageNode.get("isDefault").asBoolean());
+ image.setUrl(imageNode.get("url").asText());
+ person.setImage(image);
+ }
+
+ person.setIsPlusUser(node.get("isPlusUser").asBoolean());
+ person.setKind(node.get("kind").asText());
+
+ JsonNode nameNode = node.get("name");
+ Person.Name name = mapper.readValue(mapper.writeValueAsString(nameNode), Person.Name.class);
+ person.setName(name);
+
+ person.setObjectType(node.get("objectType").asText());
+
+ List<Person.Organizations> organizations = new ArrayList<>();
+ if( node.has("organizations")) {
+ for (JsonNode orgNode : node.get("organizations")) {
+ Person.Organizations org = mapper.readValue(mapper.writeValueAsString(orgNode), Person.Organizations.class);
+ organizations.add(org);
+ }
+ person.setOrganizations(organizations);
+ }
+
+ person.setUrl(node.get("url").asText());
+ person.setVerified(node.get("verified").asBoolean());
+
+ List<Person.Emails> emails = new ArrayList<>();
+
+ if ( node.has("emails")) {
+ for (JsonNode emailNode : node.get("emails")) {
+ Person.Emails email = mapper.readValue(mapper.writeValueAsString(emailNode), Person.Emails.class);
+ emails.add(email);
+ }
+ }
+
+ if ( node.has("tagline")) {
+ person.setTagline(node.get("tagline").asText());
+ }
+ if ( node.has("aboutMe")) {
+ person.setAboutMe(node.get("aboutMe").asText());
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserialize a Person object: {}", ex);
+ }
+
+ return person;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java
new file mode 100644
index 0000000..7039266
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.pojo.extensions.ExtensionUtil;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Image;
+import org.apache.streams.pojo.json.Provider;
+
+import com.google.api.services.plus.model.Comment;
+import com.google.api.services.plus.model.Person;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * GooglePlusActivityUtil helps convert c.g.Person and c.g.Activity into o.a.s.p.j.o.Page and o.a.s.p.j.Activity.
+ */
+public class GooglePlusActivityUtil {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivityUtil.class);
+
+ /**
+ * Given a {@link Person} object and an
+ * {@link Activity} object, fill out the appropriate details.
+ *
+ * @param item Person
+ * @param activity Activity
+ * @throws ActivitySerializerException ActivitySerializerException
+ */
+ public static void updateActivity(Person item, Activity activity) throws ActivitySerializerException {
+ activity.setActor(buildActor(item));
+ activity.setVerb("update");
+
+ activity.setId(formatId(activity.getVerb(), Optional.ofNullable(item.getId()).orElse(null)));
+
+ activity.setProvider(getProvider());
+ }
+
+ /**
+ * Given a {@link List} of {@link Comment} objects and an
+ * {@link Activity}, update that Activity to contain all comments
+ *
+ * @param comments input List of Comment
+ * @param activity output Activity
+ */
+ public static void updateActivity(List<Comment> comments, Activity activity) {
+ for (Comment comment : comments) {
+ addComment(activity, comment);
+ }
+
+ Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
+ extensions.put("comment_count", comments.size());
+ }
+
+ /**
+ * Given a Google Plus {@link com.google.api.services.plus.model.Activity},
+ * convert that into an Activity streams formatted {@link Activity}
+ *
+ * @param gPlusActivity input c.g.a.s.p.m.Activity
+ * @param activity output o.a.s.p.j.Activity
+ */
+ public static void updateActivity(com.google.api.services.plus.model.Activity gPlusActivity, Activity activity) {
+ activity.setActor(buildActor(gPlusActivity.getActor()));
+ activity.setVerb("post");
+ activity.setTitle(gPlusActivity.getTitle());
+ activity.setUrl(gPlusActivity.getUrl());
+ activity.setProvider(getProvider());
+
+ if (gPlusActivity.getObject() != null) {
+ activity.setContent(gPlusActivity.getObject().getContent());
+ }
+
+ activity.setId(formatId(activity.getVerb(), Optional.ofNullable(gPlusActivity.getId()).orElse(null)));
+
+ DateTime published = new DateTime(String.valueOf(gPlusActivity.getPublished()));
+ activity.setPublished(published);
+
+ setObject(activity, gPlusActivity.getObject());
+ addGPlusExtensions(activity, gPlusActivity);
+ }
+
+ /**
+ * Adds a single {@link Comment} to the Object.Attachments
+ * section of the passed in {@link Activity}
+ *
+ * @param activity output o.a.s.p.j.Activity
+ * @param comment input c.g.a.s.p.m.Comment
+ */
+ private static void addComment(Activity activity, Comment comment) {
+ ActivityObject obj = new ActivityObject();
+
+ obj.setId(comment.getId());
+ obj.setPublished(new DateTime(String.valueOf(comment.getPublished())));
+ obj.setUpdated(new DateTime(String.valueOf(comment.getUpdated())));
+ obj.setContent(comment.getObject().getContent());
+ obj.setObjectType(comment.getObject().getObjectType());
+
+ Map<String, Object> extensions = new HashMap<>();
+ extensions.put("googlePlus", comment);
+
+ obj.setAdditionalProperty("extensions", extensions);
+
+ if (activity.getObject() == null) {
+ activity.setObject(new ActivityObject());
+ }
+ if (activity.getObject().getAttachments() == null) {
+ activity.getObject().setAttachments(new ArrayList<>());
+ }
+
+ activity.getObject().getAttachments().add(obj);
+ }
+
+ /**
+ * Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
+ * {@link Activity} object
+ *
+ * @param activity output o.a.s.p.j.Activity
+ * @param gPlusActivity input c.g.a.s.p.m.Activity
+ */
+ private static void addGPlusExtensions(Activity activity, com.google.api.services.plus.model.Activity gPlusActivity) {
+
+ activity.getAdditionalProperties().put("googlePlus", gPlusActivity);
+
+ Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
+
+ com.google.api.services.plus.model.Activity.PlusObject object = gPlusActivity.getObject();
+
+ if (object != null) {
+ com.google.api.services.plus.model.Activity.PlusObject.Plusoners plusoners = object.getPlusoners();
+ if (plusoners != null) {
+ Map<String, Object> likes = new HashMap<>();
+ likes.put("count", plusoners.getTotalItems());
+ extensions.put("likes", likes);
+ }
+
+ com.google.api.services.plus.model.Activity.PlusObject.Resharers resharers = object.getResharers();
+ if (resharers != null) {
+ Map<String, Object> rebroadcasts = new HashMap<>();
+ rebroadcasts.put("count", resharers.getTotalItems());
+ extensions.put("rebroadcasts", rebroadcasts);
+ }
+
+ extensions.put("keywords", object.getContent());
+ }
+ }
+
+ /**
+ * Set the {@link ActivityObject} field given the passed in
+ * {@link com.google.api.services.plus.model.Activity.PlusObject}
+ *
+ * @param activity output $.object as o.a.s.p.j.ActivityObject
+ * @param plusObject input c.g.a.s.p.m.Activity.PlusObject
+ */
+ private static void setObject(Activity activity, com.google.api.services.plus.model.Activity.PlusObject plusObject) {
+ if (plusObject != null) {
+ ActivityObject activityObject = new ActivityObject();
+
+ activityObject.setContent(plusObject.getContent());
+ activityObject.setObjectType(plusObject.getObjectType());
+
+ List<ActivityObject> attachmentsList = new ArrayList<>();
+ for (com.google.api.services.plus.model.Activity.PlusObject.Attachments attachments : plusObject.getAttachments()) {
+ ActivityObject attach = new ActivityObject();
+
+ attach.setContent(attachments.getContent());
+ attach.setDisplayName(attachments.getDisplayName());
+ attach.setObjectType(attachments.getObjectType());
+ attach.setUrl(attachments.getUrl());
+
+ Image image = new Image();
+ com.google.api.services.plus.model.Activity.PlusObject.Attachments.Image image1 = attachments.getImage();
+
+ if (image1 != null) {
+ image.setUrl(image1.getUrl());
+ attach.setImage(image);
+ }
+
+ attachmentsList.add(attach);
+ }
+
+ activityObject.setAttachments(attachmentsList);
+
+ activity.setObject(activityObject);
+ }
+ }
+
+ /**
+ * Given a {@link com.google.api.services.plus.model.Activity.Actor} object, return a fully fleshed
+ * out {@link ActivityObject} actor
+ *
+ * @param gPlusActor input c.g.a.s.p.m.Activity.Actor
+ * @return {@link ActivityObject} output $.actor as o.a.s.p.j.ActivityObject
+ */
+ private static ActivityObject buildActor(com.google.api.services.plus.model.Activity.Actor gPlusActor) {
+ ActivityObject actor = new ActivityObject();
+
+ actor.setDisplayName(gPlusActor.getDisplayName());
+ actor.setId(formatId(String.valueOf(gPlusActor.getId())));
+ actor.setUrl(gPlusActor.getUrl());
+
+ Image image = new Image();
+ com.google.api.services.plus.model.Activity.Actor.Image googlePlusImage = gPlusActor.getImage();
+
+ if (googlePlusImage != null) {
+ image.setUrl(googlePlusImage.getUrl());
+ }
+ actor.setImage(image);
+
+ return actor;
+ }
+
+ /**
+ * Extract the relevant details from the passed in {@link Person} object and build
+ * an actor with them
+ *
+ * @param person Person
+ * @return Actor constructed with relevant Person details
+ */
+ private static ActivityObject buildActor(Person person) {
+ ActivityObject actor = new ActivityObject();
+
+ actor.setUrl(person.getUrl());
+ actor.setDisplayName(person.getDisplayName());
+ actor.setId(formatId(String.valueOf(person.getId())));
+
+ if (person.getAboutMe() != null) {
+ actor.setSummary(person.getAboutMe());
+ } else if (person.getTagline() != null) {
+ actor.setSummary(person.getTagline());
+ }
+
+ Image image = new Image();
+ Person.Image googlePlusImage = person.getImage();
+
+ if (googlePlusImage != null) {
+ image.setUrl(googlePlusImage.getUrl());
+ }
+ actor.setImage(image);
+
+ Map<String, Object> extensions = new HashMap<>();
+
+ extensions.put("followers", person.getCircledByCount());
+ extensions.put("googleplus", person);
+ actor.setAdditionalProperty("extensions", extensions);
+
+ return actor;
+ }
+
+ /**
+ * Gets the common googleplus {@link Provider} object
+ * @return a provider object representing GooglePlus
+ */
+ public static Provider getProvider() {
+ Provider provider = new Provider();
+ provider.setId("id:providers:googleplus");
+ provider.setDisplayName("GooglePlus");
+ return provider;
+ }
+
+ /**
+ * Formats the ID to conform with the Apache Streams activity ID convention
+ * @param idparts the parts of the ID to join
+ * @return a valid Activity ID in format "id:googleplus:part1:part2:...partN"
+ */
+ public static String formatId(String... idparts) {
+ return String.join(":",
+ Stream.concat(Arrays.stream(new String[]{"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
deleted file mode 100644
index 3b7795b..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/com/google/gplus/GPlusConfiguration.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.google.gplus.GPlusConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "follow": {
- "type": "array",
- "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "googlePlusUsers": {
- "type": "array",
- "description": "A list of user user ids and optional date parameters for the GPlus provider",
- "items": {
- "type": "object",
- "$ref": "#/definitions/userInfo"
- }
- },
- "defaultAfterDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
- },
- "defaultBeforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "DEPRICATED",
- "properties": {
- "appName": {
- "type": "string"
- },
- "pathToP12KeyFile": {
- "type": "string",
- "description": "Absolute Path to key file"
- },
- "serviceAccountEmailAddress": {
- "type": "string",
- "description": "Service Account email address for your app"
- }
- }
- }
- },
- "definitions": {
- "userInfo": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "dynamic": "true",
- "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
- "properties": {
- "userId": {
- "type": "string",
- "description": "Google+ user id"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json
new file mode 100644
index 0000000..3b7795b
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json
@@ -0,0 +1,101 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.google.gplus.GPlusConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "The protocol"
+ },
+ "host": {
+ "type": "string",
+ "description": "The host"
+ },
+ "port": {
+ "type": "integer",
+ "description": "The port"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version"
+ },
+ "endpoint": {
+ "type": "string",
+ "description": "The endpoint"
+ },
+ "follow": {
+ "type": "array",
+ "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ },
+ "googlePlusUsers": {
+ "type": "array",
+ "description": "A list of user user ids and optional date parameters for the GPlus provider",
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/userInfo"
+ }
+ },
+ "defaultAfterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
+ },
+ "defaultBeforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
+ },
+ "oauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "description": "DEPRICATED",
+ "properties": {
+ "appName": {
+ "type": "string"
+ },
+ "pathToP12KeyFile": {
+ "type": "string",
+ "description": "Absolute Path to key file"
+ },
+ "serviceAccountEmailAddress": {
+ "type": "string",
+ "description": "Service Account email address for your app"
+ }
+ }
+ }
+ },
+ "definitions": {
+ "userInfo": {
+ "type": "object",
+ "javaInterfaces" : ["java.io.Serializable"],
+ "dynamic": "true",
+ "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
+ "properties": {
+ "userId": {
+ "type": "string",
+ "description": "Google+ user id"
+ },
+ "afterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
+ },
+ "beforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeIT.java
deleted file mode 100644
index d223867..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/com/google/gplus/GooglePlusCommentSerDeIT.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Comment;
-import com.google.gplus.serializer.util.GPlusCommentDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusCommentSerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeIT.class);
- private ObjectMapper objectMapper;
- private GooglePlusActivityUtil googlePlusActivityUtil;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setupTestCommentObjects() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusActivityUtil = new GooglePlusActivityUtil();
- }
-
- @Test
- public void testCommentObjects() {
- InputStream is = GooglePlusCommentSerDeIT.class.getResourceAsStream("/google_plus_comments_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- Activity activity = new Activity();
- List<Comment> comments = new ArrayList<>();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Comment comment = objectMapper.readValue(line, Comment.class);
-
- LOGGER.info("comment: {}", comment);
-
- assertNotNull(comment);
- assertNotNull(comment.getEtag());
- assertNotNull(comment.getId());
- assertNotNull(comment.getInReplyTo());
- assertNotNull(comment.getObject());
- assertNotNull(comment.getPlusoners());
- assertNotNull(comment.getPublished());
- assertNotNull(comment.getUpdated());
- assertNotNull(comment.getSelfLink());
- assertEquals(comment.getVerb(), "post");
-
- comments.add(comment);
- }
- }
-
- assertEquals(comments.size(), 3);
-
- GooglePlusActivityUtil.updateActivity(comments, activity);
- assertNotNull(activity);
- assertNotNull(activity.getObject());
- assertEquals(activity.getObject().getAttachments().size(), 3);
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-
- @Test
- public void testEmptyComments() {
- Activity activity = new Activity();
-
- GooglePlusActivityUtil.updateActivity(new ArrayList<>(), activity);
-
- assertNull(activity.getObject());
- }
-}
[5/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java
new file mode 100644
index 0000000..f81ecc0
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.providers;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.google.gplus.configuration.UserInfo;
+import org.apache.streams.gplus.provider.GPlusUserDataCollector;
+import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
+import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.plus.Plus;
+import com.google.api.services.plus.model.Person;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Basic Units for {@link org.apache.streams.gplus.provider.GPlusUserDataCollector}.
+ */
+public class TestGPlusUserDataCollector {
+
+ private static final String NO_ERROR = "no error";
+
+ /**
+ * Test that on success a datum will be added to the queue.
+ *
+ * @throws Exception Exception
+ */
+ @Test
+ public void testSucessfullPull() throws Exception {
+ Plus plus = createMockPlus(0, null);
+ BackOffStrategy backOff = new ConstantTimeBackOffStrategy(1);
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ UserInfo user = new UserInfo();
+ user.setUserId("A");
+
+ GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOff, datums, user);
+ collector.run();
+
+ Assert.assertEquals(1, datums.size());
+ StreamsDatum datum = datums.take();
+ Assert.assertNotNull(datum);
+ Assert.assertEquals(NO_ERROR, datum.getId());
+ Assert.assertNotNull(datum.getDocument());
+ Assert.assertTrue(datum.getDocument() instanceof String);
+ }
+
+ /**
+ * Test that on failure, no datums are output.
+ *
+ * @throws Exception Exception
+ */
+ @Test
+ public void testFail() throws Exception {
+ Plus plus = createMockPlus(3, mock(GoogleJsonResponseException.class));
+ UserInfo user = new UserInfo();
+ user.setUserId("A");
+ BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
+ BackOffStrategy backOffStrategy = new ConstantTimeBackOffStrategy(1);
+
+ GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOffStrategy, datums, user);
+ collector.run();
+
+ Assert.assertEquals(0, datums.size());
+ }
+
+ private Plus createMockPlus(final int succedOnTry, final Throwable throwable) {
+ Plus plus = mock(Plus.class);
+ doAnswer(invocationOnMock -> createMockPeople(succedOnTry, throwable)).when(plus).people();
+ return plus;
+ }
+
+ private Plus.People createMockPeople(final int succedOnTry, final Throwable throwable) {
+ Plus.People people = mock(Plus.People.class);
+ try {
+ when(people.get(anyString())).thenAnswer(invocationOnMock -> createMockGetNoError(succedOnTry, throwable));
+ } catch (IOException ioe) {
+ Assert.fail("No Excpetion should have been thrown while creating mocks");
+ }
+ return people;
+ }
+
+ private Plus.People.Get createMockGetNoError(final int succedOnTry, final Throwable throwable) {
+ Plus.People.Get get = mock(Plus.People.Get.class);
+ try {
+ doAnswer(new Answer() {
+ private int counter = 0;
+
+ @Override
+ public Person answer(InvocationOnMock invocationOnMock) throws Throwable {
+ if (counter == succedOnTry) {
+ Person person = new Person();
+ person.setId(NO_ERROR);
+ return person;
+ } else {
+ ++counter;
+ throw throwable;
+ }
+ }
+ }).when(get).execute();
+ } catch (IOException ioe) {
+ Assert.fail("No Excpetion should have been thrown while creating mocks");
+ }
+ return get;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java
new file mode 100644
index 0000000..1a9720d
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.serializer.util;
+
+import org.apache.streams.jackson.StreamsJacksonMapper;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.api.services.plus.model.Activity;
+import com.google.api.services.plus.model.Person;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * GPlusEventClassifierTest tests GPlusEventClassifier.
+ */
+public class GPlusEventClassifierTest {
+
+ private static StreamsJacksonMapper mapper = StreamsJacksonMapper.getInstance();
+
+ @Test
+ public void classifyActivityTest() {
+ try {
+ Activity activity = new Activity();
+ activity.setKind("plus#activity");
+ Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(activity));
+ Assert.assertEquals(retClass, Activity.class);
+ } catch (Exception ex) {
+ //
+ }
+ }
+
+ @Test
+ public void classifyPersonTest() {
+ try {
+ Person person = new Person();
+ person.setKind("plus#person");
+ Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
+ Assert.assertEquals(retClass, Person.class);
+ } catch (Exception ex) {
+ //
+ }
+ }
+
+ @Test
+ public void classifyObjectNodeTest() {
+ try {
+ Person person = new Person();
+ person.setKind("fake");
+ Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
+ Assert.assertEquals(retClass, ObjectNode.class);
+ } catch (Exception ex) {
+ //
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/processors/GooglePlusTypeConverterIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/processors/GooglePlusTypeConverterIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/processors/GooglePlusTypeConverterIT.java
deleted file mode 100644
index 098f555..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/processors/GooglePlusTypeConverterIT.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import com.google.gplus.processor.GooglePlusTypeConverter;
-import com.google.gplus.serializer.util.GPlusActivityDeserializer;
-import com.google.gplus.serializer.util.GPlusPersonDeserializer;
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Tests conversion of gplus inputs to Activity
- */
-public class GooglePlusTypeConverterIT {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverterIT.class);
- private GooglePlusTypeConverter googlePlusTypeConverter;
- private ObjectMapper objectMapper;
-
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusTypeConverter = new GooglePlusTypeConverter();
- googlePlusTypeConverter.prepare(null);
- }
-
- @Test(dependsOnGroups={"testGPlusUserDataProvider"})
- public void testProcessPerson() throws IOException, ActivitySerializerException {
-
- File file = new File("target/test-classes/GPlusUserDataProviderIT.stdout.txt");
- InputStream is = new FileInputStream(file);
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- Person person = objectMapper.readValue(line, Person.class);
- StreamsDatum streamsDatum = new StreamsDatum(person);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
- GooglePlusActivityUtil.updateActivity(person, activity);
-
- assertEquals(retList.size(), 1);
- assert(retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- }
- }
- }
-
- @Test(dependsOnGroups={"testGPlusUserActivityProvider"})
- public void testProcessActivity() throws IOException, ActivitySerializerException{
-
- File file = new File("target/test-classes/GPlusUserActivityProviderIT.stdout.txt");
- InputStream is = new FileInputStream(file);
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- com.google.api.services.plus.model.Activity gPlusActivity = objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
- StreamsDatum streamsDatum = new StreamsDatum(gPlusActivity);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
- GooglePlusActivityUtil.updateActivity(gPlusActivity, activity);
-
- assertEquals(retList.size(), 1);
- assertTrue(retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- }
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserActivityProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserActivityProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserActivityProviderIT.java
deleted file mode 100644
index 71e825f..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserActivityProviderIT.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.test.providers;
-
-import com.google.gplus.provider.GPlusUserActivityProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-public class GPlusUserActivityProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityProviderIT.class);
-
- @Test(groups={"testGPlusUserActivityProvider"})
- public void testGPlusUserActivityProvider() throws Exception {
-
- String configfile = "./target/test-classes/GPlusUserActivityProviderIT.conf";
- String outfile = "./target/test-classes/GPlusUserActivityProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- GPlusUserActivityProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserDataProviderIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserDataProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserDataProviderIT.java
deleted file mode 100644
index 031716e..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/test/providers/GPlusUserDataProviderIT.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.test.providers;
-
-import com.google.gplus.provider.GPlusUserDataProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-public class GPlusUserDataProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataProviderIT.class);
-
- @Test(groups={"testGPlusUserDataProvider"})
- public void testGPlusUserDataProvider() throws Exception {
-
- String configfile = "./target/test-classes/GPlusUserDataProviderIT.conf";
- String outfile = "./target/test-classes/GPlusUserDataProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- GPlusUserDataProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- GPlusUserDataProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramConfiguration.json
deleted file mode 100644
index 891c1da..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramConfiguration.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.instagram.InstagramConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "clientId": {
- "type": "string",
- "description": "Your Instagram Client Id"
- },
- "usersInfo": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "properties": {
- "authorizedTokens": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "string"
- },
- "description": "Instagram tokens for authorized users of your client/app"
- },
- "users": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "object",
- "$ref": "#/definitions/user"
- },
- "description": "List of user ids to gather data for. Type of data gathered depends on provider"
- },
- "defaultAfterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for all users that don't have date ranges specified. If this is null it will pull from the earliest possible time"
- },
- "defaultBeforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for all users that don't have date ranges specified. If this is null it will pull till current time."
- }
- }
- }
- },
- "definitions": {
- "user": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "properties": {
- "userId": {
- "type": "string",
- "description": "instagram user id"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramUserInformationConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramUserInformationConfiguration.json
deleted file mode 100644
index 42eb5d6..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/com/instagram/InstagramUserInformationConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.instagram.InstagramUserInformationConfiguration",
- "extends": {"$ref":"InstagramConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "userIds": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose posts should be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramConfiguration.json
new file mode 100644
index 0000000..891c1da
--- /dev/null
+++ b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramConfiguration.json
@@ -0,0 +1,71 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.instagram.InstagramConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "clientId": {
+ "type": "string",
+ "description": "Your Instagram Client Id"
+ },
+ "usersInfo": {
+ "type": "object",
+ "javaInterfaces" : ["java.io.Serializable"],
+ "properties": {
+ "authorizedTokens": {
+ "type": "array",
+ "uniqueItems": true,
+ "items": {
+ "type": "string"
+ },
+ "description": "Instagram tokens for authorized users of your client/app"
+ },
+ "users": {
+ "type": "array",
+ "uniqueItems": true,
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/user"
+ },
+ "description": "List of user ids to gather data for. Type of data gathered depends on provider"
+ },
+ "defaultAfterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for all users that don't have date ranges specified. If this is null it will pull from the earliest possible time"
+ },
+ "defaultBeforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for all users that don't have date ranges specified. If this is null it will pull till current time."
+ }
+ }
+ }
+ },
+ "definitions": {
+ "user": {
+ "type": "object",
+ "javaInterfaces" : ["java.io.Serializable"],
+ "properties": {
+ "userId": {
+ "type": "string",
+ "description": "instagram user id"
+ },
+ "afterDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
+ },
+ "beforeDate": {
+ "type": "string",
+ "format": "date-time",
+ "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramUserInformationConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramUserInformationConfiguration.json
new file mode 100644
index 0000000..42eb5d6
--- /dev/null
+++ b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/InstagramUserInformationConfiguration.json
@@ -0,0 +1,20 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.instagram.InstagramUserInformationConfiguration",
+ "extends": {"$ref":"InstagramConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "userIds": {
+ "type": "array",
+ "description": "A list of user IDs, indicating the users whose posts should be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/Moreover.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/Moreover.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/Moreover.json
deleted file mode 100644
index 8708e82..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/Moreover.json
+++ /dev/null
@@ -1,337 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "com.moreover.Moreover",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "tags": {
- "type": "string"
- },
- "publishingPlatform": {
- "properties": {
- "totalViews": {
- "type": "string"
- },
- "itemId": {
- "type": "string"
- }
- }
- },
- "loginStatus": {
- "type": "string"
- },
- "duplicateGroupId": {
- "type": "string"
- },
- "companies": {
- "properties": {
- "symbol": {
- "type": "string"
- },
- "primary": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "contentCount": {
- "type": "string"
- },
- "exchange": {
- "type": "string"
- },
- "titleCount": {
- "type": "string"
- },
- "isin": {
- "type": "string"
- }
- }
- },
- "copyright": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "author": {
- "properties": {
- "publishingPlatform": {
- "properties": {
- "userId": {
- "type": "string"
- },
- "totalViews": {
- "type": "string"
- }
- }
- },
- "dateLastActive": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "topics": {
- "properties": {
- "name": {
- "type": "string"
- },
- "group": {
- "type": "string"
- }
- }
- },
- "title": {
- "type": "string"
- },
- "source": {
- "properties": {
- "category": {
- "type": "string"
- },
- "location": {
- "properties": {
- "region": {
- "type": "string"
- },
- "subregion": {
- "type": "string"
- },
- "zipCode": {
- "type": "string"
- },
- "state": {
- "type": "string"
- },
- "countryCode": {
- "type": "string"
- },
- "zipArea": {
- "type": "string"
- },
- "country": {
- "type": "string"
- }
- }
- },
- "editorialRank": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "feed": {
- "properties": {
- "tags": {
- "type": "string"
- },
- "genre": {
- "type": "string"
- },
- "publishingPlatform": {
- "type": "string"
- },
- "inWhiteList": {
- "type": "string"
- },
- "imageUrl": {
- "type": "string"
- },
- "copyright": {
- "type": "string"
- },
- "mediaType": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "rank": {
- "properties": {
- "autoRankOrder": {
- "type": "string"
- },
- "inboundLinkCount": {
- "type": "string"
- },
- "autoRank": {
- "type": "string"
- }
- }
- },
- "description": {
- "type": "string"
- },
- "idFromPublisher": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "dataFormat": {
- "type": "string"
- },
- "generator": {
- "type": "string"
- },
- "autoTopics": {
- "type": "string"
- },
- "language": {
- "type": "string"
- },
- "editorialTopics": {
- "type": "string"
- }
- }
- },
- "homeUrl": {
- "type": "string"
- },
- "publisher": {
- "type": "string"
- }
- }
- },
- "locations": {
- "properties": {
- "region": {
- "type": "string"
- },
- "subregion": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "state": {
- "properties": {
- "name": {
- "type": "string"
- },
- "fipsCode": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- }
- }
- },
- "longitude": {
- "type": "string"
- },
- "latitude": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "mentions": {
- "type": "string"
- },
- "country": {
- "properties": {
- "name": {
- "type": "string"
- },
- "fipsCode": {
- "type": "string"
- },
- "isoCode": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- }
- }
- }
- }
- },
- "commentsUrl": {
- "type": "string"
- },
- "dataFormat": {
- "type": "string"
- },
- "outboundUrls": {
- "type": "string"
- },
- "sequenceId": {
- "type": "string"
- },
- "publishedDate": {
- "type": "string"
- },
- "language": {
- "type": "string"
- },
- "adultLanguage": {
- "type": "string"
- },
- "harvestDate": {
- "type": "string"
- },
- "media": {
- "properties": {
- "duration": {
- "type": "string"
- },
- "audio": {
- "properties": {
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- },
- "image": {
- "properties": {
- "url": {
- "type": "string"
- }
- }
- },
- "caption": {
- "type": "string"
- },
- "video": {
- "properties": {
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- },
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/MoreoverConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/MoreoverConfiguration.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/MoreoverConfiguration.json
deleted file mode 100644
index bbbda5d..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/jsonschema/com/moreover/MoreoverConfiguration.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.moreover.MoreoverConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "apiKeys": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.moreover.MoreoverKeyData",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "key": {
- "type": "string"
- },
- "startingSequence": {
- "type": "string"
- }
- }
- }
- },
- "maxBatchSize": {
- "type": "long"
- },
- "minDelaySeconds": {
- "type": "long"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json
new file mode 100644
index 0000000..ee60b3e
--- /dev/null
+++ b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json
@@ -0,0 +1,337 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType": "org.apache.streams.moreover.Moreover",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "tags": {
+ "type": "string"
+ },
+ "publishingPlatform": {
+ "properties": {
+ "totalViews": {
+ "type": "string"
+ },
+ "itemId": {
+ "type": "string"
+ }
+ }
+ },
+ "loginStatus": {
+ "type": "string"
+ },
+ "duplicateGroupId": {
+ "type": "string"
+ },
+ "companies": {
+ "properties": {
+ "symbol": {
+ "type": "string"
+ },
+ "primary": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "contentCount": {
+ "type": "string"
+ },
+ "exchange": {
+ "type": "string"
+ },
+ "titleCount": {
+ "type": "string"
+ },
+ "isin": {
+ "type": "string"
+ }
+ }
+ },
+ "copyright": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "author": {
+ "properties": {
+ "publishingPlatform": {
+ "properties": {
+ "userId": {
+ "type": "string"
+ },
+ "totalViews": {
+ "type": "string"
+ }
+ }
+ },
+ "dateLastActive": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "topics": {
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "group": {
+ "type": "string"
+ }
+ }
+ },
+ "title": {
+ "type": "string"
+ },
+ "source": {
+ "properties": {
+ "category": {
+ "type": "string"
+ },
+ "location": {
+ "properties": {
+ "region": {
+ "type": "string"
+ },
+ "subregion": {
+ "type": "string"
+ },
+ "zipCode": {
+ "type": "string"
+ },
+ "state": {
+ "type": "string"
+ },
+ "countryCode": {
+ "type": "string"
+ },
+ "zipArea": {
+ "type": "string"
+ },
+ "country": {
+ "type": "string"
+ }
+ }
+ },
+ "editorialRank": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "feed": {
+ "properties": {
+ "tags": {
+ "type": "string"
+ },
+ "genre": {
+ "type": "string"
+ },
+ "publishingPlatform": {
+ "type": "string"
+ },
+ "inWhiteList": {
+ "type": "string"
+ },
+ "imageUrl": {
+ "type": "string"
+ },
+ "copyright": {
+ "type": "string"
+ },
+ "mediaType": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "rank": {
+ "properties": {
+ "autoRankOrder": {
+ "type": "string"
+ },
+ "inboundLinkCount": {
+ "type": "string"
+ },
+ "autoRank": {
+ "type": "string"
+ }
+ }
+ },
+ "description": {
+ "type": "string"
+ },
+ "idFromPublisher": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "dataFormat": {
+ "type": "string"
+ },
+ "generator": {
+ "type": "string"
+ },
+ "autoTopics": {
+ "type": "string"
+ },
+ "language": {
+ "type": "string"
+ },
+ "editorialTopics": {
+ "type": "string"
+ }
+ }
+ },
+ "homeUrl": {
+ "type": "string"
+ },
+ "publisher": {
+ "type": "string"
+ }
+ }
+ },
+ "locations": {
+ "properties": {
+ "region": {
+ "type": "string"
+ },
+ "subregion": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "state": {
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "fipsCode": {
+ "type": "string"
+ },
+ "confidence": {
+ "type": "string"
+ }
+ }
+ },
+ "longitude": {
+ "type": "string"
+ },
+ "latitude": {
+ "type": "string"
+ },
+ "confidence": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "mentions": {
+ "type": "string"
+ },
+ "country": {
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "fipsCode": {
+ "type": "string"
+ },
+ "isoCode": {
+ "type": "string"
+ },
+ "confidence": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "commentsUrl": {
+ "type": "string"
+ },
+ "dataFormat": {
+ "type": "string"
+ },
+ "outboundUrls": {
+ "type": "string"
+ },
+ "sequenceId": {
+ "type": "string"
+ },
+ "publishedDate": {
+ "type": "string"
+ },
+ "language": {
+ "type": "string"
+ },
+ "adultLanguage": {
+ "type": "string"
+ },
+ "harvestDate": {
+ "type": "string"
+ },
+ "media": {
+ "properties": {
+ "duration": {
+ "type": "string"
+ },
+ "audio": {
+ "properties": {
+ "url": {
+ "type": "string"
+ },
+ "mimeType": {
+ "type": "string"
+ }
+ }
+ },
+ "image": {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "caption": {
+ "type": "string"
+ },
+ "video": {
+ "properties": {
+ "url": {
+ "type": "string"
+ },
+ "mimeType": {
+ "type": "string"
+ }
+ }
+ },
+ "url": {
+ "type": "string"
+ },
+ "mimeType": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json
new file mode 100644
index 0000000..bbbda5d
--- /dev/null
+++ b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json
@@ -0,0 +1,38 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.moreover.MoreoverConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "apiKeys": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "javaType" : "org.apache.streams.moreover.MoreoverKeyData",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "key": {
+ "type": "string"
+ },
+ "startingSequence": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "maxBatchSize": {
+ "type": "long"
+ },
+ "minDelaySeconds": {
+ "type": "long"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
index 779c2ab..bf4720a 100644
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
@@ -21,6 +21,7 @@ package org.apache.streams.rss.provider.perpetual;
import org.apache.streams.rss.FeedDetails;
import org.apache.streams.rss.provider.RssStreamProviderTask;
+import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
@@ -29,7 +30,6 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
-import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@ -55,20 +55,20 @@ public class RssFeedSchedulerTest {
RssFeedScheduler scheduler = new RssFeedScheduler(mockService, createFeedList(), new LinkedBlockingQueue<>(), 1);
scheduler.scheduleFeeds();
- assertEquals("Expected 2 Feeds to be scheduled", 2, queuedTasks.size());
- assertEquals("Expected Feed 1 to be queued first", "1", queuedTasks.get(0));
- assertEquals("Expected Feed 2 to be queued second", "2", queuedTasks.get(1));
+ Assert.assertEquals("Expected 2 Feeds to be scheduled", 2, queuedTasks.size());
+ Assert.assertEquals("Expected Feed 1 to be queued first", "1", queuedTasks.get(0));
+ Assert.assertEquals("Expected Feed 2 to be queued second", "2", queuedTasks.get(1));
safeSleep(1);
scheduler.scheduleFeeds();
- assertEquals("Only feed 1 should have been re-queued", 3, queuedTasks.size());
- assertEquals("Only feed 1 should have been re-queued", "1", queuedTasks.get(2));
+ Assert.assertEquals("Only feed 1 should have been re-queued", 3, queuedTasks.size());
+ Assert.assertEquals("Only feed 1 should have been re-queued", "1", queuedTasks.get(2));
safeSleep(60 * 1000);
scheduler.scheduleFeeds();
- assertEquals("Both feeds should have been re-queued", 5, queuedTasks.size());
- assertEquals("1", queuedTasks.get(3));
- assertEquals("2", queuedTasks.get(4));
+ Assert.assertEquals("Both feeds should have been re-queued", 5, queuedTasks.size());
+ Assert.assertEquals("1", queuedTasks.get(3));
+ Assert.assertEquals("2", queuedTasks.get(4));
}
private List<FeedDetails> createFeedList() {
@@ -79,7 +79,7 @@ public class RssFeedSchedulerTest {
list.add(fd);
fd = new FeedDetails();
- fd.setPollIntervalMillis( 60L * 1000);
+ fd.setPollIntervalMillis(60L * 1000);
fd.setUrl("2");
list.add(fd);
return list;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
index fffe7a1..a77a129 100644
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
+++ b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
@@ -27,13 +27,13 @@ import org.apache.streams.core.StreamsProvider;
import org.apache.streams.core.StreamsResultSet;
import org.apache.streams.data.util.RFC3339Utils;
import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.sysomos.SysomosConfiguration;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.util.concurrent.Uninterruptibles;
-import com.sysomos.SysomosConfiguration;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigParseOptions;
@@ -61,24 +61,17 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Streams Provider for the Sysomos Heartbeat API
- *
* <p/>
* Configuration:
* The provider takes either a Map[String,Object] containing the mode (backfill and terminate OR continuous) and a
* Map[String,String] of heartbeat IDs to document target ids or a string of the format
- * ${heartbeatId}:${documentId},...,${heartbeatId}:${documentId}
+ * ${heartbeatId}:${documentId},...,${heartbeatId}:${documentId}
* This configuration will configure the provider to backfill to the specified document and either terminate or not
* depending on the mode flag. Continuous mode is assumed, and is the ony mode supported by the String configuration.
- *
*/
public class SysomosProvider implements StreamsProvider {
public static final String STREAMS_ID = "SysomosProvider";
-
- public enum Mode { CONTINUOUS, BACKFILL_AND_TERMINATE }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosProvider.class);
-
public static final String ENDING_TIME_KEY = "addedBefore";
public static final String STARTING_TIME_KEY = "addedAfter";
public static final String MODE_KEY = "mode";
@@ -86,16 +79,14 @@ public class SysomosProvider implements StreamsProvider {
public static final int LATENCY = 10000; //Default minLatency for querying the Sysomos API in milliseconds
public static final long PROVIDER_BATCH_SIZE = 10000L; //Default maximum size of the queue
public static final long API_BATCH_SIZE = 1000L; //Default maximum size of an API request
-
- protected volatile Queue<StreamsDatum> providerQueue;
-
+ private static final Logger LOGGER = LoggerFactory.getLogger(SysomosProvider.class);
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final Set<String> completedHeartbeats = new HashSet<>();
private final long maxQueued;
private final long minLatency;
private final long scheduledLatency;
private final long maxApiBatch;
-
+ protected volatile Queue<StreamsDatum> providerQueue;
private SysomosClient client;
private SysomosConfiguration config;
private ScheduledExecutorService stream;
@@ -105,7 +96,6 @@ public class SysomosProvider implements StreamsProvider {
private Mode mode = Mode.CONTINUOUS;
private boolean started = false;
private AtomicInteger count;
-
/**
* SysomosProvider constructor.
* @param sysomosConfiguration SysomosConfiguration
@@ -121,6 +111,57 @@ public class SysomosProvider implements StreamsProvider {
this.count = new AtomicInteger();
}
+ /**
+ * To use from command line:
+ * <p/>
+ * Supply configuration similar to src/test/resources/rss.conf
+ * <p/>
+ * Launch using:
+ * <p/>
+ * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
+ * @param args args
+ * @throws Exception Exception
+ */
+ public static void main(String[] args) throws Exception {
+
+ Preconditions.checkArgument(args.length >= 2);
+
+ String configfile = args[0];
+ String outfile = args[1];
+
+ Config reference = ConfigFactory.load();
+ File file = new File(configfile);
+ assert (file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
+
+ Config typesafe = testResourceConfig.withFallback(reference).resolve();
+
+ StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
+ SysomosConfiguration config = new ComponentConfigurator<>(SysomosConfiguration.class).detectConfiguration(typesafe, "rss");
+ SysomosProvider provider = new SysomosProvider(config);
+
+ ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
+ provider.prepare(config);
+ provider.startStream();
+ do {
+ Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
+ for (StreamsDatum datum : provider.readCurrent()) {
+ String json;
+ try {
+ json = mapper.writeValueAsString(datum.getDocument());
+ outStream.println(json);
+ } catch (JsonProcessingException ex) {
+ System.err.println(ex.getMessage());
+ }
+ }
+ }
+ while (provider.isRunning());
+ provider.cleanUp();
+ outStream.flush();
+ }
+
public SysomosConfiguration getConfig() {
return config;
}
@@ -197,7 +238,7 @@ public class SysomosProvider implements StreamsProvider {
public boolean isRunning() {
return providerQueue.size() > 0
|| (completedHeartbeats.size() < this.getConfig().getHeartbeatIds().size()
- && !(stream.isTerminated()
+ && !(stream.isTerminated()
|| stream.isShutdown()));
}
@@ -205,7 +246,7 @@ public class SysomosProvider implements StreamsProvider {
public void prepare(Object configurationObject) {
this.providerQueue = constructQueue();
if (configurationObject instanceof Map) {
- extractConfigFromMap((Map) configurationObject);
+ extractConfigFromMap((Map)configurationObject);
} else if (configurationObject instanceof String) {
documentIds = Splitter.on(";").trimResults().withKeyValueSeparator("=").split((String)configurationObject);
}
@@ -338,58 +379,5 @@ public class SysomosProvider implements StreamsProvider {
return this.count.get();
}
- /**
- * To use from command line:
- *
- * <p/>
- * Supply configuration similar to src/test/resources/rss.conf
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- SysomosConfiguration config = new ComponentConfigurator<>(SysomosConfiguration.class).detectConfiguration(typesafe, "rss");
- SysomosProvider provider = new SysomosProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning() );
- provider.cleanUp();
- outStream.flush();
- }
+ public enum Mode {CONTINUOUS, BACKFILL_AND_TERMINATE}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/Sysomos.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/Sysomos.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/Sysomos.json
deleted file mode 100644
index d9080e8..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/Sysomos.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "com.sysomos.json.Sysomos",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "tweetJsonLink": {
- "type": "string"
- },
- "location": {
- "properties": {
- "locationString": {
- "type": "string"
- },
- "country": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "twitterFollowing": {
- "type": "string"
- },
- "twitterFollowers": {
- "type": "string"
- },
- "tweetid": {
- "type": "string"
- },
- "mediaType": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "docid": {
- "type": "string"
- },
- "sentiment": {
- "type": "string"
- },
- "time": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "tweetHbLink": {
- "type": "string"
- },
- "influenceLevel": {
- "type": "string"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/SysomosConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/SysomosConfiguration.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/SysomosConfiguration.json
deleted file mode 100644
index 4ce336e..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/com/sysomos/SysomosConfiguration.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "com.sysomos.SysomosConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint",
- "default": "http://api.sysomos.com/"
- },
- "heartbeatIds": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- },
- "minDelayMs": {
- "type": "String",
- "format": "utc-millisec"
- },
- "scheduledDelayMs": {
- "type": "String",
- "format": "utc-millisec"
- },
- "maxBatchSize": {
- "type": "String",
- "format": "utc-millisec"
- },
- "apiBatchSize": {
- "type": "String",
- "format": "utc-millisec"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json
new file mode 100644
index 0000000..b5feb61
--- /dev/null
+++ b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json
@@ -0,0 +1,61 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType": "org.apache.streams.sysomos.Sysomos",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "tweetJsonLink": {
+ "type": "string"
+ },
+ "location": {
+ "properties": {
+ "locationString": {
+ "type": "string"
+ },
+ "country": {
+ "type": "string"
+ }
+ }
+ },
+ "link": {
+ "type": "string"
+ },
+ "twitterFollowing": {
+ "type": "string"
+ },
+ "twitterFollowers": {
+ "type": "string"
+ },
+ "tweetid": {
+ "type": "string"
+ },
+ "mediaType": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "docid": {
+ "type": "string"
+ },
+ "sentiment": {
+ "type": "string"
+ },
+ "time": {
+ "type": "string"
+ },
+ "title": {
+ "type": "string"
+ },
+ "tweetHbLink": {
+ "type": "string"
+ },
+ "influenceLevel": {
+ "type": "string"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json
new file mode 100644
index 0000000..487455f
--- /dev/null
+++ b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json
@@ -0,0 +1,43 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.sysomos.SysomosConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "apiKey": {
+ "type": "string"
+ },
+ "endpoint": {
+ "type": "string",
+ "description": "The endpoint",
+ "default": "http://api.sysomos.com/"
+ },
+ "heartbeatIds": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "string"
+ }
+ },
+ "minDelayMs": {
+ "type": "String",
+ "format": "utc-millisec"
+ },
+ "scheduledDelayMs": {
+ "type": "String",
+ "format": "utc-millisec"
+ },
+ "maxBatchSize": {
+ "type": "String",
+ "format": "utc-millisec"
+ },
+ "apiBatchSize": {
+ "type": "String",
+ "format": "utc-millisec"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosJsonSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosJsonSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosJsonSerDeIT.java
deleted file mode 100644
index ab1c2a4..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosJsonSerDeIT.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.sysomos.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.sysomos.json.Sysomos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests ability to convert String json form to {@link com.sysomos.json.Sysomos} form
- */
-public class SysomosJsonSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosJsonSerDeIT.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void testSysomosJsonSerDe() {
-
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = SysomosJsonSerDeIT.class.getResourceAsStream("/sysomos_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Sysomos ser = mapper.readValue(line, Sysomos.class);
-
- String des = mapper.writeValueAsString(ser);
- LOGGER.debug(des);
- }
- } catch ( Exception ex ) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosXmlSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosXmlSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosXmlSerDeIT.java
deleted file mode 100644
index e078d02..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/java/com/sysomos/test/SysomosXmlSerDeIT.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.sysomos.test;
-
-import com.fasterxml.aalto.stax.InputFactoryImpl;
-import com.fasterxml.aalto.stax.OutputFactoryImpl;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
-import com.fasterxml.jackson.dataformat.xml.XmlFactory;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import com.sysomos.xml.BeatApi;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests ability to convert String xml form to {@link com.sysomos.xml.BeatApi} form
- */
-public class SysomosXmlSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosXmlSerDeIT.class);
-
- private XmlMapper xmlMapper;
-
- /**
- * before.
- */
- @Before
- public void before() {
-
- XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(),
- new OutputFactoryImpl());
-
- JacksonXmlModule module = new JacksonXmlModule();
-
- module.setDefaultUseWrapper(false);
-
- xmlMapper = new XmlMapper(xmlFactory, module);
-
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
- Boolean.TRUE);
- xmlMapper.configure(
- DeserializationFeature.READ_ENUMS_USING_TO_STRING,
- Boolean.TRUE);
-
- }
-
- @Test
- public void test() {
-
- InputStream is = SysomosXmlSerDeIT.class.getResourceAsStream("/sysomos_xmls.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- BeatApi ser = xmlMapper.readValue(line, BeatApi.class);
-
- String des = xmlMapper.writeValueAsString(ser);
- LOGGER.debug(des);
- }
- } catch ( Exception ex ) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java
new file mode 100644
index 0000000..56555ad
--- /dev/null
+++ b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.sysomos.test;
+
+import org.apache.streams.sysomos.Sysomos;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Tests ability to convert String json form to {@link org.apache.streams.sysomos.Sysomos} form
+ */
+public class SysomosJsonSerDeIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SysomosJsonSerDeIT.class);
+
+ private ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ public void testSysomosJsonSerDe() {
+
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
+ mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
+ mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
+
+ InputStream is = SysomosJsonSerDeIT.class.getResourceAsStream("/sysomos_jsons.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ LOGGER.debug(line);
+
+ Sysomos ser = mapper.readValue(line, Sysomos.class);
+
+ String des = mapper.writeValueAsString(ser);
+ LOGGER.debug(des);
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
[4/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
new file mode 100644
index 0000000..091552f
--- /dev/null
+++ b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.sysomos.test;
+
+import com.fasterxml.aalto.stax.InputFactoryImpl;
+import com.fasterxml.aalto.stax.OutputFactoryImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
+import com.fasterxml.jackson.dataformat.xml.XmlFactory;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.sysomos.xml.BeatApi;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Tests ability to convert String xml form to {@link com.sysomos.xml.BeatApi} form
+ */
+public class SysomosXmlSerDeIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SysomosXmlSerDeIT.class);
+
+ private XmlMapper xmlMapper;
+
+ /**
+ * before.
+ */
+ @Before
+ public void before() {
+
+ XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(),
+ new OutputFactoryImpl());
+
+ JacksonXmlModule module = new JacksonXmlModule();
+
+ module.setDefaultUseWrapper(false);
+
+ xmlMapper = new XmlMapper(xmlFactory, module);
+
+ xmlMapper
+ .configure(
+ DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
+ Boolean.TRUE);
+ xmlMapper
+ .configure(
+ DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
+ Boolean.TRUE);
+ xmlMapper
+ .configure(
+ DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
+ Boolean.TRUE);
+ xmlMapper.configure(
+ DeserializationFeature.READ_ENUMS_USING_TO_STRING,
+ Boolean.TRUE);
+
+ }
+
+ @Test
+ public void test() {
+
+ InputStream is = SysomosXmlSerDeIT.class.getResourceAsStream("/sysomos_xmls.txt");
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ try {
+ while (br.ready()) {
+ String line = br.readLine();
+ LOGGER.debug(line);
+
+ BeatApi ser = xmlMapper.readValue(line, BeatApi.class);
+
+ String des = xmlMapper.writeValueAsString(ser);
+ LOGGER.debug(des);
+ }
+ } catch ( Exception ex ) {
+ ex.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
deleted file mode 100644
index 03b801d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Delete",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "delete": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "status": {
- "type": "object",
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user_id_str": {
- "type": "string"
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
deleted file mode 100644
index 320db12..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Follow",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "follower": {
- "$ref": "User.json"
- },
- "followee": {
- "$ref": "User.json"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
deleted file mode 100644
index ae7a021..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.FriendList",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "friends": {
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "friends_str": {
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
deleted file mode 100644
index a449525..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Retweet",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"tweet.json"},
- "properties": {
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
deleted file mode 100644
index 69048d1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol",
- "default": "https"
- },
- "host": {
- "type": "string",
- "description": "The host",
- "default": "api.twitter.com"
- },
- "port": {
- "type": "integer",
- "description": "The port",
- "default": 443
- },
- "version": {
- "type": "string",
- "description": "The version",
- "default": "1.1"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "jsonStoreEnabled": {
- "default" : true,
- "type": "string"
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "appName": {
- "type": "string"
- },
- "consumerKey": {
- "type": "string"
- },
- "consumerSecret": {
- "type": "string"
- },
- "accessToken": {
- "type": "string"
- },
- "accessTokenSecret": {
- "type": "string"
- }
- }
- },
- "basicauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "username": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
- },
- "retrySleepMs": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit",
- "default": 100000
- },
- "retryMax": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit",
- "default": 10
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
deleted file mode 100644
index 89fc7af..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "ids_only": {
- "type": "boolean",
- "description": "Whether to collect ids only, or full profiles",
- "default": "true"
- },
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 50000
- },
- "max_pages": {
- "type": "integer",
- "description": "Max pages per user to request",
- "default": 10
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 5000
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
deleted file mode 100644
index 6fa2a73..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "includeEntities": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter-level": {
- "type": "string",
- "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream"
- },
- "with": {
- "type": "string",
- "description": "Typically following or user"
- },
- "replies": {
- "type": "string",
- "description": "Set to all, to see all @replies"
- },
- "follow": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "integer"
- }
- },
- "track": {
- "type": "array",
- "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
deleted file mode 100644
index 37ed60e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 3200
- },
- "max_pages": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 16
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
deleted file mode 100644
index 405c87a..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 200
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
deleted file mode 100644
index c6cb798..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json
+++ /dev/null
@@ -1,129 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.User",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "location": {
- "type": "string"
- },
- "default_profile": {
- "type": "boolean"
- },
- "statuses_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_background_tile": {
- "type": "boolean"
- },
- "lang": {
- "type": "string"
- },
- "profile_link_color": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "protected": {
- "type": "boolean"
- },
- "favourites_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_text_color": {
- "type": "string"
- },
- "verified": {
- "type": "boolean"
- },
- "description": {
- "type": "string"
- },
- "contributors_enabled": {
- "type": "boolean"
- },
- "name": {
- "type": "string"
- },
- "profile_sidebar_border_color": {
- "type": "string"
- },
- "profile_background_color": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "default_profile_image": {
- "type": "boolean"
- },
- "followers_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "profile_image_url_https": {
- "type": "string"
- },
- "profile_background_image_url": {
- "type": "string"
- },
- "profile_background_image_url_https": {
- "type": "string"
- },
- "follow_request_sent": {
- "type": "boolean"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "time_zone": {
- "type": "string"
- },
- "profile_use_background_image": {
- "type": "boolean"
- },
- "friends_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_sidebar_fill_color": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "is_translator": {
- "type": "boolean"
- },
- "listed_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "status": {
- "$ref": "tweet.json"
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
deleted file mode 100644
index 23f8365..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "created_at": {
- "type": "string",
- "format" : "date-time"
- },
- "event_type": {
- "type": "string",
- "enum" : [
- "access_revoked",
- "block",
- "unblock",
- "favorite",
- "unfavorite",
- "follow",
- "unfollow",
- "list_created",
- "list_destroyed",
- "list_updated",
- "list_member_added",
- "list_member_removed",
- "list_user_subscribed",
- "list_user_unsubscribed",
- "user_update"
- ]
- },
- "source": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target_object": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
deleted file mode 100644
index e529559..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Tweet",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "retweeted": {
- "type": "boolean"
- },
- "in_reply_to_screen_name": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter_level": {
- "type": "string"
- },
- "contributors": {
- "ignore_malformed": false,
- "type": "array",
- "items": [{
- "type" : "number"
- }]
- },
- "place": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Place",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "bounding_box": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "number"
- }
- }
- }
- }
- }
- }
- }
- },
- "entities": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.pojo.Entities",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "user_mentions": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserMentions",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "name": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type" : "integer"
- }]
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- }
- }
- }
- },
- "hashtags": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.Hashtag",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type": "integer"
- }]
- }
- }
- }
- },
- "urls": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.Url",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [
- {
- "type" : "integer"
- }
- ]
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "in_reply_to_status_id_str": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "in_reply_to_user_id_str": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "favorited": {
- "type": "boolean"
- },
- "possibly_sensitive": {
- "type": "boolean"
- },
- "in_reply_to_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "created_at": {
- "type" : "string",
- "format" : "date-time"
- },
- "in_reply_to_user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "favorite_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "retweet_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user": {
- "$ref": "User.json"
- },
- "is_quote_status": {
- "type": "boolean"
- },
- "quoted_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "quoted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being quoted.",
- "$ref" : "tweet.json"
- },
- "retweeted_status_id": {
- "type": "integer"
- },
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
new file mode 100644
index 0000000..69048d1
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json
@@ -0,0 +1,87 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "protocol": {
+ "type": "string",
+ "description": "The protocol",
+ "default": "https"
+ },
+ "host": {
+ "type": "string",
+ "description": "The host",
+ "default": "api.twitter.com"
+ },
+ "port": {
+ "type": "integer",
+ "description": "The port",
+ "default": 443
+ },
+ "version": {
+ "type": "string",
+ "description": "The version",
+ "default": "1.1"
+ },
+ "endpoint": {
+ "type": "string",
+ "description": "The endpoint"
+ },
+ "jsonStoreEnabled": {
+ "default" : true,
+ "type": "string"
+ },
+ "oauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "appName": {
+ "type": "string"
+ },
+ "consumerKey": {
+ "type": "string"
+ },
+ "consumerSecret": {
+ "type": "string"
+ },
+ "accessToken": {
+ "type": "string"
+ },
+ "accessTokenSecret": {
+ "type": "string"
+ }
+ }
+ },
+ "basicauth": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "username": {
+ "type": "string"
+ },
+ "password": {
+ "type": "string"
+ }
+ }
+ },
+ "retrySleepMs": {
+ "type": "integer",
+ "description": "ms to sleep when hitting a rate limit",
+ "default": 100000
+ },
+ "retryMax": {
+ "type": "integer",
+ "description": "ms to sleep when hitting a rate limit",
+ "default": 10
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
new file mode 100644
index 0000000..89fc7af
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration",
+ "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "ids_only": {
+ "type": "boolean",
+ "description": "Whether to collect ids only, or full profiles",
+ "default": "true"
+ },
+ "max_items": {
+ "type": "integer",
+ "description": "Max items per user to collect",
+ "default": 50000
+ },
+ "max_pages": {
+ "type": "integer",
+ "description": "Max pages per user to request",
+ "default": 10
+ },
+ "page_size": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 5000
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
new file mode 100644
index 0000000..6fa2a73
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json
@@ -0,0 +1,45 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration",
+ "extends": {"$ref":"TwitterConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "includeEntities": {
+ "type": "string"
+ },
+ "truncated": {
+ "type": "boolean"
+ },
+ "filter-level": {
+ "type": "string",
+ "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream"
+ },
+ "with": {
+ "type": "string",
+ "description": "Typically following or user"
+ },
+ "replies": {
+ "type": "string",
+ "description": "Set to all, to see all @replies"
+ },
+ "follow": {
+ "type": "array",
+ "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "track": {
+ "type": "array",
+ "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
new file mode 100644
index 0000000..37ed60e
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration",
+ "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "max_items": {
+ "type": "integer",
+ "description": "Max items per user to collect",
+ "default": 3200
+ },
+ "max_pages": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 16
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
new file mode 100644
index 0000000..405c87a
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration",
+ "extends": {"$ref":"TwitterConfiguration.json"},
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "info": {
+ "type": "array",
+ "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
+ "items": {
+ "type": "string"
+ }
+ },
+ "page_size": {
+ "type": "integer",
+ "description": "Max items per page to request",
+ "default": 200
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
new file mode 100644
index 0000000..03b801d
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
@@ -0,0 +1,37 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Delete",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "delete": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "status": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "user_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "user_id_str": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
new file mode 100644
index 0000000..320db12
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
@@ -0,0 +1,18 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Follow",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "follower": {
+ "$ref": "User.json"
+ },
+ "followee": {
+ "$ref": "User.json"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
new file mode 100644
index 0000000..ae7a021
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
@@ -0,0 +1,24 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.FriendList",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "friends": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "friends_str": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
new file mode 100644
index 0000000..a449525
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.Retweet",
+ "javaInterfaces": ["java.io.Serializable"],
+ "extends": {"$ref":"tweet.json"},
+ "properties": {
+ "retweeted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being retweeted.",
+ "$ref" : "tweet.json"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
new file mode 100644
index 0000000..c6cb798
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
@@ -0,0 +1,129 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.User",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "location": {
+ "type": "string"
+ },
+ "default_profile": {
+ "type": "boolean"
+ },
+ "statuses_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_background_tile": {
+ "type": "boolean"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "profile_link_color": {
+ "type": "string"
+ },
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "protected": {
+ "type": "boolean"
+ },
+ "favourites_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_text_color": {
+ "type": "string"
+ },
+ "verified": {
+ "type": "boolean"
+ },
+ "description": {
+ "type": "string"
+ },
+ "contributors_enabled": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "string"
+ },
+ "profile_sidebar_border_color": {
+ "type": "string"
+ },
+ "profile_background_color": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "default_profile_image": {
+ "type": "boolean"
+ },
+ "followers_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "geo_enabled": {
+ "type": "boolean"
+ },
+ "profile_image_url_https": {
+ "type": "string"
+ },
+ "profile_background_image_url": {
+ "type": "string"
+ },
+ "profile_background_image_url_https": {
+ "type": "string"
+ },
+ "follow_request_sent": {
+ "type": "boolean"
+ },
+ "url": {
+ "type": "string"
+ },
+ "utc_offset": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "time_zone": {
+ "type": "string"
+ },
+ "profile_use_background_image": {
+ "type": "boolean"
+ },
+ "friends_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "profile_sidebar_fill_color": {
+ "type": "string"
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "profile_image_url": {
+ "type": "string"
+ },
+ "is_translator": {
+ "type": "boolean"
+ },
+ "listed_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "status": {
+ "$ref": "tweet.json"
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
new file mode 100644
index 0000000..23f8365
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
@@ -0,0 +1,52 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
+ "javaInterfaces": ["java.io.Serializable"],
+ "description": "",
+ "properties": {
+ "created_at": {
+ "type": "string",
+ "format" : "date-time"
+ },
+ "event_type": {
+ "type": "string",
+ "enum" : [
+ "access_revoked",
+ "block",
+ "unblock",
+ "favorite",
+ "unfavorite",
+ "follow",
+ "unfollow",
+ "list_created",
+ "list_destroyed",
+ "list_updated",
+ "list_member_added",
+ "list_member_removed",
+ "list_user_subscribed",
+ "list_user_unsubscribed",
+ "user_update"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target_object": {
+ "type": "object"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
new file mode 100644
index 0000000..e529559
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
@@ -0,0 +1,219 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.Tweet",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "retweeted": {
+ "type": "boolean"
+ },
+ "in_reply_to_screen_name": {
+ "type": "string"
+ },
+ "truncated": {
+ "type": "boolean"
+ },
+ "filter_level": {
+ "type": "string"
+ },
+ "contributors": {
+ "ignore_malformed": false,
+ "type": "array",
+ "items": [{
+ "type" : "number"
+ }]
+ },
+ "place": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.Place",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "bounding_box": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "type": {
+ "type": "string"
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "entities": {
+ "type": "object",
+ "dynamic": "true",
+ "javaType" : "org.apache.streams.twitter.pojo.Entities",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "user_mentions": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType" : "org.apache.streams.twitter.pojo.UserMentions",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [{
+ "type" : "integer"
+ }]
+ },
+ "screen_name": {
+ "type": "string"
+ },
+ "id_str": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "hashtags": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType": "org.apache.streams.twitter.pojo.Hashtag",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [{
+ "type": "integer"
+ }]
+ }
+ }
+ }
+ },
+ "urls": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "javaType": "org.apache.streams.twitter.Url",
+ "javaInterfaces": ["java.io.Serializable"],
+ "properties": {
+ "expanded_url": {
+ "type": "string"
+ },
+ "indices": {
+ "type": "array",
+ "items": [
+ {
+ "type" : "integer"
+ }
+ ]
+ },
+ "display_url": {
+ "type": "string"
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "in_reply_to_status_id_str": {
+ "type": "string"
+ },
+ "id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "in_reply_to_user_id_str": {
+ "type": "string"
+ },
+ "source": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string"
+ },
+ "favorited": {
+ "type": "boolean"
+ },
+ "possibly_sensitive": {
+ "type": "boolean"
+ },
+ "in_reply_to_status_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "created_at": {
+ "type" : "string",
+ "format" : "date-time"
+ },
+ "in_reply_to_user_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "favorite_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "retweet_count": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "id_str": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "User.json"
+ },
+ "is_quote_status": {
+ "type": "boolean"
+ },
+ "quoted_status_id": {
+ "ignore_malformed": false,
+ "type": "integer"
+ },
+ "quoted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being quoted.",
+ "$ref" : "tweet.json"
+ },
+ "retweeted_status_id": {
+ "type": "integer"
+ },
+ "retweeted_status": {
+ "type": "object",
+ "required" : false,
+ "description" : "Describes the tweet being retweeted.",
+ "$ref" : "tweet.json"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
deleted file mode 100644
index d8967d1..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.Video;
-import com.youtube.serializer.YoutubeActivityUtil;
-import com.youtube.serializer.YoutubeChannelDeserializer;
-import com.youtube.serializer.YoutubeEventClassifier;
-import com.youtube.serializer.YoutubeVideoDeserializer;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-
-public class YoutubeTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "YoutubeTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverter.class);
-
- private StreamsJacksonMapper mapper;
- private Queue<Video> inQueue;
- private Queue<StreamsDatum> outQueue;
- private int count = 0;
-
- public YoutubeTypeConverter() {}
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum streamsDatum) {
- StreamsDatum result = null;
-
- try {
- Object item = streamsDatum.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- Activity activity;
-
- if (item instanceof String) {
- item = deserializeItem(item);
- }
-
- if (item instanceof Video) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Video)item, activity, streamsDatum.getId());
- } else if (item instanceof Channel) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Channel)item, activity, null);
- } else {
- throw new NotImplementedException("Type conversion not implement for type : " + item.getClass().getName());
- }
-
- if (activity != null) {
- result = new StreamsDatum(activity);
- count++;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while converting Video to Activity: {}", ex);
- }
-
- if ( result != null ) {
- List<StreamsDatum> streamsDatumList = new ArrayList<>();
- streamsDatumList.add(result);
- return streamsDatumList;
- } else {
- return new ArrayList<>();
- }
- }
-
- private Object deserializeItem(Object item) {
- try {
- Class klass = YoutubeEventClassifier.detectClass((String) item);
- if (klass.equals(Video.class)) {
- item = mapper.readValue((String) item, Video.class);
- } else if (klass.equals(Channel.class)) {
- item = mapper.readValue((String) item, Channel.class);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserializeItem: {}", ex);
- }
-
- return item;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- mapper.registerModule(simpleModule);
- simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Channel.class, new YoutubeChannelDeserializer());
- mapper.registerModule(simpleModule);
- }
-
- @Override
- public void cleanUp() {}
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
deleted file mode 100644
index fd238db..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.gson.Gson;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects YoutubeChannelData on behalf of YoutubeChannelProvider.
- */
-public class YoutubeChannelDataCollector extends YoutubeDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelDataCollector.class);
- private static final String CONTENT = "snippet,contentDetails,statistics,topicDetails";
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_ATTEMPTS = 5;
-
- private YouTube youTube;
- private BlockingQueue<StreamsDatum> queue;
- private BackOffStrategy strategy;
- private UserInfo userInfo;
- private YoutubeConfiguration youtubeConfig;
-
- /**
- * YoutubeChannelDataCollector constructor.
- * @param youTube YouTube
- * @param queue BlockingQueue of StreamsDatum
- * @param strategy BackOffStrategy
- * @param userInfo UserInfo
- * @param youtubeConfig YoutubeConfiguration
- */
- public YoutubeChannelDataCollector(
- YouTube youTube,
- BlockingQueue<StreamsDatum> queue,
- BackOffStrategy strategy,
- UserInfo userInfo,
- YoutubeConfiguration youtubeConfig) {
- this.youTube = youTube;
- this.queue = queue;
- this.strategy = strategy;
- this.userInfo = userInfo;
- this.youtubeConfig = youtubeConfig;
- }
-
- @Override
- public void run() {
- Gson gson = new Gson();
- try {
- int attempt = 0;
- YouTube.Channels.List channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setKey(this.youtubeConfig.getApiKey());
- boolean tryAgain = false;
- do {
- try {
- List<Channel> channels = channelLists.execute().getItems();
- for (Channel channel : channels) {
- String json = gson.toJson(channel);
- this.queue.put(new StreamsDatum(json, channel.getId()));
- }
- if (StringUtils.isEmpty(channelLists.getPageToken())) {
- channelLists = null;
- } else {
- channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setOauthToken(this.youtubeConfig.getApiKey())
- .setPageToken(channelLists.getPageToken());
- }
- } catch (GoogleJsonResponseException gjre) {
- LOGGER.warn("GoogleJsonResposneException caught : {}", gjre);
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.strategy);
- ++attempt;
- } catch (Throwable throwable) {
- LOGGER.warn("Unable to get channel info for id : {}", this.userInfo.getUserId());
- LOGGER.warn("Excpection thrown while trying to get channel info : {}", throwable);
- }
- }
- while ((tryAgain && attempt < MAX_ATTEMPTS) || channelLists != null);
-
- } catch (Throwable throwable) {
- LOGGER.warn(throwable.getMessage());
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
deleted file mode 100644
index d9b1e14..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.youtube.pojo.YoutubeConfiguration;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of channels.
- */
-public class YoutubeChannelProvider extends YoutubeProvider {
-
- public YoutubeChannelProvider() {
- super();
- }
-
- public YoutubeChannelProvider(YoutubeConfiguration config) {
- super(config);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- return new YoutubeChannelDataCollector(youtube, queue, strategy, userInfo, this.config);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * youtube.oauth.pathToP12KeyFile
- * youtube.oauth.serviceAccountEmailAddress
- * youtube.apiKey
- * youtube.youtubeUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
- YoutubeChannelProvider provider = new YoutubeChannelProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- if ( datum.getDocument() instanceof String ) {
- json = (String) datum.getDocument();
- } else {
- json = mapper.writeValueAsString(datum.getDocument());
- }
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
deleted file mode 100644
index 3eede18..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.youtube.provider;
-
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base Collector for Youtube Data.
- */
-public abstract class YoutubeDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeDataCollector.class);
-
- /**
- * Looks at the status code of the expception. If the code indicates that the request should be retried,
- * it executes the back off strategy and returns true.
- * @param gjre
- * @param backOff
- * @return returns true if the error code of the exception indicates the request should be retried.
- */
- public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
- boolean tryAgain = false;
-
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request : {}", gjre);
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- break;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- backOff.backOff();
- break;
- }
-
- return tryAgain;
- }
-}
\ No newline at end of file
[8/9] incubator-streams git commit: STREAMS-463: Move every class in
all repos underneath org.apache.streams,
this closes apache/incubator-streams#356
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
deleted file mode 100644
index d9c7dd7..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/AbstractGPlusProvider.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.plus.Plus;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.gson.Gson;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Provider that creates a GPlus client and will run task that queue data to an outing queue.
- */
-public abstract class AbstractGPlusProvider implements StreamsProvider {
-
- public static final String STREAMS_ID = "AbstractGPlusProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AbstractGPlusProvider.class);
- private static final Set<String> SCOPE = new HashSet<String>() {
- {
- add("https://www.googleapis.com/auth/plus.login");
- }
- };
- private static final int MAX_BATCH_SIZE = 1000;
-
- private static final HttpTransport TRANSPORT = new NetHttpTransport();
- private static final JacksonFactory JSON_FACTORY = new JacksonFactory();
- private static final Gson GSON = new Gson();
-
- private GPlusConfiguration config;
-
- List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- private ListeningExecutorService executor;
-
- private BlockingQueue<StreamsDatum> datumQueue;
- private AtomicBoolean isComplete;
- private boolean previousPullWasEmpty;
-
- protected GoogleCredential credential;
- protected Plus plus;
-
- public AbstractGPlusProvider() {
- this.config = new ComponentConfigurator<>(GPlusConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gplus"));
- }
-
- public AbstractGPlusProvider(GPlusConfiguration config) {
- this.config = config;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(config.getOauth().getPathToP12KeyFile());
- Objects.requireNonNull(config.getOauth().getAppName());
- Objects.requireNonNull(config.getOauth().getServiceAccountEmailAddress());
-
- try {
- this.plus = createPlusClient();
- } catch (IOException | GeneralSecurityException ex) {
- LOGGER.error("Failed to created oauth for GPlus : {}", ex);
- throw new RuntimeException(ex);
- }
- // GPlus rate limits you to 5 calls per second, so there is not a need to execute more than one
- // collector unless you have multiple oauth tokens
- //TODO make this configurable based on the number of oauth tokens
- this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
- this.datumQueue = new LinkedBlockingQueue<>(1000);
- this.isComplete = new AtomicBoolean(false);
- this.previousPullWasEmpty = false;
- }
-
- @Override
- public void startStream() {
-
- BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
- for (UserInfo user : this.config.getGooglePlusUsers()) {
- if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
- user.setAfterDate(this.config.getDefaultAfterDate());
- }
- if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- }
- this.executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.plus, user));
- }
- this.executor.shutdown();
- }
-
- protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo);
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- int batchCount = 0;
- while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
- StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
- if (datum != null) {
- ++batchCount;
- ComponentUtils.offerUntilSuccess(datum, batch);
- }
- }
- boolean pullIsEmpty = batch.isEmpty() && this.datumQueue.isEmpty() && this.executor.isTerminated();
- this.isComplete.set(this.previousPullWasEmpty && pullIsEmpty);
- this.previousPullWasEmpty = pullIsEmpty;
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @VisibleForTesting
- protected Plus createPlusClient() throws IOException, GeneralSecurityException {
- credential = new GoogleCredential.Builder()
- .setJsonFactory(JSON_FACTORY)
- .setTransport(TRANSPORT)
- .setServiceAccountScopes(SCOPE)
- .setServiceAccountId(this.config.getOauth().getServiceAccountEmailAddress())
- .setServiceAccountPrivateKeyFromP12File(new File(this.config.getOauth().getPathToP12KeyFile()))
- .build();
- return new Plus.Builder(TRANSPORT,JSON_FACTORY, credential).setApplicationName(this.config.getOauth().getAppName()).build();
- }
-
- @Override
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(this.executor, 10, 10);
- this.executor = null;
- }
-
- public GPlusConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(GPlusConfiguration config) {
- this.config = config;
- }
-
- /**
- * Set and overwrite the default before date that was read from the configuration file.
- * @param defaultBeforeDate defaultBeforeDate
- */
- public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
- this.config.setDefaultBeforeDate(defaultBeforeDate);
- }
-
- /**
- * Set and overwrite the default after date that was read from teh configuration file.
- * @param defaultAfterDate defaultAfterDate
- */
- public void setDefaultAfterDate(DateTime defaultAfterDate) {
- this.config.setDefaultAfterDate(defaultAfterDate);
- }
-
- /**
- * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
- * @param userIds userIds
- */
- public void setUserInfoWithDefaultDates(Set<String> userIds) {
- List<UserInfo> gplusUsers = new LinkedList<>();
- for (String userId : userIds) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(this.config.getDefaultAfterDate());
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- gplusUsers.add(user);
- }
- this.config.setGooglePlusUsers(gplusUsers);
- }
-
- /**
- * Set and overwrite user into from the configuration file. Only sets after date.
- * @param usersAndAfterDates usersAndAfterDates
- */
- public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
- List<UserInfo> gplusUsers = new LinkedList<>();
- for (String userId : usersAndAfterDates.keySet()) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(usersAndAfterDates.get(userId));
- gplusUsers.add(user);
- }
- this.config.setGooglePlusUsers(gplusUsers);
- }
-
- @Override
- public boolean isRunning() {
- if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isComplete.set(true);
- LOGGER.info("Exiting");
- }
- return !isComplete.get();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
deleted file mode 100644
index eda55d2..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusActivitySerializer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * GPlusActivitySerializer converts gplus activities to as1 activities.
- */
-public class GPlusActivitySerializer implements ActivitySerializer<com.google.api.services.plus.model.Activity> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
-
- AbstractGPlusProvider provider;
-
- public GPlusActivitySerializer(AbstractGPlusProvider provider) {
-
- this.provider = provider;
- }
-
- public GPlusActivitySerializer() {
- }
-
- @Override
- public String serializationFormat() {
- return "gplus.v1";
- }
-
- @Override
- public com.google.api.services.plus.model.Activity serialize(Activity deserialized) {
- throw new NotImplementedException("Not currently implemented");
- }
-
- @Override
- public Activity deserialize(com.google.api.services.plus.model.Activity gplusActivity) {
- Activity activity = new Activity();
-
- GooglePlusActivityUtil.updateActivity(gplusActivity, activity);
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<com.google.api.services.plus.model.Activity> serializedList) {
- throw new NotImplementedException("Not currently implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
deleted file mode 100644
index 351e5be..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusDataCollector.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * GPlusDataCollector collects GPlus Data on behalf of providers.
- */
-public abstract class GPlusDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusDataCollector.class);
-
- /**
- * Looks at the status code of the exception. If the code indicates that the request should be retried,
- * it executes the back off strategy and returns true.
- * @param gjre GoogleJsonResponseException
- * @param backOff BackOffStrategy
- * @return returns true if the error code of the exception indicates the request should be retried.
- */
- public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
- boolean tryAgain = false;
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request : {}", gjre);
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- break;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- backOff.backOff();
- break;
- }
- return tryAgain;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
deleted file mode 100644
index 31d8b5c..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityCollector.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.ActivityFeed;
-import com.google.gplus.serializer.util.GPlusActivityDeserializer;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects the public activities of a GPlus user. Has ability to filter by date ranges.
- */
-public class GPlusUserActivityCollector extends GPlusDataCollector {
-
- /**
- * Key for all public activities
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final String PUBLIC_COLLECTION = "public";
- /**
- * Max results allowed per request
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final long MAX_RESULTS = 100;
- private static final int MAX_ATTEMPTS = 5;
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- static { //set up mapper for Google Activity Object
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
- simpleModule.addSerializer(
- com.google.api.client.util.DateTime.class,
- new StdSerializer<com.google.api.client.util.DateTime>(com.google.api.client.util.DateTime.class) {
- @Override
- public void serialize(
- com.google.api.client.util.DateTime dateTime,
- JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider)
- throws IOException {
- jsonGenerator.writeString(dateTime.toStringRfc3339());
- }
- });
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private BlockingQueue<StreamsDatum> datumQueue;
- private BackOffStrategy backOff;
- private Plus plus;
- private UserInfo userInfo;
-
- /**
- * GPlusUserActivityCollector constructor.
- * @param plus Plus
- * @param datumQueue BlockingQueue<StreamsDatum>
- * @param backOff BackOffStrategy
- * @param userInfo UserInfo
- */
- public GPlusUserActivityCollector(Plus plus, BlockingQueue<StreamsDatum> datumQueue, BackOffStrategy backOff, UserInfo userInfo) {
- this.plus = plus;
- this.datumQueue = datumQueue;
- this.backOff = backOff;
- this.userInfo = userInfo;
- }
-
- @Override
- public void run() {
- collectActivityData();
- }
-
- protected void collectActivityData() {
- try {
- ActivityFeed feed = null;
- boolean tryAgain = false;
- int attempt = 0;
- DateTime afterDate = userInfo.getAfterDate();
- DateTime beforeDate = userInfo.getBeforeDate();
- do {
- try {
- if (feed == null) {
- feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
- .setMaxResults(MAX_RESULTS).execute();
- } else {
- feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
- .setMaxResults(MAX_RESULTS)
- .setPageToken(feed.getNextPageToken()).execute();
- }
- this.backOff.reset(); //successful pull reset api.
- for (com.google.api.services.plus.model.Activity activity : feed.getItems()) {
- DateTime published = new DateTime(activity.getPublished().getValue());
- if ((afterDate == null && beforeDate == null)
- || (beforeDate == null && afterDate.isBefore(published))
- || (afterDate == null && beforeDate.isAfter(published))
- || ((afterDate != null && beforeDate != null) && (afterDate.isBefore(published) && beforeDate.isAfter(published)))) {
- String json = MAPPER.writeValueAsString(activity);
- this.datumQueue.put(new StreamsDatum(json, activity.getId()));
- } else if (afterDate != null && afterDate.isAfter(published)) {
- feed.setNextPageToken(null); // do not fetch next page
- break;
- }
- }
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
- ++attempt;
- }
- }
- while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
- } catch (Throwable th) {
- if (th instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- th.printStackTrace();
- LOGGER.warn("Unable to pull Activities for user={} : {}",this.userInfo.getUserId(), th);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
deleted file mode 100644
index 97b08fd..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserActivityProvider.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.services.plus.Plus;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of accounts.
- *
- * <p/>
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * gplus.oauth.pathToP12KeyFile
- * gplus.oauth.serviceAccountEmailAddress
- * gplus.apiKey
- * gplus.googlePlusUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=com.google.gplus.provider.GPlusUserActivityProvider -Dexec.args="application.conf activity.json"
- */
-public class GPlusUserActivityProvider extends AbstractGPlusProvider {
-
- private static final String STREAMS_ID = "GPlusUserActivityProvider";
-
- public GPlusUserActivityProvider() {
- super();
- }
-
- public GPlusUserActivityProvider(GPlusConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- return new GPlusUserActivityCollector(plus, queue, strategy, userInfo);
- }
-
- /**
- * Retrieve recent activity from a list of accounts.
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
- GPlusUserActivityProvider provider = new GPlusUserActivityProvider(config);
-
- Gson gson = new Gson();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = gson.toJson(datum.getDocument());
- }
- outStream.println(json);
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
deleted file mode 100644
index e0df58d..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataCollector.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Person;
-import com.google.gplus.serializer.util.GPlusPersonDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects user profile information for a specific GPlus user.
- */
-public class GPlusUserDataCollector extends GPlusDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_ATTEMPTS = 5;
-
- static { //set up Mapper for Person objects
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private BackOffStrategy backOffStrategy;
- private Plus plus;
- private BlockingQueue<StreamsDatum> datumQueue;
- private UserInfo userInfo;
-
- /**
- * GPlusUserDataCollector constructor.
- * @param plus Plus
- * @param backOffStrategy BackOffStrategy
- * @param datumQueue BlockingQueue of StreamsDatum
- * @param userInfo UserInfo
- */
- public GPlusUserDataCollector(Plus plus, BackOffStrategy backOffStrategy, BlockingQueue<StreamsDatum> datumQueue, UserInfo userInfo) {
- this.plus = plus;
- this.backOffStrategy = backOffStrategy;
- this.datumQueue = datumQueue;
- this.userInfo = userInfo;
- }
-
- protected void queueUserHistory() {
- try {
- boolean tryAgain;
- int attempts = 0;
- com.google.api.services.plus.model.Person person = null;
- do {
- try {
- person = this.plus.people().get(userInfo.getUserId()).execute();
- this.backOffStrategy.reset();
- tryAgain = person == null;
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOffStrategy);
- }
- ++attempts;
- }
- while (tryAgain && attempts < MAX_ATTEMPTS);
- String json = MAPPER.writeValueAsString(person);
- this.datumQueue.put(new StreamsDatum(json, person.getId()));
- } catch (Throwable throwable) {
- LOGGER.warn("Unable to pull user data for user={} : {}", userInfo.getUserId(), throwable);
- if (throwable instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- @Override
- public void run() {
- queueUserHistory();
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
deleted file mode 100644
index 28bcb55..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/provider/GPlusUserDataProvider.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.services.plus.Plus;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve current profile status for a list of accounts.
- *
- * <p/>
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * gplus.oauth.pathToP12KeyFile
- * gplus.oauth.serviceAccountEmailAddress
- * gplus.apiKey
- * gplus.googlePlusUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=com.google.gplus.provider.GPlusUserDataProvider -Dexec.args="application.conf profiles.json"
- */
-public class GPlusUserDataProvider extends AbstractGPlusProvider {
-
- public static final String STREAMS_ID = "GPlusUserDataProvider";
-
- public GPlusUserDataProvider() {
- super();
- }
-
- public GPlusUserDataProvider(GPlusConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- return new GPlusUserDataCollector(plus, strategy, queue, userInfo);
- }
-
- /**
- * Retrieve current profile status for a list of accounts.
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
- GPlusUserDataProvider provider = new GPlusUserDataProvider(config);
-
- Gson gson = new Gson();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = gson.toJson(datum.getDocument());
- }
- outStream.println(json);
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
deleted file mode 100644
index f13267a..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusActivityDeserializer.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.plus.model.Activity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Custom deserializer for GooglePlus' Person model.
- */
-public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
-
- /**
- * Because the GooglePlus Activity object {@link com.google.api.services.plus.model.Activity} contains complex objects
- * within its hierarchy, we have to use a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Activity} object
- * @throws IOException IOException
- * @throws JsonProcessingException JsonProcessingException
- */
- @Override
- public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Activity activity = new Activity();
-
- try {
- activity.setUrl(node.get("url").asText());
- activity.setEtag(node.get("etag").asText());
- activity.setTitle(node.get("title").asText());
- activity.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
- activity.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
- activity.setId(node.get("id").asText());
- activity.setVerb(node.get("verb").asText());
-
- activity.setActor(buildActor(node));
-
- activity.setObject(buildPlusObject(node));
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
- }
-
- return activity;
- }
-
- /**
- * Given a raw JsonNode, build out the G+ {@link com.google.api.services.plus.model.Activity.Actor} object
- *
- * @param node node
- * @return {@link com.google.api.services.plus.model.Activity.Actor} object
- */
- private Activity.Actor buildActor(JsonNode node) {
- Activity.Actor actor = new Activity.Actor();
- JsonNode actorNode = node.get("actor");
-
- actor.setId(actorNode.get("id").asText());
- actor.setDisplayName(actorNode.get("displayName").asText());
- actor.setUrl(actorNode.get("url").asText());
-
- Activity.Actor.Image image = new Activity.Actor.Image();
- JsonNode imageNode = actorNode.get("image");
- image.setUrl(imageNode.get("url").asText());
-
- actor.setImage(image);
-
- return actor;
- }
-
- /**
- * Given a JsonNode, build out all aspects of the {@link com.google.api.services.plus.model.Activity.PlusObject} object
- *
- * @param node node
- * @return {@link com.google.api.services.plus.model.Activity.PlusObject} object
- */
- private Activity.PlusObject buildPlusObject(JsonNode node) {
- Activity.PlusObject object = new Activity.PlusObject();
- JsonNode objectNode = node.get("object");
- object.setObjectType(objectNode.get("objectType").asText());
- object.setContent(objectNode.get("content").asText());
- object.setUrl(objectNode.get("url").asText());
-
- Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
- JsonNode repliesNode = objectNode.get("replies");
- replies.setTotalItems(repliesNode.get("totalItems").asLong());
- replies.setSelfLink(repliesNode.get("selfLink").asText());
- object.setReplies(replies);
-
- Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
- JsonNode plusonersNode = objectNode.get("plusoners");
- plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
- plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
- object.setPlusoners(plusoners);
-
- Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
- JsonNode resharersNode = objectNode.get("resharers");
- resharers.setTotalItems(resharersNode.get("totalItems").asLong());
- resharers.setSelfLink(resharersNode.get("selfLink").asText());
- object.setResharers(resharers);
-
- object.setAttachments(buildAttachments(objectNode));//attachments);
-
- return object;
- }
-
- /**
- * Given a raw JsonNode representation of an Activity's attachments, build out that
- * list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
- *
- * @param objectNode objectNode
- * @return list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
- */
- private List<Activity.PlusObject.Attachments> buildAttachments(JsonNode objectNode) {
- List<Activity.PlusObject.Attachments> attachments = new ArrayList<>();
- if ( objectNode.has("attachments") ) {
- for (JsonNode attachmentNode : objectNode.get("attachments")) {
- Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
- attachments1.setObjectType(attachmentNode.get("objectType").asText());
- if (attachmentNode.has("displayName")) {
- attachments1.setDisplayName(attachmentNode.get("displayName").asText());
- }
- if (attachmentNode.has("content")) {
- attachments1.setContent(attachmentNode.get("content").asText());
- }
- if (attachmentNode.has("url")) {
- attachments1.setUrl(attachmentNode.get("url").asText());
- }
-
- if( attachmentNode.has("image")) {
- Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
- JsonNode imageNode1 = attachmentNode.get("image");
- image1.setUrl(imageNode1.get("url").asText());
- attachments1.setImage(image1);
- }
-
- attachments.add(attachments1);
- }
- }
- return attachments;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
deleted file mode 100644
index 5b92c43..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusCommentDeserializer.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.plus.model.Comment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * GPlusCommentDeserializer converts gplus comments to as1 comments.
- */
-public class GPlusCommentDeserializer extends JsonDeserializer<Comment> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
-
- /**
- * Because the GooglePlus Comment object {@link com.google.api.services.plus.model.Comment} contains complex objects
- * within its hierarchy, we have to use a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Comment} object
- * @throws java.io.IOException IOException
- * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
- */
- @Override
- public Comment deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
- throws IOException, JsonProcessingException {
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
- Comment comment = new Comment();
-
- try {
- comment.setEtag(node.get("etag").asText());
- comment.setVerb(node.get("verb").asText());
- comment.setId(node.get("id").asText());
- comment.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
- comment.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
-
- Comment.Actor actor = new Comment.Actor();
- JsonNode actorNode = node.get("actor");
- actor.setDisplayName(actorNode.get("displayName").asText());
- actor.setUrl(actorNode.get("url").asText());
-
- Comment.Actor.Image image = new Comment.Actor.Image();
- JsonNode imageNode = actorNode.get("image");
- image.setUrl(imageNode.get("url").asText());
-
- actor.setImage(image);
-
- comment.setObject(objectMapper.readValue(objectMapper.writeValueAsString(node.get("object")), Comment.PlusObject.class));
-
- comment.setSelfLink(node.get("selfLink").asText());
-
- List<Comment.InReplyTo> replies = new ArrayList<>();
- for (JsonNode reply : node.get("inReplyTo")) {
- Comment.InReplyTo irt = objectMapper.readValue(objectMapper.writeValueAsString(reply), Comment.InReplyTo.class);
- replies.add(irt);
- }
-
- comment.setInReplyTo(replies);
-
- Comment.Plusoners plusoners = new Comment.Plusoners();
- JsonNode plusonersNode = node.get("plusoners");
- plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
- comment.setPlusoners(plusoners);
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
- }
-
- return comment;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusEventClassifier.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusEventClassifier.java
deleted file mode 100644
index 4923d3e..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusEventClassifier.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.Person;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Objects;
-
-/**
- * GPlusEventClassifier classifies GPlus Events.
- */
-public class GPlusEventClassifier implements Serializable {
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private static final String ACTIVITY_IDENTIFIER = "\"plus#activity\"";
- private static final String PERSON_IDENTIFIER = "\"plus#person\"";
-
- /**
- * Detect likely class of String json.
- * @param json String json
- * @return likely class
- */
- public static Class detectClass(String json) {
- Objects.requireNonNull(json);
- Preconditions.checkArgument(StringUtils.isNotEmpty(json));
-
- ObjectNode objectNode;
- try {
- objectNode = (ObjectNode) mapper.readTree(json);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
-
- if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(ACTIVITY_IDENTIFIER)) {
- return Activity.class;
- } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(PERSON_IDENTIFIER)) {
- return Person.class;
- } else {
- return ObjectNode.class;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
deleted file mode 100644
index 38bb168..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GPlusPersonDeserializer.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.plus.model.Person;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Custom deserializer for GooglePlus' Person model.
- */
-public class GPlusPersonDeserializer extends JsonDeserializer<Person> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusPersonDeserializer.class);
-
- /**
- * Because the GooglePlus Person object contains complex objects within its hierarchy, we have to use
- * a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Person} object
- * @throws IOException IOException
- * @throws JsonProcessingException JsonProcessingException
- */
- @Override
- public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Person person = new Person();
- try {
- person.setId(node.get("id").asText());
- person.setCircledByCount((Integer) (node.get("circledByCount")).numberValue());
- person.setDisplayName(node.get("displayName").asText());
- if( node.has("etag")) {
- person.setEtag(node.get("etag").asText());
- }
- if( node.has("gender")) {
- person.setGender(node.get("gender").asText());
- }
-
- Person.Image image = new Person.Image();
- if( node.has("image") ) {
- JsonNode imageNode = node.get("image");
- image.setIsDefault(imageNode.get("isDefault").asBoolean());
- image.setUrl(imageNode.get("url").asText());
- person.setImage(image);
- }
-
- person.setIsPlusUser(node.get("isPlusUser").asBoolean());
- person.setKind(node.get("kind").asText());
-
- JsonNode nameNode = node.get("name");
- Person.Name name = mapper.readValue(mapper.writeValueAsString(nameNode), Person.Name.class);
- person.setName(name);
-
- person.setObjectType(node.get("objectType").asText());
-
- List<Person.Organizations> organizations = new ArrayList<>();
- if( node.has("organizations")) {
- for (JsonNode orgNode : node.get("organizations")) {
- Person.Organizations org = mapper.readValue(mapper.writeValueAsString(orgNode), Person.Organizations.class);
- organizations.add(org);
- }
- person.setOrganizations(organizations);
- }
-
- person.setUrl(node.get("url").asText());
- person.setVerified(node.get("verified").asBoolean());
-
- List<Person.Emails> emails = new ArrayList<>();
-
- if ( node.has("emails")) {
- for (JsonNode emailNode : node.get("emails")) {
- Person.Emails email = mapper.readValue(mapper.writeValueAsString(emailNode), Person.Emails.class);
- emails.add(email);
- }
- }
-
- if ( node.has("tagline")) {
- person.setTagline(node.get("tagline").asText());
- }
- if ( node.has("aboutMe")) {
- person.setAboutMe(node.get("aboutMe").asText());
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize a Person object: {}", ex);
- }
-
- return person;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
deleted file mode 100644
index 27b25a2..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/com/google/gplus/serializer/util/GooglePlusActivityUtil.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.google.gplus.serializer.util;
-
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import com.google.api.services.plus.model.Comment;
-import com.google.api.services.plus.model.Person;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusActivityUtil helps convert c.g.Person and c.g.Activity into o.a.s.p.j.o.Page and o.a.s.p.j.Activity.
- */
-public class GooglePlusActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivityUtil.class);
-
- /**
- * Given a {@link Person} object and an
- * {@link Activity} object, fill out the appropriate details.
- *
- * @param item Person
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Person item, Activity activity) throws ActivitySerializerException {
- activity.setActor(buildActor(item));
- activity.setVerb("update");
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(item.getId()).orElse(null)));
-
- activity.setProvider(getProvider());
- }
-
- /**
- * Given a {@link List} of {@link Comment} objects and an
- * {@link Activity}, update that Activity to contain all comments
- *
- * @param comments input List of Comment
- * @param activity output Activity
- */
- public static void updateActivity(List<Comment> comments, Activity activity) {
- for (Comment comment : comments) {
- addComment(activity, comment);
- }
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("comment_count", comments.size());
- }
-
- /**
- * Given a Google Plus {@link com.google.api.services.plus.model.Activity},
- * convert that into an Activity streams formatted {@link Activity}
- *
- * @param gPlusActivity input c.g.a.s.p.m.Activity
- * @param activity output o.a.s.p.j.Activity
- */
- public static void updateActivity(com.google.api.services.plus.model.Activity gPlusActivity, Activity activity) {
- activity.setActor(buildActor(gPlusActivity.getActor()));
- activity.setVerb("post");
- activity.setTitle(gPlusActivity.getTitle());
- activity.setUrl(gPlusActivity.getUrl());
- activity.setProvider(getProvider());
-
- if (gPlusActivity.getObject() != null) {
- activity.setContent(gPlusActivity.getObject().getContent());
- }
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(gPlusActivity.getId()).orElse(null)));
-
- DateTime published = new DateTime(String.valueOf(gPlusActivity.getPublished()));
- activity.setPublished(published);
-
- setObject(activity, gPlusActivity.getObject());
- addGPlusExtensions(activity, gPlusActivity);
- }
-
- /**
- * Adds a single {@link Comment} to the Object.Attachments
- * section of the passed in {@link Activity}
- *
- * @param activity output o.a.s.p.j.Activity
- * @param comment input c.g.a.s.p.m.Comment
- */
- private static void addComment(Activity activity, Comment comment) {
- ActivityObject obj = new ActivityObject();
-
- obj.setId(comment.getId());
- obj.setPublished(new DateTime(String.valueOf(comment.getPublished())));
- obj.setUpdated(new DateTime(String.valueOf(comment.getUpdated())));
- obj.setContent(comment.getObject().getContent());
- obj.setObjectType(comment.getObject().getObjectType());
-
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("googlePlus", comment);
-
- obj.setAdditionalProperty("extensions", extensions);
-
- if (activity.getObject() == null) {
- activity.setObject(new ActivityObject());
- }
- if (activity.getObject().getAttachments() == null) {
- activity.getObject().setAttachments(new ArrayList<>());
- }
-
- activity.getObject().getAttachments().add(obj);
- }
-
- /**
- * Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
- * {@link Activity} object
- *
- * @param activity output o.a.s.p.j.Activity
- * @param gPlusActivity input c.g.a.s.p.m.Activity
- */
- private static void addGPlusExtensions(Activity activity, com.google.api.services.plus.model.Activity gPlusActivity) {
-
- activity.getAdditionalProperties().put("googlePlus", gPlusActivity);
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- com.google.api.services.plus.model.Activity.PlusObject object = gPlusActivity.getObject();
-
- if (object != null) {
- com.google.api.services.plus.model.Activity.PlusObject.Plusoners plusoners = object.getPlusoners();
- if (plusoners != null) {
- Map<String, Object> likes = new HashMap<>();
- likes.put("count", plusoners.getTotalItems());
- extensions.put("likes", likes);
- }
-
- com.google.api.services.plus.model.Activity.PlusObject.Resharers resharers = object.getResharers();
- if (resharers != null) {
- Map<String, Object> rebroadcasts = new HashMap<>();
- rebroadcasts.put("count", resharers.getTotalItems());
- extensions.put("rebroadcasts", rebroadcasts);
- }
-
- extensions.put("keywords", object.getContent());
- }
- }
-
- /**
- * Set the {@link ActivityObject} field given the passed in
- * {@link com.google.api.services.plus.model.Activity.PlusObject}
- *
- * @param activity output $.object as o.a.s.p.j.ActivityObject
- * @param plusObject input c.g.a.s.p.m.Activity.PlusObject
- */
- private static void setObject(Activity activity, com.google.api.services.plus.model.Activity.PlusObject plusObject) {
- if (plusObject != null) {
- ActivityObject activityObject = new ActivityObject();
-
- activityObject.setContent(plusObject.getContent());
- activityObject.setObjectType(plusObject.getObjectType());
-
- List<ActivityObject> attachmentsList = new ArrayList<>();
- for (com.google.api.services.plus.model.Activity.PlusObject.Attachments attachments : plusObject.getAttachments()) {
- ActivityObject attach = new ActivityObject();
-
- attach.setContent(attachments.getContent());
- attach.setDisplayName(attachments.getDisplayName());
- attach.setObjectType(attachments.getObjectType());
- attach.setUrl(attachments.getUrl());
-
- Image image = new Image();
- com.google.api.services.plus.model.Activity.PlusObject.Attachments.Image image1 = attachments.getImage();
-
- if (image1 != null) {
- image.setUrl(image1.getUrl());
- attach.setImage(image);
- }
-
- attachmentsList.add(attach);
- }
-
- activityObject.setAttachments(attachmentsList);
-
- activity.setObject(activityObject);
- }
- }
-
- /**
- * Given a {@link com.google.api.services.plus.model.Activity.Actor} object, return a fully fleshed
- * out {@link ActivityObject} actor
- *
- * @param gPlusActor input c.g.a.s.p.m.Activity.Actor
- * @return {@link ActivityObject} output $.actor as o.a.s.p.j.ActivityObject
- */
- private static ActivityObject buildActor(com.google.api.services.plus.model.Activity.Actor gPlusActor) {
- ActivityObject actor = new ActivityObject();
-
- actor.setDisplayName(gPlusActor.getDisplayName());
- actor.setId(formatId(String.valueOf(gPlusActor.getId())));
- actor.setUrl(gPlusActor.getUrl());
-
- Image image = new Image();
- com.google.api.services.plus.model.Activity.Actor.Image googlePlusImage = gPlusActor.getImage();
-
- if (googlePlusImage != null) {
- image.setUrl(googlePlusImage.getUrl());
- }
- actor.setImage(image);
-
- return actor;
- }
-
- /**
- * Extract the relevant details from the passed in {@link Person} object and build
- * an actor with them
- *
- * @param person Person
- * @return Actor constructed with relevant Person details
- */
- private static ActivityObject buildActor(Person person) {
- ActivityObject actor = new ActivityObject();
-
- actor.setUrl(person.getUrl());
- actor.setDisplayName(person.getDisplayName());
- actor.setId(formatId(String.valueOf(person.getId())));
-
- if (person.getAboutMe() != null) {
- actor.setSummary(person.getAboutMe());
- } else if (person.getTagline() != null) {
- actor.setSummary(person.getTagline());
- }
-
- Image image = new Image();
- Person.Image googlePlusImage = person.getImage();
-
- if (googlePlusImage != null) {
- image.setUrl(googlePlusImage.getUrl());
- }
- actor.setImage(image);
-
- Map<String, Object> extensions = new HashMap<>();
-
- extensions.put("followers", person.getCircledByCount());
- extensions.put("googleplus", person);
- actor.setAdditionalProperty("extensions", extensions);
-
- return actor;
- }
-
- /**
- * Gets the common googleplus {@link Provider} object
- * @return a provider object representing GooglePlus
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:googleplus");
- provider.setDisplayName("GooglePlus");
- return provider;
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:googleplus:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java
new file mode 100644
index 0000000..1128a93
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.processor;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.pojo.json.Activity;
+
+import com.google.api.services.plus.model.Comment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * GooglePlusCommentProcessor collects comments about a google plus activity.
+ */
+public class GooglePlusCommentProcessor implements StreamsProcessor {
+
+ private static final String STREAMS_ID = "GooglePlusCommentProcessor";
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentProcessor.class);
+ private int count;
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = entry.getDocument();
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+
+ //Get G+ activity ID from our own activity ID
+ if (item instanceof Activity) {
+ Activity activity = (Activity) item;
+ String activityId = getGPlusID(activity.getId());
+
+ //Call Google Plus API to get list of comments for this activity ID
+ /* TODO: FILL ME OUT WITH THE API CALL **/
+ List<Comment> comments = new ArrayList<>();
+
+ GooglePlusActivityUtil.updateActivity(comments, activity);
+ result = new StreamsDatum(activity);
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ LOGGER.error("Exception while converting Comment to Activity: {}", ex.getMessage());
+ }
+
+ if ( result != null ) {
+ return Stream.of(result).collect(Collectors.toList());
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ count = 0;
+ }
+
+ @Override
+ public void cleanUp() {
+
+ }
+
+ private String getGPlusID(String activityId) {
+ String[] activityParts = activityId.split(":");
+ return (activityParts.length > 0) ? activityParts[activityParts.length - 1] : "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java
new file mode 100644
index 0000000..ff11b45
--- /dev/null
+++ b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.gplus.processor;
+
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
+import org.apache.streams.gplus.serializer.util.GPlusEventClassifier;
+import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
+import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.plus.model.Person;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * GooglePlusTypeConverter is a StreamsProcessor that converts gplus activities to activitystreams activities.
+ */
+public class GooglePlusTypeConverter implements StreamsProcessor {
+
+ public static final String STREAMS_ID = "GooglePlusTypeConverter";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverter.class);
+ private StreamsJacksonMapper mapper;
+ private Queue<Person> inQueue;
+ private Queue<StreamsDatum> outQueue;
+ private int count = 0;
+
+ public GooglePlusTypeConverter() {}
+
+ public Queue<StreamsDatum> getProcessorOutputQueue() {
+ return outQueue;
+ }
+
+ public void setProcessorInputQueue(Queue<Person> inputQueue) {
+ inQueue = inputQueue;
+ }
+
+ @Override
+ public String getId() {
+ return STREAMS_ID;
+ }
+
+ @Override
+ public List<StreamsDatum> process(StreamsDatum entry) {
+ StreamsDatum result = null;
+
+ try {
+ Object item = entry.getDocument();
+
+ LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
+ Activity activity = null;
+
+ if (item instanceof String) {
+ item = deserializeItem(item);
+ }
+
+ if (item instanceof Person) {
+ activity = new Activity();
+ GooglePlusActivityUtil.updateActivity((Person) item, activity);
+ } else if (item instanceof com.google.api.services.plus.model.Activity) {
+ activity = new Activity();
+ GooglePlusActivityUtil.updateActivity((com.google.api.services.plus.model.Activity) item, activity);
+ }
+
+ if (activity != null) {
+ result = new StreamsDatum(activity);
+ count++;
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ LOGGER.error("Exception while converting Person to Activity: {}", ex.getMessage());
+ }
+
+ if (result != null) {
+ return Stream.of(result).collect(Collectors.toList());
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+ private Object deserializeItem(Object item) {
+ try {
+ Class klass = GPlusEventClassifier.detectClass((String) item);
+
+ if (klass.equals(Person.class)) {
+ item = mapper.readValue((String) item, Person.class);
+ } else if (klass.equals(com.google.api.services.plus.model.Activity.class)) {
+ item = mapper.readValue((String) item, com.google.api.services.plus.model.Activity.class);
+ }
+ } catch (Exception ex) {
+ LOGGER.error("Exception while trying to deserializeItem: {}", ex);
+ }
+
+ return item;
+ }
+
+ @Override
+ public void prepare(Object configurationObject) {
+ mapper = StreamsJacksonMapper.getInstance();
+
+ SimpleModule simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
+ mapper.registerModule(simpleModule);
+
+ simpleModule = new SimpleModule();
+ simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
+ mapper.registerModule(simpleModule);
+ }
+
+ @Override
+ public void cleanUp() {
+ //No-op
+ }
+}