You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/07/21 17:45:32 UTC
[46/47] git commit: tested UserInformationProvider it's running but
leaving a lot of fields empty
tested UserInformationProvider
it's running but leaving a lot of fields empty
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/3f6a015b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/3f6a015b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/3f6a015b
Branch: refs/heads/STREAMS-46
Commit: 3f6a015ba605a680aeafe513c0b5cc6c26fb2c40
Parents: 49d00d7
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu Jul 3 20:56:43 2014 -0700
Committer: sblackmon <sb...@w2odigital.com>
Committed: Mon Jul 21 10:30:44 2014 -0500
----------------------------------------------------------------------
.../FacebookUserInformationProvider.java | 116 +++++--------------
1 file changed, 27 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/3f6a015b/streams-contrib/streams-provider-facebook/src/main/java/com/facebook/provider/FacebookUserInformationProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/com/facebook/provider/FacebookUserInformationProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/com/facebook/provider/FacebookUserInformationProvider.java
index b89a925..a167947 100644
--- a/streams-contrib/streams-provider-facebook/src/main/java/com/facebook/provider/FacebookUserInformationProvider.java
+++ b/streams-contrib/streams-provider-facebook/src/main/java/com/facebook/provider/FacebookUserInformationProvider.java
@@ -64,8 +64,7 @@ public class FacebookUserInformationProvider implements StreamsProvider, Seriali
public void setConfig(FacebookUserInformationConfiguration config) { this.facebookUserInformationConfiguration = config; }
- protected Iterator<Long[]> idsBatches;
- protected Iterator<String[]> screenNameBatches;
+ protected Iterator<String[]> idsBatches;
protected ExecutorService executor;
@@ -121,32 +120,9 @@ public class FacebookUserInformationProvider implements StreamsProvider, Seriali
running.set(true);
}
- private void loadBatch(String[] ids) {
- Facebook client = getFacebookClient();
- int keepTrying = 0;
-
- // keep trying to load, give it 5 attempts.
- //while (keepTrying < 10)
- while (keepTrying < 1)
- {
- try {
- for( User user : client.getUsers(ids)) {
- String json = DataObjectFactory.getRawJSON(user);
-
- providerQueue.offer(new StreamsDatum(json));
-//
- };
- } catch (FacebookException e) {
- e.printStackTrace();
- return;
- }
-
- }
- }
-
public StreamsResultSet readCurrent() {
-// Preconditions.checkArgument(idsBatches.hasNext() || screenNameBatches.hasNext());
+ Preconditions.checkArgument(idsBatches.hasNext());
LOGGER.info("readCurrent");
@@ -154,48 +130,34 @@ public class FacebookUserInformationProvider implements StreamsProvider, Seriali
try {
User me = client.users().getMe();
- ResponseList<Friend> friendResponseList = client.friends().getFriends(me.getId());
- Paging<Friend> friendPaging;
- do {
+ String json = mapper.writeValueAsString(me);
+ providerQueue.add(
+ new StreamsDatum(json, DateTime.now())
+ );
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ } catch (FacebookException e) {
+ e.printStackTrace();
+ }
- for( Friend friend : friendResponseList ) {
+ while( idsBatches.hasNext() ) {
+ try {
+ List<User> userList = client.users().getUsers(idsBatches.next());
+ for (User user : userList) {
- String json;
try {
- json = mapper.writeValueAsString(friend);
+ String json = mapper.writeValueAsString(user);
providerQueue.add(
- new StreamsDatum(json)
+ new StreamsDatum(json, DateTime.now())
);
} catch (JsonProcessingException e) {
-// e.printStackTrace();
+ // e.printStackTrace();
}
}
- friendPaging = friendResponseList.getPaging();
- friendResponseList = client.fetchNext(friendPaging);
- } while( friendPaging != null &&
- friendResponseList != null );
-
-// // Getting Next page
-// for( Post post : feed ) {
-// String json;
-// try {
-// json = mapper.writeValueAsString(post);
-// providerQueue.add(
-// new StreamsDatum(json)
-// );
-// } catch (JsonProcessingException e) {
-// e.printStackTrace();
-// }
-// }
-
-// paging = feed.getPaging();
-// feed = client.fetchPrevious(paging);
-// } while( paging != null &&
-// feed != null );
-
- } catch (FacebookException e) {
- e.printStackTrace();
+ } catch (FacebookException e) {
+ e.printStackTrace();
+ }
}
LOGGER.info("Finished. Cleaning up...");
@@ -260,52 +222,28 @@ public class FacebookUserInformationProvider implements StreamsProvider, Seriali
Preconditions.checkNotNull(facebookUserInformationConfiguration.getOauth().getUserAccessToken());
Preconditions.checkNotNull(facebookUserInformationConfiguration.getInfo());
- List<String> screenNames = new ArrayList<String>();
- List<String[]> screenNameBatches = new ArrayList<String[]>();
-
- List<Long> ids = new ArrayList<Long>();
- List<Long[]> idsBatches = new ArrayList<Long[]>();
+ List<String> ids = new ArrayList<String>();
+ List<String[]> idsBatches = new ArrayList<String[]>();
for(String s : facebookUserInformationConfiguration.getInfo()) {
if(s != null)
{
- String potentialScreenName = s.replaceAll("@", "").trim().toLowerCase();
-
- // See if it is a long, if it is, add it to the user iD list, if it is not, add it to the
- // screen name list
- try {
- ids.add(Long.parseLong(potentialScreenName));
- } catch (Exception e) {
- screenNames.add(potentialScreenName);
- }
-
- // Twitter allows for batches up to 100 per request, but you cannot mix types
+ ids.add(s);
if(ids.size() >= 100) {
// add the batch
- idsBatches.add(ids.toArray(new Long[ids.size()]));
+ idsBatches.add(ids.toArray(new String[ids.size()]));
// reset the Ids
- ids = new ArrayList<Long>();
+ ids = new ArrayList<String>();
}
- if(screenNames.size() >= 100) {
- // add the batch
- screenNameBatches.add(screenNames.toArray(new String[ids.size()]));
- // reset the Ids
- screenNames = new ArrayList<String>();
- }
}
}
-
if(ids.size() > 0)
- idsBatches.add(ids.toArray(new Long[ids.size()]));
-
- if(screenNames.size() > 0)
- screenNameBatches.add(screenNames.toArray(new String[ids.size()]));
+ idsBatches.add(ids.toArray(new String[ids.size()]));
this.idsBatches = idsBatches.iterator();
- this.screenNameBatches = screenNameBatches.iterator();
}
protected Facebook getFacebookClient()