You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/19 01:06:36 UTC
[4/6] incubator-streams git commit: Resolves STREAMS-416
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
deleted file mode 100644
index fbd5e52..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.DataSiftClient;
-import com.datasift.client.DataSiftConfig;
-import com.datasift.client.core.Stream;
-import com.datasift.client.stream.DeletedInteraction;
-import com.datasift.client.stream.Interaction;
-import com.datasift.client.stream.StreamEventListener;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.datasift.DatasiftConfiguration;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * Requires Java Version 1.7!
- * {@code DatasiftStreamProvider} is an implementation of the {@link org.apache.streams.core.StreamsProvider} interface. The provider
- * uses the Datasift java api to make connections. A single provider creates one connection per StreamHash in the configuration.
- */
-public class DatasiftStreamProvider implements StreamsProvider {
-
- private final static String STREAMS_ID = "DatasiftStreamProvider";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class);
-
- private DatasiftConfiguration config;
- private ConcurrentLinkedQueue<Interaction> interactions = new ConcurrentLinkedQueue<Interaction>();
- private Map<String, DataSiftClient> clients;
- private StreamEventListener eventListener;
- private ObjectMapper mapper;
-
- /**
- * Constructor that searches for available configurations
- *
- * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter.
- */
- public DatasiftStreamProvider(StreamEventListener listener) {
- this(listener, null);
- }
-
- // to set up a webhook we need to be able to return a reference to this queue
- public Queue<Interaction> getInteractions() {
- return interactions;
- }
-
- /**
- * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter.
- * @param config Configuration to use
- */
- public DatasiftStreamProvider(StreamEventListener listener, DatasiftConfiguration config) {
- if (config == null) {
- Config datasiftConfig = StreamsConfigurator.config.getConfig("datasift");
- this.config = DatasiftStreamConfigurator.detectConfiguration(datasiftConfig);
- } else {
- this.config = config;
- }
- this.eventListener = listener;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- Preconditions.checkNotNull(this.config);
- Preconditions.checkNotNull(this.config.getStreamHash());
- Preconditions.checkNotNull(this.config.getStreamHash().get(0));
- Preconditions.checkNotNull(this.config.getApiKey());
- Preconditions.checkNotNull(this.config.getUserName());
- Preconditions.checkNotNull(this.clients);
-
- for (String hash : this.config.getStreamHash()) {
- startStreamForHash(hash);
- }
-
- }
-
- /**
- * Creates a connection to datasift and starts collection of data from the resulting string.
- *
- * @param streamHash
- */
- public void startStreamForHash(String streamHash) {
- shutDownStream(streamHash);
- DataSiftClient client = getNewClient(this.config.getUserName(), this.config.getApiKey());
- client.liveStream().onStreamEvent(this.eventListener);
- client.liveStream().onError(new ErrorHandler(this, streamHash));
-
- client.liveStream().subscribe(new Subscription(Stream.fromString(streamHash), this.interactions));
- synchronized (this.clients) {
- this.clients.put(streamHash, client);
- }
- }
-
- /**
- * Exposed for testing purposes.
- *
- * @param userName
- * @param apiKey
- * @return
- */
- protected DataSiftClient getNewClient(String userName, String apiKey) {
- return new DataSiftClient(new DataSiftConfig(userName, apiKey));
- }
-
-
- /**
- * If a stream has been opened for the supplied stream hash, that stream will be shutdown.
- *
- * @param streamHash
- */
- public void shutDownStream(String streamHash) {
- synchronized (clients) {
- if (!this.clients.containsKey(streamHash))
- return;
- DataSiftClient client = this.clients.get(streamHash);
- LOGGER.debug("Shutting down stream for hash: {}", streamHash);
- client.shutdown();
- this.clients.remove(client);
- }
- }
-
- /**
- * Shuts down all open streams from datasift.
- */
- public void stop() {
- synchronized (clients) {
- for (DataSiftClient client : this.clients.values()) {
- client.shutdown();
- }
- }
- }
-
- // PRIME EXAMPLE OF WHY WE NEED NEW INTERFACES FOR PROVIDERS
- @Override
- //This is a hack. It is only like this because of how perpetual streams work at the moment. Read list server to debate/vote for new interfaces.
- public StreamsResultSet readCurrent() {
- Queue<StreamsDatum> datums = Queues.newConcurrentLinkedQueue();
- StreamsDatum datum = null;
- Interaction interaction;
- while (!this.interactions.isEmpty()) {
- interaction = this.interactions.poll();
- try {
- datum = new StreamsDatum(this.mapper.writeValueAsString(interaction.getData()), interaction.getData().get("interaction").get("id").textValue());
- } catch (JsonProcessingException jpe) {
- LOGGER.error("Exception while converting Interaction to String : {}", jpe);
- }
- if (datum != null) {
- while (!datums.offer(datum)) {
- Thread.yield();
- }
- }
-
- }
- return new StreamsResultSet(datums);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return this.clients != null && this.clients.size() > 0;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.interactions = new ConcurrentLinkedQueue<Interaction>();
- this.clients = Maps.newHashMap();
- this.mapper = StreamsDatasiftMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- public DatasiftConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(DatasiftConfiguration config) {
- this.config = config;
- }
-
-
- /**
- * THIS CLASS NEEDS TO BE REPLACED/OVERRIDDEN BY ALL USERS. TWITTERS TERMS OF SERVICE SAYS THAT EVERYONE MUST
- * DELETE TWEETS FROM THEIR DATA STORE IF THEY RECEIVE A DELETE NOTICE.
- */
- public static class DeleteHandler extends StreamEventListener {
-
- public void onDelete(DeletedInteraction di) {
- //go off and delete the interaction if you have it stored. This is a strict requirement!
- LOGGER.info("DELETED:\n " + di);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
deleted file mode 100644
index d5f19d5..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.stream.ErrorListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Listens for exceptions from the datasift streams and resets connections on errors.
- */
-public class ErrorHandler extends ErrorListener {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ErrorHandler.class);
-
- private String streamHash;
- private DatasiftStreamProvider provider;
-
- public ErrorHandler(DatasiftStreamProvider provider, String streamHash) {
- this.provider = provider;
- this.streamHash = streamHash;
- }
-
- @Override
- public void exceptionCaught(Throwable throwable) {
- LOGGER.error("DatasiftClient received Exception : {}", throwable);
- LOGGER.info("Attempting to restart client for stream hash : {}", this.streamHash);
- this.provider.startStreamForHash(this.streamHash);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
deleted file mode 100644
index 058ba1d..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.datasift.provider;
-
-import com.datasift.client.core.Stream;
-import com.datasift.client.stream.DataSiftMessage;
-import com.datasift.client.stream.Interaction;
-import com.datasift.client.stream.StreamSubscription;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Queue;
-
-/**
- * Adds incomming {@link com.datasift.client.stream.Interaction} to the queue for the provider.
- */
-public class Subscription extends StreamSubscription {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Subscription.class);
- private Queue<Interaction> sharedQueue;
-
- public Subscription(Stream stream, Queue<Interaction> sharedQueue) {
- super(stream);
- this.sharedQueue = sharedQueue;
- }
-
- @Override
- public void onDataSiftLogMessage(DataSiftMessage dataSiftMessage) {
- if (dataSiftMessage.isError()) //should we restart the subscription on error?
- LOGGER.error("Datasift Error : {}", dataSiftMessage.getMessage());
- else if (dataSiftMessage.isWarning())
- LOGGER.warn("Datasift Warning : {}", dataSiftMessage.getMessage());
- else
- LOGGER.info("Datasift Info : {}", dataSiftMessage.getMessage());
- }
-
- @Override
- public void onMessage(Interaction interaction) {
- while (!this.sharedQueue.offer(interaction)) {
- Thread.yield();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
deleted file mode 100644
index b587cd6..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-
-import java.util.List;
-
-/**
- *
- */
-public class DatasiftActivitySerializer implements ActivitySerializer<Datasift> {
-
- private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance();
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Datasift serialize(Activity deserialized) throws ActivitySerializerException {
- return null;
- }
-
- @Override
- public Activity deserialize(Datasift serialized) throws ActivitySerializerException {
- ActivitySerializer serializer = DatasiftEventClassifier.bestSerializer(serialized);
- return serializer.deserialize(serialized);
- }
-
- public Activity deserialize(String json) throws ActivitySerializerException {
- try {
- return deserialize(MAPPER.readValue(json, Datasift.class));
- } catch (Exception e) {
- throw new ActivitySerializerException(e);
- }
- }
-
- @Override
- public List<Activity> deserializeAll(List<Datasift> serializedList) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
deleted file mode 100644
index 7d7d547..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.twitter.Twitter;
-
-/**
- * Created by sblackmon on 11/6/14.
- */
-public class DatasiftEventClassifier {
-
- public static Class detectClass(Datasift event) {
-
- if(event.getTwitter() != null) {
- return Twitter.class;
- } else if(event.getInstagram() != null) {
- return Instagram.class;
- } else {
- return Interaction.class;
- }
- }
-
- public static ActivitySerializer bestSerializer(Datasift event) {
-
- if(event.getTwitter() != null) {
- return DatasiftTwitterActivitySerializer.getInstance();
- } else if(event.getInstagram() != null) {
- return DatasiftInstagramActivitySerializer.getInstance();
- } else {
- return DatasiftInteractionActivitySerializer.getInstance();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
deleted file mode 100644
index 1ed805d..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.serializer;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.instagram.From;
-import org.apache.streams.datasift.instagram.Instagram;
-import org.apache.streams.instagram.serializer.util.InstagramActivityUtil;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- *
- */
-public class DatasiftInstagramActivitySerializer extends DatasiftInteractionActivitySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInstagramActivitySerializer.class);
-
- private static DatasiftInstagramActivitySerializer instance = new DatasiftInstagramActivitySerializer();
-
- public static DatasiftInstagramActivitySerializer getInstance() {
- return instance;
- }
-
- @Override
- public Activity convert(Datasift event) {
- Activity activity = super.convert(event);
-
- Instagram instagram = event.getInstagram();
-
- activity.setActor(buildActor(event, instagram));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setProvider(InstagramActivityUtil.getProvider());
- activity.setLinks(getLinks(event.getInstagram()));
-
- activity.setVerb(selectVerb(event));
- addInstagramExtensions(activity, instagram);
- return activity;
- }
-
- /**
- * Gets links from the object
- * @return
- */
- private List<String> getLinks(Instagram instagram) {
- List<String> result = Lists.newLinkedList();
- if( instagram.getMedia() != null ) {
- result.add(instagram.getMedia().getImage());
- result.add(instagram.getMedia().getVideo());
- }
- return result;
- }
-
- public Actor buildActor(Datasift event, Instagram instagram) {
- Actor actor = super.buildActor(event.getInteraction());
- From user = instagram.getFrom();
-
- actor.setDisplayName(firstStringIfNotNull(user.getFullName()));
- actor.setId(formatId(Optional.fromNullable(
- user.getId())
- .or(actor.getId())));
-
- Image profileImage = new Image();
- String profileUrl = null;
- profileUrl = user.getProfilePicture();
- profileImage.setUrl(profileUrl);
- actor.setImage(profileImage);
-
- return actor;
- }
-
- public void addInstagramExtensions(Activity activity, Instagram instagram) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- List<String> hashTags;
- if(instagram.getMedia() != null) {
- hashTags = instagram.getMedia().getTags();
- extensions.put("hashtags", hashTags);
- extensions.put("keywords", activity.getContent());
- } else {
- extensions.put("keywords", activity.getContent());
-
- }
-
- }
-
- private String selectVerb(Datasift event) {
- if( event.getInteraction().getSubtype().equals("like"))
- return "like";
- else
- return "post";
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:instagram", idparts));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
deleted file mode 100644
index 79bf940..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.serializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.links.Links;
-import org.apache.streams.datasift.util.StreamsDatasiftMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Generator;
-import org.apache.streams.pojo.json.Icon;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
-
-/**
- *
- */
-public class DatasiftInteractionActivitySerializer implements ActivitySerializer<Datasift>, Serializable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInteractionActivitySerializer.class);
-
- private static DatasiftInteractionActivitySerializer instance = new DatasiftInteractionActivitySerializer();
-
- public static DatasiftInteractionActivitySerializer getInstance() {
- return instance;
- }
-
- ObjectMapper mapper = StreamsDatasiftMapper.getInstance();
-
- @Override
- public String serializationFormat() {
- return "application/json+datasift.com.v1.1";
- }
-
- @Override
- public Datasift serialize(Activity deserialized) {
- throw new UnsupportedOperationException("Cannot currently serialize to Datasift JSON");
- }
-
- public Activity deserialize(String datasiftJson) {
- try {
- return deserialize(this.mapper.readValue(datasiftJson, Datasift.class));
- } catch (Exception e) {
- LOGGER.error("Exception while trying convert,\n {},\n to a Datasift object.", datasiftJson);
- LOGGER.error("Exception : {}", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Activity deserialize(Datasift serialized) {
-
- try {
-
- Activity activity = convert(serialized);
-
- return activity;
-
- } catch (Exception e) {
- throw new IllegalArgumentException("Unable to deserialize", e);
- }
-
- }
-
- @Override
- public List<Activity> deserializeAll(List<Datasift> datasifts) {
- List<Activity> activities = Lists.newArrayList();
- for( Datasift datasift : datasifts ) {
- activities.add(deserialize(datasift));
- }
- return activities;
- }
-
- public static Generator buildGenerator(Interaction interaction) {
- Generator generator = new Generator();
- generator.setDisplayName(interaction.getSource());
- generator.setId(interaction.getSource());
- return generator;
- }
-
- public static Icon getIcon(Interaction interaction) {
- return null;
- }
-
- public static Provider buildProvider(Interaction interaction) {
- Provider provider = new Provider();
- provider.setId("id:providers:"+interaction.getType());
- provider.setDisplayName(interaction.getType());
- return provider;
- }
-
- public static String getUrls(Interaction interaction) {
- return null;
- }
-
- public static void addDatasiftExtension(Activity activity, Datasift datasift) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("datasift", datasift);
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:datasift", idparts));
- }
-
- public Activity convert(Datasift event) {
-
- Preconditions.checkNotNull(event);
- Preconditions.checkNotNull(event.getInteraction());
-
- Activity activity = new Activity();
- activity.setActor(buildActor(event.getInteraction()));
- activity.setVerb(selectVerb(event));
- activity.setObject(buildActivityObject(event.getInteraction()));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
- activity.setProvider(buildProvider(event.getInteraction()));
- activity.setTitle(event.getInteraction().getTitle());
- activity.setContent(event.getInteraction().getContent());
- activity.setUrl(event.getInteraction().getLink());
- activity.setLinks(getLinks(event));
- addDatasiftExtension(activity, event);
- if( event.getInteraction().getGeo() != null) {
- addLocationExtension(activity, event.getInteraction());
- }
- return activity;
- }
-
- private String selectVerb(Datasift event) {
- return "post";
- }
-
- public Actor buildActor(Interaction interaction) {
- Actor actor = new Actor();
- org.apache.streams.datasift.interaction.Author author = interaction.getAuthor();
- if(author == null) {
- LOGGER.warn("Interaction does not contain author information.");
- return actor;
- }
- String userName = author.getUsername();
- String name = author.getName();
- Long id = author.getId();
- if(userName != null) {
- actor.setDisplayName(userName);
- } else {
- actor.setDisplayName(name);
- }
-
- if(id != null) {
- actor.setId(id.toString());
- } else {
- if(userName != null)
- actor.setId(userName);
- else
- actor.setId(name);
- }
- Image image = new Image();
- image.setUrl(interaction.getAuthor().getAvatar());
- actor.setImage(image);
- if (interaction.getAuthor().getLink()!=null){
- actor.setUrl(interaction.getAuthor().getLink());
- }
- return actor;
- }
-
- public static ActivityObject buildActivityObject(Interaction interaction) {
- ActivityObject actObj = new ActivityObject();
- actObj.setObjectType(interaction.getContenttype());
- actObj.setUrl(interaction.getLink());
- actObj.setId(formatId("post",interaction.getId()));
- actObj.setContent(interaction.getContent());
-
- return actObj;
- }
-
- public static List<String> getLinks(Datasift event) {
- List<String> result = Lists.newArrayList();
- Links links = event.getLinks();
- if(links == null)
- return null;
- for(Object link : links.getNormalizedUrl()) {
- if(link != null) {
- if(link instanceof String) {
- result.add((String) link);
- } else {
- LOGGER.warn("link is not of type String : {}", link.getClass().getName());
- }
- }
- }
- return result;
- }
-
- public static ActivityObject buildTarget(Interaction interaction) {
- return null;
- }
-
- public static void addLocationExtension(Activity activity, Interaction interaction) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Map<String, Object> location = new HashMap<String, Object>();
- Map<String, Double> coordinates = new HashMap<String, Double>();
- coordinates.put("latitude", interaction.getGeo().getLatitude());
- coordinates.put("longitude", interaction.getGeo().getLongitude());
- location.put("coordinates", coordinates);
- extensions.put("location", location);
- }
-
- public static String firstStringIfNotNull(List<Object> list) {
- if( list != null && list.size() > 0) {
- return (String) list.get(0);
- } else return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
deleted file mode 100644
index cbd29dd..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-package org.apache.streams.datasift.serializer;
-
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.streams.datasift.Datasift;
-import org.apache.streams.datasift.interaction.Author;
-import org.apache.streams.datasift.interaction.Interaction;
-import org.apache.streams.datasift.twitter.DatasiftTwitterUser;
-import org.apache.streams.datasift.twitter.Retweet;
-import org.apache.streams.datasift.twitter.Twitter;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Actor;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.twitter.converter.util.TwitterActivityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class DatasiftTwitterActivitySerializer extends DatasiftInteractionActivitySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftTwitterActivitySerializer.class);
-
- private static DatasiftTwitterActivitySerializer instance = new DatasiftTwitterActivitySerializer();
-
- public static DatasiftTwitterActivitySerializer getInstance() {
- return instance;
- }
-
- @Override
- public Activity convert(Datasift event) {
- Activity activity = new Activity();
- Twitter twitter = event.getTwitter();
- boolean retweet = twitter.getRetweet() != null;
-
- activity.setActor(buildActor(event, twitter)); //TODO
- if(retweet) {
- activity.setVerb("share");
- } else {
- activity.setVerb("post");
- }
- activity.setObject(buildActivityObject(event.getInteraction()));
- activity.setId(formatId(activity.getVerb(), event.getInteraction().getId()));
- activity.setTarget(buildTarget(event.getInteraction()));
- activity.setPublished(event.getInteraction().getCreatedAt());
- activity.setGenerator(buildGenerator(event.getInteraction()));
- activity.setIcon(getIcon(event.getInteraction()));
- activity.setProvider(TwitterActivityUtil.getProvider());
- activity.setTitle(event.getInteraction().getTitle());
- activity.setContent(event.getInteraction().getContent());
- activity.setUrl(event.getInteraction().getLink());
- if(retweet)
- activity.setLinks(getLinks(twitter.getRetweet()));
- else
- activity.setLinks(getLinks(twitter));
- addDatasiftExtension(activity, event);
- if( twitter.getGeo() != null) {
- addLocationExtension(activity, twitter);
- }
- addTwitterExtensions(activity, twitter, event.getInteraction());
- return activity;
- }
-
- /**
- * Get the links from this tweet as a list
- * @param twitter
- * @return the links from the tweet
- */
- public List<String> getLinks(Twitter twitter) {
- return getLinks(twitter.getLinks());
- }
-
- /**
- * Get the links from this tweet as a list
- * @param retweet
- * @return the links from the tweet
- */
- public List<String> getLinks(Retweet retweet) {
- return getLinks(retweet.getLinks());
- }
-
- /**
- * Converts the list of objects to a list of strings
- * @param links
- * @return
- */
- private List<String> getLinks(List<Object> links) {
- if(links == null)
- return Lists.newArrayList();
- List<String> result = Lists.newLinkedList();
- for(Object obj : links) {
- if(obj instanceof String) {
- result.add((String) obj);
- } else {
- LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName());
- }
- }
- return result;
- }
-
- public Actor buildActor(Datasift event, Twitter twitter) {
- DatasiftTwitterUser user = twitter.getUser();
- Actor actor = super.buildActor(event.getInteraction());
- if(user == null) {
- user = twitter.getRetweet().getUser();
- }
-
- actor.setDisplayName(user.getName());
- actor.setId(formatId(Optional.fromNullable(
- user.getIdStr())
- .or(Optional.of(user.getId().toString()))
- .orNull()));
- actor.setSummary(user.getDescription());
- try {
- actor.setPublished(user.getCreatedAt());
- } catch (Exception e) {
- LOGGER.warn("Exception trying to parse date : {}", e);
- }
-
- if(user.getUrl() != null) {
- actor.setUrl(user.getUrl());
- }
-
- Map<String, Object> extensions = new HashMap<String,Object>();
- extensions.put("location", user.getLocation());
- extensions.put("posts", user.getStatusesCount());
- extensions.put("followers", user.getFollowersCount());
- extensions.put("screenName", user.getScreenName());
- if(user.getAdditionalProperties() != null) {
- extensions.put("favorites", user.getFavouritesCount());
- }
-
- Image profileImage = new Image();
- String profileUrl = null;
- Author author = event.getInteraction().getAuthor();
- if( author != null )
- profileUrl = author.getAvatar();
- if(profileUrl == null && user.getProfileImageUrlHttps() != null) {
- Object url = user.getProfileImageUrlHttps();
- if(url instanceof String)
- profileUrl = (String) url;
- }
- if(profileUrl == null) {
- profileUrl = user.getProfileImageUrl();
- }
- profileImage.setUrl(profileUrl);
- actor.setImage(profileImage);
-
- actor.setAdditionalProperty("extensions", extensions);
- return actor;
- }
-
- public void addLocationExtension(Activity activity, Twitter twitter) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Map<String, Object> location = Maps.newHashMap();
- double[] coordiantes = new double[] { twitter.getGeo().getLongitude(), twitter.getGeo().getLatitude() };
- Map<String, Object> coords = Maps.newHashMap();
- coords.put("coordinates", coordiantes);
- coords.put("type", "geo_point");
- location.put("coordinates", coords);
- extensions.put("location", location);
- }
-
- public void addTwitterExtensions(Activity activity, Twitter twitter, Interaction interaction) {
- Retweet retweet = twitter.getRetweet();
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- List<String> hashTags = Lists.newLinkedList();
- List<Object> hts = Lists.newLinkedList();
- if(twitter.getHashtags() != null) {
- hts = twitter.getHashtags();
- } else if (retweet != null) {
- hts = retweet.getHashtags();
- }
- if(hts != null) {
- for(Object ht : twitter.getHashtags()) {
- if(ht instanceof String) {
- hashTags.add((String) ht);
- } else {
- LOGGER.warn("Hashtag was not instance of String : {}", ht.getClass().getName());
- }
- }
- }
- extensions.put("hashtags", hashTags);
-
-
- if(retweet != null) {
- Map<String, Object> rebroadcasts = Maps.newHashMap();
- rebroadcasts.put("perspectival", true);
- rebroadcasts.put("count", retweet.getCount());
- extensions.put("rebroadcasts", rebroadcasts);
- }
-
- if(interaction.getAdditionalProperties() != null) {
- ArrayList<Map<String,Object>> userMentions = createUserMentions(interaction);
-
- if(userMentions.size() > 0)
- extensions.put("user_mentions", userMentions);
- }
-
- extensions.put("keywords", interaction.getContent());
- }
-
- /**
- * Returns an ArrayList of all UserMentions in this interaction
- * Note: The ID list and the handle lists do not necessarily correspond 1:1 for this provider
- * If those lists are the same size, then they will be merged into individual UserMention
- * objects. However, if they are not the same size, a new UserMention object will be created
- * for each entry in both lists.
- *
- * @param interaction
- * @return
- */
- private ArrayList<Map<String,Object>> createUserMentions(Interaction interaction) {
- ArrayList<String> mentions = (ArrayList<String>) interaction.getAdditionalProperties().get("mentions");
- ArrayList<Long> mentionIds = (ArrayList<Long>) interaction.getAdditionalProperties().get("mention_ids");
- ArrayList<Map<String,Object>> userMentions = new ArrayList<Map<String,Object>>();
-
- if(mentions != null && !mentions.isEmpty()) {
- for(int x = 0; x < mentions.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("displayName", mentions.get(x));
- actor.put("handle", mentions.get(x));
-
- userMentions.add(actor);
- }
- }
- if(mentionIds != null && !mentionIds.isEmpty()) {
- for(int x = 0; x < mentionIds.size(); x ++) {
- Map<String, Object> actor = new HashMap<String, Object>();
- actor.put("id", "id:twitter:" + mentionIds.get(x));
-
- userMentions.add(actor);
- }
- }
-
- return userMentions;
- }
-
- public static String formatId(String... idparts) {
- return Joiner.on(":").join(Lists.asList("id:twitter", idparts));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
deleted file mode 100644
index 93ab28b..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.datasift.util;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.io.IOException;
-
-/**
- * Created by sblackmon on 3/27/14.
- *
- * Depracated: Use StreamsJacksonMapper instead
- */
-@Deprecated()
-public class StreamsDatasiftMapper extends StreamsJacksonMapper {
-
- public static final String DATASIFT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z";
-
- public static final DateTimeFormatter DATASIFT_FORMATTER = DateTimeFormat.forPattern(DATASIFT_FORMAT);
-
- public static final Long getMillis(String dateTime) {
-
- // this function is for pig which doesn't handle exceptions well
- try {
- Long result = DATASIFT_FORMATTER.parseMillis(dateTime);
- return result;
- } catch( Exception e ) {
- return null;
- }
-
- }
-
- private static final StreamsDatasiftMapper INSTANCE = new StreamsDatasiftMapper();
-
- public static StreamsDatasiftMapper getInstance(){
- return INSTANCE;
- }
-
- public StreamsDatasiftMapper() {
- super();
- registerModule(new SimpleModule()
- {
- {
- addDeserializer(DateTime.class, new StdDeserializer<DateTime>(DateTime.class) {
- @Override
- public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException, JsonProcessingException {
- DateTime result = null;
- try {
- result = DATASIFT_FORMATTER.parseDateTime(jpar.getValueAsString());
- } catch (Exception e) {}
- if (result == null) {
- try {
- result = RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString());
- } catch (Exception e) {}
- }
- return result;
- }
- });
- }
- });
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
deleted file mode 100644
index 52d9912..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json
+++ /dev/null
@@ -1,462 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.Datasift",
- "properties": {
- "blog": {
- "javaType": "org.apache.streams.datasift.blog.Blog",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "type": "object",
- "javaType": "org.apache.streams.datasift.blog.BlogAuthor",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "domain": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- },
- "demographic": {
- "dynamic": "true",
- "properties": {
- "gender": {
- "type": "string"
- }
- }
- },
- "facebook": {
- "type": "object",
- "$ref": "facebook/DatasiftFacebook.json"
- },
- "instagram": {
- "type": "object",
- "$ref": "instagram/DatasiftInstagram.json"
- },
- "interaction": {
- "type": "object",
- "$ref": "interaction/DatasiftInteraction.json"
- },
- "klout": {
- "javaType": "org.apache.streams.datasift.interaction.Klout",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "score": {
- "type": "integer"
- }
- }
- },
- "language": {
- "javaType": "org.apache.streams.datasift.language.Language",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "confidence": {
- "type": "integer"
- },
- "tag": {
- "type": "string"
- },
- "tag_extended": {
- "type": "string"
- }
- }
- },
- "links": {
- "javaType": "org.apache.streams.datasift.links.Links",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "code": {
- "type": "array",
- "items": [
- {
- "type": "integer"
- }
- ]
- },
- "created_at": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "hops": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "meta": {
- "dynamic": "true",
- "javaType": "org.apache.streams.datasift.links.Meta",
- "type": "object",
- "properties": {
- "charset": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "content_type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "description": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "keywords": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "lang": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "opengraph": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "site_name": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- },
- "twitter": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.interaction.meta.Twitter",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "card": {
- "type": "string"
- },
- "creator": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "image": {
- "type": "string"
- },
- "player": {
- "type": "string"
- },
- "player_height": {
- "type": "string"
- },
- "player_width": {
- "type": "string"
- },
- "site": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- ]
- }
- }
- },
- "normalized_url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "retweet_count": {
- "type": "array",
- "items": [
- {
- "type": "long"
- }
- ]
- },
- "title": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "url": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "salience": {
- "javaType": "org.apache.streams.datasift.salience.Salience",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "javaType": "org.apache.streams.datasift.salience.content.Content",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.content.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- },
- "title": {
- "javaType": "org.apache.streams.datasift.salience.title.Title",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "entities": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.salience.title.Entities",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "about": {
- "type": "integer"
- },
- "confident": {
- "type": "integer"
- },
- "evidence": {
- "type": "integer"
- },
- "label": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "sentiment": {
- "type": "integer"
- },
- "themes": {
- "type": "string"
- },
- "type": {
- "type": "string"
- }
- }
- }
- ]
- },
- "sentiment": {
- "type": "integer"
- }
- }
- }
- }
- },
- "trends": {
- "javaType": "org.apache.streams.datasift.trends.Trends",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "content": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "source": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- },
- "type": {
- "type": "array",
- "items": [
- {
- "type": "string"
- }
- ]
- }
- }
- },
- "twitter": {
- "type": "object",
- "$ref": "twitter/DatasiftTwitter.json"
- },
- "youtube": {
- "javaType": "org.apache.streams.datasift.youtube.YouTube",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "author": {
- "javaType": "org.apache.streams.datasift.youtube.Author",
- "type": "object",
- "dynamic": "true",
- "properties": {
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "commentslink": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "contenttype": {
- "type": "string"
- },
- "created_at": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "videolink": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
deleted file mode 100644
index 99f889e..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json
+++ /dev/null
@@ -1,136 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.datasift.DatasiftConfiguration",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "userName": {
- "type": "string"
- },
- "streamHash": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "streamCsdl": {
- "type": "array",
- "minItems": 0,
- "items": {
- "type": "string"
- }
- },
- "managedSources": {
- "type": "array",
- "minItems": 0,
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource",
- "type": "object",
- "properties": {
- "auth": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth",
- "type": "object",
- "properties": {
- "identity_id": {
- "type": "string"
- },
- "parameters": {
- "type": "object",
- "properties": {
- "value": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "created_at": {
- "type": "number"
- },
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parameters": {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams",
- "type": "object",
- "properties": {
- "comments": {
- "type": "boolean"
- },
- "likes": {
- "type": "boolean"
- }
- }
- },
- "resources": {
- "type": "array",
- "items": [
- {
- "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource",
- "type": "object",
- "properties": {
- "parameters": {
- "type": "object",
- "properties": {
- "distance": {
- "type": "number"
- },
- "exact_match": {
- "type": "string"
- },
- "foursq": {
- "type": "string"
- },
- "lat": {
- "type": "number"
- },
- "lng": {
- "type": "number"
- },
- "type": {
- "type": "string",
- "enum" : ["user", "tag", "area", "location", "popular"]
- },
- "usermeta": {
- "type": "string"
- },
- "value": {
- "type": "number"
- }
- }
- },
- "resource_id": {
- "type": "string"
- }
- }
- }
- ]
- },
- "source_type": {
- "type": "string",
- "enum" : ["facebook_page", "googleplus", "instagram", "yammer"]
- },
- "status": {
- "type": "string"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
deleted file mode 100644
index 4c9fe5c..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.DatasiftPushConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
deleted file mode 100644
index 5c43fea..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.DatasiftStreamConfiguration",
- "extends": {"$ref":"DatasiftConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streamHash": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
deleted file mode 100644
index 7f3bee6..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "javaType": "org.apache.streams.datasift.DatasiftWebhookData",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "hash": {
- "type": "string"
- },
- "hash_type": {
- "type": "string"
- },
- "count": {
- "type": "long"
- },
- "delivered_at": {
- "type": "string",
- "format": "date-time"
- },
- "interactions": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.datasift.Datasift",
- "javaInterfaces": ["java.io.Serializable"]
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
deleted file mode 100644
index 6085124..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.facebook.Facebook",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "application" :
- {
- "type" : "string"
- },
- "author" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.Author",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "avatar" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "link" :
- {
- "type" : "string"
- },
- "name" :
- {
- "type" : "string"
- }
- }
- },
- "created_at" :
- {
- "type" : "string",
- "format" : "date-time"
- },
- "id" :
- {
- "type" : "string"
- },
- "likes" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.Likes",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "count" :
- {
- "type" : "integer"
- },
- "ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "names" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- }
- }
- },
- "message" :
- {
- "type" : "string"
- },
- "source" :
- {
- "type" : "string"
- },
- "to" :
- {
- "javaType" : "org.apache.streams.datasift.facebook.To",
- "type" : "object",
- "dynamic" : "true",
- "properties" :
- {
- "ids" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "names" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- }
- }
- },
- "type" :
- {
- "type" : "string"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
deleted file mode 100644
index 8c209b1..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.datasift.instagram.Instagram",
- "javaInterfaces": ["java.io.Serializable"],
- "properties" :
- {
- "caption" :
- {
- "type" : "object",
- "properties" :
- {
- "created_time" :
- {
- "type" : "number"
- },
- "id" :
- {
- "type" : "string"
- },
- "text" :
- {
- "type" : "string"
- },
- "from" :
- {
- "type" : "object",
- "properties" :
- {
- "full_name" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "profile_picture" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- }
- }
- }
- }
- },
- "created_at" :
- {
- "type" : "string",
- "format" : "date-time"
- },
- "from" :
- {
- "javaType" : "org.apache.streams.datasift.instagram.From",
- "type" : "object",
- "properties" :
- {
- "full_name" :
- {
- "type" : "array",
- "items" :
- [
- {
- "type" : "string"
- }
- ]
- },
- "id" :
- {
- "type" : "string"
- },
- "profile_picture" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- }
- }
- },
- "id" :
- {
- "type" : "string"
- },
- "images" :
- {
- "type" : "object",
- "properties" :
- {
- "low_resolution" :
- {
- "type" : "object"
- },
- "standard_resolution" :
- {
- "type" : "object"
- },
- "thumbnail" :
- {
- "type" : "object"
- }
- }
- },
- "media" :
- {
- "javaType" : "org.apache.streams.datasift.instagram.Media",
- "type" : "object",
- "properties" :
- {
- "caption" :
- {
- "type" : "string"
- },
- "created_time" :
- {
- "type" : "number"
- },
- "filter" :
- {
- "type" : "string"
- },
- "id" :
- {
- "type" : "string"
- },
- "image" :
- {
- "type" : "string"
- },
- "link" :
- {
- "type" : "string"
- },
- "tags" :
- {
- "type" : "array",
- "items" :
- {
- "type" : "string"
- }
- },
- "type" :
- {
- "type" : "string"
- },
- "username" :
- {
- "type" : "string"
- },
- "video" :
- {
- "type" : "string"
- }
- }
- },
- "tags" :
- {
- "type" : "array",
- "items" :
- {
- "type" : "string"
- }
- },
- "type" :
- {
- "type" : "string"
- }
- }
-}
\ No newline at end of file