You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/11/11 20:54:10 UTC
[03/32] incubator-streams git commit: added HttpConfigurator
added HttpConfigurator
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b8ccf9f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b8ccf9f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b8ccf9f6
Branch: refs/heads/STREAMS-212
Commit: b8ccf9f6869c64f1f6e25f668942ea0de9fec2eb
Parents: 34232ad
Author: sblackmon <sb...@apache.org>
Authored: Mon Sep 15 18:38:09 2014 -0500
Committer: sblackmon <sb...@apache.org>
Committed: Mon Sep 15 18:38:09 2014 -0500
----------------------------------------------------------------------
.../components/http/HttpConfigurator.java | 53 ++++++++++++++++++++
.../components/http/SimpleHTTPGetProcessor.java | 23 +++++----
.../processor/TwitterUrlApiProcessor.java | 8 +++
3 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
new file mode 100644
index 0000000..36801b8
--- /dev/null
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/HttpConfigurator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.components.http;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigRenderOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
+ */
+public class HttpConfigurator {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(HttpConfigurator.class);
+
+ private final static ObjectMapper mapper = new ObjectMapper();
+
+ public static HttpProcessorConfiguration detectConfiguration(Config config) {
+
+ HttpProcessorConfiguration httpProcessorConfiguration = null;
+
+ try {
+ httpProcessorConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), HttpProcessorConfiguration.class);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.warn("Could not parse http configuration", e.getMessage());
+ }
+ return httpProcessorConfiguration;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
index d76d839..dec9d03 100644
--- a/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
+++ b/streams-components/streams-processor-http/src/main/java/org/apache/streams/components/http/SimpleHTTPGetProcessor.java
@@ -19,6 +19,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.data.util.ActivityUtil;
@@ -34,6 +35,7 @@ import javax.validation.ValidatorFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -59,6 +61,11 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
// //private PeoplePatternConfiguration peoplePatternConfiguration = null;
// //private String authHeader;
//
+ public SimpleHTTPGetProcessor() {
+ LOGGER.info("creating SimpleHTTPGetProcessor");
+ this.configuration = HttpConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("http"));
+ }
+
public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
LOGGER.info("creating SimpleHTTPGetProcessor");
LOGGER.info(processorConfiguration.toString());
@@ -137,9 +144,7 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
return result;
}
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- //httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
+ HttpGet httpget = prepareHttpGet(uri);
CloseableHttpResponse response = null;
@@ -184,6 +189,12 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
}
+ public HttpGet prepareHttpGet(URI uri) {
+ HttpGet httpget = new HttpGet(uri);
+ httpget.addHeader("content-type", this.configuration.getContentType());
+ return httpget;
+ }
+
@Override
public void prepare(Object configurationObject) {
@@ -198,12 +209,6 @@ public abstract class SimpleHTTPGetProcessor implements StreamsProcessor {
.setPath(this.configuration.getResourceUri());
httpclient = HttpClients.createDefault();
- // StringBuilder stringBuilder = new StringBuilder();
-// stringBuilder.append(peoplePatternConfiguration.getUsername());
-// stringBuilder.append(":");
-// stringBuilder.append(peoplePatternConfiguration.getPassword());
-// String string = stringBuilder.toString();
-// authHeader = Base64.encodeBase64String(string.getBytes());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b8ccf9f6/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
index 77965c4..438937f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
@@ -5,6 +5,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.streams.components.http.HttpProcessorConfiguration;
import org.apache.streams.components.http.SimpleHTTPGetProcessor;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.core.StreamsProcessor;
import org.apache.streams.pojo.json.Activity;
@@ -17,6 +18,13 @@ import java.util.Map;
*/
public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
+ public TwitterUrlApiProcessor() {
+ super();
+ this.configuration.setHostname("urls.api.twitter.com");
+ this.configuration.setResourceUri("/1/urls/count.json");
+ this.configuration.setExtension("twitter_url_count");
+ }
+
public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
super(processorConfiguration);
this.configuration.setHostname("urls.api.twitter.com");