You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by no...@apache.org on 2009/12/08 07:15:47 UTC
svn commit: r888276 - in
/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport:
BinURLDataSource.java URLDataSource.java
Author: noble
Date: Tue Dec 8 06:15:47 2009
New Revision: 888276
URL: http://svn.apache.org/viewvc?rev=888276&view=rev
Log:
SOLR-1583 added BinURLDataSource
Added:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java (with props)
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java?rev=888276&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java Tue Dec 8 06:15:47 2009
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import static org.apache.solr.handler.dataimport.URLDataSource.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Properties;
+/**
+ * <p> A data source implementation which can be used to read binary streams using HTTP. </p> <p/> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
+ * <p/>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @version $Id$
+ * @since solr 1.5
+ */
+public class BinURLDataSource extends DataSource<InputStream>{
+ Logger LOG = LoggerFactory.getLogger(BinURLDataSource.class);
+
+ private String baseUrl;
+ private int connectionTimeout = URLDataSource.CONNECTION_TIMEOUT;
+
+ private int readTimeout = URLDataSource.READ_TIMEOUT;
+
+ private Context context;
+
+ private Properties initProps;
+
+ public BinURLDataSource() { }
+
+ public void init(Context context, Properties initProps) {
+ this.context = context;
+ this.initProps = initProps;
+
+ baseUrl = getInitPropWithReplacements(BASE_URL);
+ String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
+ String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
+ if (cTimeout != null) {
+ try {
+ connectionTimeout = Integer.parseInt(cTimeout);
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid connection timeout: " + cTimeout);
+ }
+ }
+ if (rTimeout != null) {
+ try {
+ readTimeout = Integer.parseInt(rTimeout);
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid read timeout: " + rTimeout);
+ }
+ }
+ }
+
+ public InputStream getData(String query) {
+ URL url = null;
+ try {
+ if (URIMETHOD.matcher(query).find()) url = new URL(query);
+ else url = new URL(baseUrl + query);
+ LOG.debug("Accessing URL: " + url.toString());
+ URLConnection conn = url.openConnection();
+ conn.setConnectTimeout(connectionTimeout);
+ conn.setReadTimeout(readTimeout);
+ return conn.getInputStream();
+ } catch (Exception e) {
+ LOG.error("Exception thrown while getting data", e);
+ wrapAndThrow (SEVERE, e, "Exception in invoking url " + url);
+ return null;//unreachable
+ }
+ }
+
+ public void close() { }
+
+ private String getInitPropWithReplacements(String propertyName) {
+ final String expr = initProps.getProperty(propertyName);
+ if (expr == null) {
+ return null;
+ }
+ return context.replaceTokens(expr);
+ }
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java?rev=888276&r1=888275&r2=888276&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/URLDataSource.java Tue Dec 8 06:15:47 2009
@@ -129,7 +129,7 @@
return context.replaceTokens(expr);
}
- private static final Pattern URIMETHOD = Pattern.compile("\\w{3,}:/");
+ static final Pattern URIMETHOD = Pattern.compile("\\w{3,}:/");
private static final Pattern CHARSET_PATTERN = Pattern.compile(".*?charset=(.*)$", Pattern.CASE_INSENSITIVE);