You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by eh...@apache.org on 2011/10/27 05:53:21 UTC
svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt
core/src/test/org/apache/solr/request/TestRemoteStreaming.java
solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
Author: ehatcher
Date: Thu Oct 27 03:53:20 2011
New Revision: 1189596
URL: http://svn.apache.org/viewvc?rev=1189596&view=rev
Log:
SOLR-2854: Load URL content streams when requested rather than automatically
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1189596&r1=1189595&r2=1189596&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Oct 27 03:53:20 2011
@@ -262,6 +262,12 @@ Bug Fixes
* SOLR-2654: Directorys used by a SolrCore are now closed when they are no longer used.
(Mark Miller)
+* SOLR-2854: Now load URL content stream data (via stream.url) when called for during request handling,
+ rather than loading URL content streams automatically regardless of use.
+ (David Smiley and Ryan McKinley via ehatcher)
+
+
+
Other Changes
----------------------
@@ -340,7 +346,7 @@ Other Changes
from org.codehaus.woodstox:wstx-asl dependency. (David Smiley via Steve Rowe)
* SOLR-2588: Moved VelocityResponseWriter back to contrib module in order to
- remove it as a mandatory core dependency. (Erik Hatcher)
+ remove it as a mandatory core dependency. (ehatcher)
Documentation
----------------------
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java?rev=1189596&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java Thu Oct 27 03:53:20 2011
@@ -0,0 +1,116 @@
+package org.apache.solr.request;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+
+/**
+ * See SOLR-2854.
+ */
+public class TestRemoteStreaming extends SolrJettyTestBase {
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ createJetty(ExternalPaths.EXAMPLE_HOME, null, null);
+ }
+
+ @Before
+ public void doBefore() throws IOException, SolrServerException {
+ //add document and commit, and ensure it's there
+ SolrServer server1 = getSolrServer();
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField( "id", "xxxx" );
+ server1.add(doc);
+ server1.commit();
+ assertTrue(searchFindsIt());
+ }
+
+ @Test
+ public void testMakeDeleteAllUrl() throws Exception {
+ getUrlForString(makeDeleteAllUrl());
+ assertFalse(searchFindsIt());
+ }
+
+ @Test
+ public void testStreamUrl() throws Exception {
+ CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer) getSolrServer();
+ String streamUrl = solrServer.getBaseURL()+"/select?q=*:*&fl=id&wt=csv";
+
+ String getUrl = solrServer.getBaseURL()+"/debug/dump?wt=xml&stream.url="+URLEncoder.encode(streamUrl,"UTF-8");
+ String content = getUrlForString(getUrl);
+ assertTrue(content.contains("xxxx"));
+ //System.out.println(content);
+ }
+
+ private String getUrlForString(String getUrl) throws IOException {
+ Object obj = new URL(getUrl).getContent();
+ if (obj instanceof InputStream) {
+ InputStream inputStream = (InputStream) obj;
+ try {
+ StringWriter strWriter = new StringWriter();
+ IOUtils.copy(inputStream,strWriter);
+ return strWriter.toString();
+ } finally {
+ IOUtils.closeQuietly(inputStream);
+ }
+ }
+ return null;
+ }
+
+ /** Do a select query with the stream.url. Solr should NOT access that URL, and so the data should be there. */
+ @Test
+ public void testNoUrlAccess() throws Exception {
+ SolrQuery query = new SolrQuery();
+ query.setQuery( "*:*" );//for anything
+ query.add("stream.url",makeDeleteAllUrl());
+ getSolrServer().query(query);
+ assertTrue(searchFindsIt());//still there
+ }
+
+ /** Compose a url that if you get it, it will delete all the data. */
+ private String makeDeleteAllUrl() throws UnsupportedEncodingException {
+ CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer) getSolrServer();
+ String deleteQuery = "<delete><query>*:*</query></delete>";
+ return solrServer.getBaseURL()+"/update?commit=true&stream.body="+ URLEncoder.encode(deleteQuery, "UTF-8");
+ }
+
+ private boolean searchFindsIt() throws SolrServerException {
+ SolrQuery query = new SolrQuery();
+ query.setQuery( "id:xxxx" );
+ QueryResponse rsp = getSolrServer().query(query);
+ return rsp.getResults().getNumFound() != 0;
+ }
+}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java?rev=1189596&r1=1189595&r2=1189596&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java Thu Oct 27 03:53:20 2011
@@ -72,19 +72,18 @@ public abstract class ContentStreamBase
public static class URLStream extends ContentStreamBase
{
private final URL url;
- final URLConnection conn;
public URLStream( URL url ) throws IOException {
this.url = url;
- this.conn = this.url.openConnection();
+ }
+
+ public InputStream getStream() throws IOException {
+ URLConnection conn = this.url.openConnection();
contentType = conn.getContentType();
name = url.toExternalForm();
size = new Long( conn.getContentLength() );
sourceInfo = "url";
- }
-
- public InputStream getStream() throws IOException {
return conn.getInputStream();
}
}
RE: svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/test/org/apache/solr/request/TestRemoteStreaming.java solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
Posted by Uwe Schindler <uw...@thetaphi.de>.
This seem to cause test failures in trunk!
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: ehatcher@apache.org [mailto:ehatcher@apache.org]
> Sent: Thursday, October 27, 2011 5:53 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt
> core/src/test/org/apache/solr/request/TestRemoteStreaming.java
> solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
>
> Author: ehatcher
> Date: Thu Oct 27 03:53:20 2011
> New Revision: 1189596
>
> URL: http://svn.apache.org/viewvc?rev=1189596&view=rev
> Log:
> SOLR-2854: Load URL content streams when requested rather than
> automatically
>
> Added:
>
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java
> Modified:
> lucene/dev/trunk/solr/CHANGES.txt
>
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java
>
> Modified: lucene/dev/trunk/solr/CHANGES.txt
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=118959
> 6&r1=1189595&r2=1189596&view=diff
> ================================================================
> ==============
> --- lucene/dev/trunk/solr/CHANGES.txt (original)
> +++ lucene/dev/trunk/solr/CHANGES.txt Thu Oct 27 03:53:20 2011
> @@ -262,6 +262,12 @@ Bug Fixes
> * SOLR-2654: Directorys used by a SolrCore are now closed when they are no
> longer used.
> (Mark Miller)
>
> +* SOLR-2854: Now load URL content stream data (via stream.url) when
> +called for during request handling,
> + rather than loading URL content streams automatically regardless of use.
> + (David Smiley and Ryan McKinley via ehatcher)
> +
> +
> +
> Other Changes
> ----------------------
>
> @@ -340,7 +346,7 @@ Other Changes
> from org.codehaus.woodstox:wstx-asl dependency. (David Smiley via Steve
> Rowe)
>
> * SOLR-2588: Moved VelocityResponseWriter back to contrib module in order
> to
> - remove it as a mandatory core dependency. (Erik Hatcher)
> + remove it as a mandatory core dependency. (ehatcher)
>
> Documentation
> ----------------------
>
> Added:
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/
> solr/request/TestRemoteStreaming.java?rev=1189596&view=auto
> ================================================================
> ==============
> ---
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java (added)
> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemo
> +++ teStreaming.java Thu Oct 27 03:53:20 2011
> @@ -0,0 +1,116 @@
> +package org.apache.solr.request;
> +
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed
> +with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version
> +2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +import org.apache.commons.io.IOUtils;
> +import org.apache.solr.SolrJettyTestBase;
> +import org.apache.solr.client.solrj.SolrQuery;
> +import org.apache.solr.client.solrj.SolrServer;
> +import org.apache.solr.client.solrj.SolrServerException;
> +import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
> +import org.apache.solr.client.solrj.response.QueryResponse;
> +import org.apache.solr.common.SolrInputDocument;
> +import org.apache.solr.util.ExternalPaths;
> +import org.junit.Before;
> +import org.junit.BeforeClass;
> +import org.junit.Test;
> +
> +import java.io.IOException;
> +import java.io.InputStream;
> +import java.io.StringWriter;
> +import java.io.UnsupportedEncodingException;
> +import java.net.URL;
> +import java.net.URLEncoder;
> +
> +/**
> + * See SOLR-2854.
> + */
> +public class TestRemoteStreaming extends SolrJettyTestBase {
> +
> + @BeforeClass
> + public static void beforeTest() throws Exception {
> + createJetty(ExternalPaths.EXAMPLE_HOME, null, null); }
> +
> + @Before
> + public void doBefore() throws IOException, SolrServerException {
> + //add document and commit, and ensure it's there
> + SolrServer server1 = getSolrServer();
> + SolrInputDocument doc = new SolrInputDocument();
> + doc.addField( "id", "xxxx" );
> + server1.add(doc);
> + server1.commit();
> + assertTrue(searchFindsIt());
> + }
> +
> + @Test
> + public void testMakeDeleteAllUrl() throws Exception {
> + getUrlForString(makeDeleteAllUrl());
> + assertFalse(searchFindsIt());
> + }
> +
> + @Test
> + public void testStreamUrl() throws Exception {
> + CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer)
> getSolrServer();
> + String streamUrl =
> + solrServer.getBaseURL()+"/select?q=*:*&fl=id&wt=csv";
> +
> + String getUrl =
> solrServer.getBaseURL()+"/debug/dump?wt=xml&stream.url="+URLEncoder.en
> code(streamUrl,"UTF-8");
> + String content = getUrlForString(getUrl);
> + assertTrue(content.contains("xxxx"));
> + //System.out.println(content);
> + }
> +
> + private String getUrlForString(String getUrl) throws IOException {
> + Object obj = new URL(getUrl).getContent();
> + if (obj instanceof InputStream) {
> + InputStream inputStream = (InputStream) obj;
> + try {
> + StringWriter strWriter = new StringWriter();
> + IOUtils.copy(inputStream,strWriter);
> + return strWriter.toString();
> + } finally {
> + IOUtils.closeQuietly(inputStream);
> + }
> + }
> + return null;
> + }
> +
> + /** Do a select query with the stream.url. Solr should NOT access
> + that URL, and so the data should be there. */ @Test public void
> + testNoUrlAccess() throws Exception {
> + SolrQuery query = new SolrQuery();
> + query.setQuery( "*:*" );//for anything
> + query.add("stream.url",makeDeleteAllUrl());
> + getSolrServer().query(query);
> + assertTrue(searchFindsIt());//still there }
> +
> + /** Compose a url that if you get it, it will delete all the data. */
> + private String makeDeleteAllUrl() throws UnsupportedEncodingException {
> + CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer)
> getSolrServer();
> + String deleteQuery = "<delete><query>*:*</query></delete>";
> + return solrServer.getBaseURL()+"/update?commit=true&stream.body="+
> + URLEncoder.encode(deleteQuery, "UTF-8"); }
> +
> + private boolean searchFindsIt() throws SolrServerException {
> + SolrQuery query = new SolrQuery();
> + query.setQuery( "id:xxxx" );
> + QueryResponse rsp = getSolrServer().query(query);
> + return rsp.getResults().getNumFound() != 0;
> + }
> +}
>
> Modified:
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/
> solr/common/util/ContentStreamBase.java?rev=1189596&r1=1189595&r2=118
> 9596&view=diff
> ================================================================
> ==============
> ---
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java (original)
> +++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/Con
> +++ tentStreamBase.java Thu Oct 27 03:53:20 2011
> @@ -72,19 +72,18 @@ public abstract class ContentStreamBase
> public static class URLStream extends ContentStreamBase
> {
> private final URL url;
> - final URLConnection conn;
>
> public URLStream( URL url ) throws IOException {
> this.url = url;
> - this.conn = this.url.openConnection();
> + }
> +
> + public InputStream getStream() throws IOException {
> + URLConnection conn = this.url.openConnection();
>
> contentType = conn.getContentType();
> name = url.toExternalForm();
> size = new Long( conn.getContentLength() );
> sourceInfo = "url";
> - }
> -
> - public InputStream getStream() throws IOException {
> return conn.getInputStream();
> }
> }
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org