You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by eh...@apache.org on 2011/10/27 05:53:21 UTC

svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/test/org/apache/solr/request/TestRemoteStreaming.java solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java

Author: ehatcher
Date: Thu Oct 27 03:53:20 2011
New Revision: 1189596

URL: http://svn.apache.org/viewvc?rev=1189596&view=rev
Log:
SOLR-2854: Load URL content streams when requested rather than automatically

Added:
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1189596&r1=1189595&r2=1189596&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Oct 27 03:53:20 2011
@@ -262,6 +262,12 @@ Bug Fixes
 * SOLR-2654: Directorys used by a SolrCore are now closed when they are no longer used.
   (Mark Miller)
   
+* SOLR-2854: Now load URL content stream data (via stream.url) when called for during request handling,
+  rather than loading URL content streams automatically regardless of use.
+  (David Smiley and Ryan McKinley via ehatcher)
+  
+  
+  
 Other Changes
 ----------------------
 
@@ -340,7 +346,7 @@ Other Changes
   from org.codehaus.woodstox:wstx-asl dependency. (David Smiley via Steve Rowe)
 
 * SOLR-2588: Moved VelocityResponseWriter back to contrib module in order to 
-  remove it as a mandatory core dependency.  (Erik Hatcher)
+  remove it as a mandatory core dependency.  (ehatcher)
 
 Documentation
 ----------------------

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java?rev=1189596&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java Thu Oct 27 03:53:20 2011
@@ -0,0 +1,116 @@
+package org.apache.solr.request;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+
+/**
+ * See SOLR-2854.
+ */
+public class TestRemoteStreaming extends SolrJettyTestBase {
+
+  @BeforeClass
+  public static void beforeTest() throws Exception {
+    createJetty(ExternalPaths.EXAMPLE_HOME, null, null);
+  }
+
+  @Before
+  public void doBefore() throws IOException, SolrServerException {
+    //add document and commit, and ensure it's there
+    SolrServer server1 = getSolrServer();
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField( "id", "xxxx" );
+    server1.add(doc);
+    server1.commit();
+    assertTrue(searchFindsIt());
+  }
+
+  @Test
+  public void testMakeDeleteAllUrl() throws Exception {
+    getUrlForString(makeDeleteAllUrl());
+    assertFalse(searchFindsIt());
+  }
+
+  @Test
+  public void testStreamUrl() throws Exception {
+    CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer) getSolrServer();
+    String streamUrl = solrServer.getBaseURL()+"/select?q=*:*&fl=id&wt=csv";
+
+    String getUrl = solrServer.getBaseURL()+"/debug/dump?wt=xml&stream.url="+URLEncoder.encode(streamUrl,"UTF-8");
+    String content = getUrlForString(getUrl);
+    assertTrue(content.contains("xxxx"));
+    //System.out.println(content);
+  }
+
+  private String getUrlForString(String getUrl) throws IOException {
+    Object obj = new URL(getUrl).getContent();
+    if (obj instanceof InputStream) {
+      InputStream inputStream = (InputStream) obj;
+      try {
+        StringWriter strWriter = new StringWriter();
+        IOUtils.copy(inputStream,strWriter);
+        return strWriter.toString();
+      } finally {
+        IOUtils.closeQuietly(inputStream);
+      }
+    }
+    return null;
+  }
+
+  /** Do a select query with the stream.url. Solr should NOT access that URL, and so the data should be there. */
+  @Test
+  public void testNoUrlAccess() throws Exception {
+    SolrQuery query = new SolrQuery();
+    query.setQuery( "*:*" );//for anything
+    query.add("stream.url",makeDeleteAllUrl());
+    getSolrServer().query(query);
+    assertTrue(searchFindsIt());//still there
+  }
+
+  /** Compose a url that if you get it, it will delete all the data. */
+  private String makeDeleteAllUrl() throws UnsupportedEncodingException {
+    CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer) getSolrServer();
+    String deleteQuery = "<delete><query>*:*</query></delete>";
+    return solrServer.getBaseURL()+"/update?commit=true&stream.body="+ URLEncoder.encode(deleteQuery, "UTF-8");
+  }
+
+  private boolean searchFindsIt() throws SolrServerException {
+    SolrQuery query = new SolrQuery();
+    query.setQuery( "id:xxxx" );
+    QueryResponse rsp = getSolrServer().query(query);
+    return rsp.getResults().getNumFound() != 0;
+  }
+}

Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java?rev=1189596&r1=1189595&r2=1189596&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java Thu Oct 27 03:53:20 2011
@@ -72,19 +72,18 @@ public abstract class ContentStreamBase 
   public static class URLStream extends ContentStreamBase
   {
     private final URL url;
-    final URLConnection conn;
     
     public URLStream( URL url ) throws IOException {
       this.url = url; 
-      this.conn = this.url.openConnection();
+    }
+
+    public InputStream getStream() throws IOException {
+      URLConnection conn = this.url.openConnection();
       
       contentType = conn.getContentType();
       name = url.toExternalForm();
       size = new Long( conn.getContentLength() );
       sourceInfo = "url";
-    }
-
-    public InputStream getStream() throws IOException {
       return conn.getInputStream();
     }
   }



RE: svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/test/org/apache/solr/request/TestRemoteStreaming.java solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java

Posted by Uwe Schindler <uw...@thetaphi.de>.
This seem to cause test failures in trunk!

-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de


> -----Original Message-----
> From: ehatcher@apache.org [mailto:ehatcher@apache.org]
> Sent: Thursday, October 27, 2011 5:53 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1189596 - in /lucene/dev/trunk/solr: CHANGES.txt
> core/src/test/org/apache/solr/request/TestRemoteStreaming.java
> solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
> 
> Author: ehatcher
> Date: Thu Oct 27 03:53:20 2011
> New Revision: 1189596
> 
> URL: http://svn.apache.org/viewvc?rev=1189596&view=rev
> Log:
> SOLR-2854: Load URL content streams when requested rather than
> automatically
> 
> Added:
> 
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java
> Modified:
>     lucene/dev/trunk/solr/CHANGES.txt
> 
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java
> 
> Modified: lucene/dev/trunk/solr/CHANGES.txt
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=118959
> 6&r1=1189595&r2=1189596&view=diff
> ================================================================
> ==============
> --- lucene/dev/trunk/solr/CHANGES.txt (original)
> +++ lucene/dev/trunk/solr/CHANGES.txt Thu Oct 27 03:53:20 2011
> @@ -262,6 +262,12 @@ Bug Fixes
>  * SOLR-2654: Directorys used by a SolrCore are now closed when they are no
> longer used.
>    (Mark Miller)
> 
> +* SOLR-2854: Now load URL content stream data (via stream.url) when
> +called for during request handling,
> +  rather than loading URL content streams automatically regardless of use.
> +  (David Smiley and Ryan McKinley via ehatcher)
> +
> +
> +
>  Other Changes
>  ----------------------
> 
> @@ -340,7 +346,7 @@ Other Changes
>    from org.codehaus.woodstox:wstx-asl dependency. (David Smiley via Steve
> Rowe)
> 
>  * SOLR-2588: Moved VelocityResponseWriter back to contrib module in order
> to
> -  remove it as a mandatory core dependency.  (Erik Hatcher)
> +  remove it as a mandatory core dependency.  (ehatcher)
> 
>  Documentation
>  ----------------------
> 
> Added:
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/
> solr/request/TestRemoteStreaming.java?rev=1189596&view=auto
> ================================================================
> ==============
> ---
> lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemoteStrea
> ming.java (added)
> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/request/TestRemo
> +++ teStreaming.java Thu Oct 27 03:53:20 2011
> @@ -0,0 +1,116 @@
> +package org.apache.solr.request;
> +
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed
> +with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version
> +2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +import org.apache.commons.io.IOUtils;
> +import org.apache.solr.SolrJettyTestBase;
> +import org.apache.solr.client.solrj.SolrQuery;
> +import org.apache.solr.client.solrj.SolrServer;
> +import org.apache.solr.client.solrj.SolrServerException;
> +import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
> +import org.apache.solr.client.solrj.response.QueryResponse;
> +import org.apache.solr.common.SolrInputDocument;
> +import org.apache.solr.util.ExternalPaths;
> +import org.junit.Before;
> +import org.junit.BeforeClass;
> +import org.junit.Test;
> +
> +import java.io.IOException;
> +import java.io.InputStream;
> +import java.io.StringWriter;
> +import java.io.UnsupportedEncodingException;
> +import java.net.URL;
> +import java.net.URLEncoder;
> +
> +/**
> + * See SOLR-2854.
> + */
> +public class TestRemoteStreaming extends SolrJettyTestBase {
> +
> +  @BeforeClass
> +  public static void beforeTest() throws Exception {
> +    createJetty(ExternalPaths.EXAMPLE_HOME, null, null);  }
> +
> +  @Before
> +  public void doBefore() throws IOException, SolrServerException {
> +    //add document and commit, and ensure it's there
> +    SolrServer server1 = getSolrServer();
> +    SolrInputDocument doc = new SolrInputDocument();
> +    doc.addField( "id", "xxxx" );
> +    server1.add(doc);
> +    server1.commit();
> +    assertTrue(searchFindsIt());
> +  }
> +
> +  @Test
> +  public void testMakeDeleteAllUrl() throws Exception {
> +    getUrlForString(makeDeleteAllUrl());
> +    assertFalse(searchFindsIt());
> +  }
> +
> +  @Test
> +  public void testStreamUrl() throws Exception {
> +    CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer)
> getSolrServer();
> +    String streamUrl =
> + solrServer.getBaseURL()+"/select?q=*:*&fl=id&wt=csv";
> +
> +    String getUrl =
> solrServer.getBaseURL()+"/debug/dump?wt=xml&stream.url="+URLEncoder.en
> code(streamUrl,"UTF-8");
> +    String content = getUrlForString(getUrl);
> +    assertTrue(content.contains("xxxx"));
> +    //System.out.println(content);
> +  }
> +
> +  private String getUrlForString(String getUrl) throws IOException {
> +    Object obj = new URL(getUrl).getContent();
> +    if (obj instanceof InputStream) {
> +      InputStream inputStream = (InputStream) obj;
> +      try {
> +        StringWriter strWriter = new StringWriter();
> +        IOUtils.copy(inputStream,strWriter);
> +        return strWriter.toString();
> +      } finally {
> +        IOUtils.closeQuietly(inputStream);
> +      }
> +    }
> +    return null;
> +  }
> +
> +  /** Do a select query with the stream.url. Solr should NOT access
> + that URL, and so the data should be there. */  @Test  public void
> + testNoUrlAccess() throws Exception {
> +    SolrQuery query = new SolrQuery();
> +    query.setQuery( "*:*" );//for anything
> +    query.add("stream.url",makeDeleteAllUrl());
> +    getSolrServer().query(query);
> +    assertTrue(searchFindsIt());//still there  }
> +
> +  /** Compose a url that if you get it, it will delete all the data. */
> + private String makeDeleteAllUrl() throws UnsupportedEncodingException {
> +    CommonsHttpSolrServer solrServer = (CommonsHttpSolrServer)
> getSolrServer();
> +    String deleteQuery = "<delete><query>*:*</query></delete>";
> +    return solrServer.getBaseURL()+"/update?commit=true&stream.body="+
> + URLEncoder.encode(deleteQuery, "UTF-8");  }
> +
> +  private boolean searchFindsIt() throws SolrServerException {
> +    SolrQuery query = new SolrQuery();
> +    query.setQuery( "id:xxxx" );
> +    QueryResponse rsp = getSolrServer().query(query);
> +    return rsp.getResults().getNumFound() != 0;
> +  }
> +}
> 
> Modified:
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/
> solr/common/util/ContentStreamBase.java?rev=1189596&r1=1189595&r2=118
> 9596&view=diff
> ================================================================
> ==============
> ---
> lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStre
> amBase.java (original)
> +++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/Con
> +++ tentStreamBase.java Thu Oct 27 03:53:20 2011
> @@ -72,19 +72,18 @@ public abstract class ContentStreamBase
>    public static class URLStream extends ContentStreamBase
>    {
>      private final URL url;
> -    final URLConnection conn;
> 
>      public URLStream( URL url ) throws IOException {
>        this.url = url;
> -      this.conn = this.url.openConnection();
> +    }
> +
> +    public InputStream getStream() throws IOException {
> +      URLConnection conn = this.url.openConnection();
> 
>        contentType = conn.getContentType();
>        name = url.toExternalForm();
>        size = new Long( conn.getContentLength() );
>        sourceInfo = "url";
> -    }
> -
> -    public InputStream getStream() throws IOException {
>        return conn.getInputStream();
>      }
>    }



---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org