You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2021/12/18 20:33:43 UTC
[maven-wagon] branch WAGON-616 created (now 303f7ed)
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a change to branch WAGON-616
in repository https://gitbox.apache.org/repos/asf/maven-wagon.git.
at 303f7ed [WAGON-618] Remove HTTP file listing with JSoup
This branch includes the following new commits:
new fd69ac1 Bump version to 3.5.0-SNAPSHOT
new 303f7ed [WAGON-618] Remove HTTP file listing with JSoup
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[maven-wagon] 01/02: Bump version to 3.5.0-SNAPSHOT
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch WAGON-616
in repository https://gitbox.apache.org/repos/asf/maven-wagon.git
commit fd69ac13e8e7accab521638f968a6908f9b4f0bc
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Sat Dec 18 19:44:07 2021 +0100
Bump version to 3.5.0-SNAPSHOT
---
pom.xml | 2 +-
wagon-provider-api/pom.xml | 2 +-
wagon-provider-test/pom.xml | 2 +-
wagon-providers/pom.xml | 2 +-
wagon-providers/wagon-file/pom.xml | 2 +-
wagon-providers/wagon-ftp/pom.xml | 2 +-
wagon-providers/wagon-http-lightweight/pom.xml | 2 +-
wagon-providers/wagon-http-shared/pom.xml | 2 +-
wagon-providers/wagon-http/pom.xml | 2 +-
wagon-providers/wagon-scm/pom.xml | 2 +-
wagon-providers/wagon-ssh-common-test/pom.xml | 2 +-
wagon-providers/wagon-ssh-common/pom.xml | 2 +-
wagon-providers/wagon-ssh-external/pom.xml | 2 +-
wagon-providers/wagon-ssh/pom.xml | 2 +-
wagon-providers/wagon-webdav-jackrabbit/pom.xml | 2 +-
wagon-tcks/pom.xml | 2 +-
wagon-tcks/wagon-tck-http/pom.xml | 2 +-
17 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/pom.xml b/pom.xml
index 110f1de..79a4bfc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@ under the License.
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Maven Wagon</name>
diff --git a/wagon-provider-api/pom.xml b/wagon-provider-api/pom.xml
index 2d0da87..c5dbfb9 100644
--- a/wagon-provider-api/pom.xml
+++ b/wagon-provider-api/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-provider-test/pom.xml b/wagon-provider-test/pom.xml
index c41f597..73a3083 100644
--- a/wagon-provider-test/pom.xml
+++ b/wagon-provider-test/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/pom.xml b/wagon-providers/pom.xml
index 3d466ee..9a18b3a 100644
--- a/wagon-providers/pom.xml
+++ b/wagon-providers/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-file/pom.xml b/wagon-providers/wagon-file/pom.xml
index 36b882d..d663ec4 100644
--- a/wagon-providers/wagon-file/pom.xml
+++ b/wagon-providers/wagon-file/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-ftp/pom.xml b/wagon-providers/wagon-ftp/pom.xml
index 685d746..eb5119a 100644
--- a/wagon-providers/wagon-ftp/pom.xml
+++ b/wagon-providers/wagon-ftp/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-http-lightweight/pom.xml b/wagon-providers/wagon-http-lightweight/pom.xml
index ddb8f81..be20d3f 100644
--- a/wagon-providers/wagon-http-lightweight/pom.xml
+++ b/wagon-providers/wagon-http-lightweight/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-http-shared/pom.xml b/wagon-providers/wagon-http-shared/pom.xml
index e6be371..86fd099 100644
--- a/wagon-providers/wagon-http-shared/pom.xml
+++ b/wagon-providers/wagon-http-shared/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-http/pom.xml b/wagon-providers/wagon-http/pom.xml
index d80712a..055df59 100644
--- a/wagon-providers/wagon-http/pom.xml
+++ b/wagon-providers/wagon-http/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-scm/pom.xml b/wagon-providers/wagon-scm/pom.xml
index 936e9ee..ef3e96e 100644
--- a/wagon-providers/wagon-scm/pom.xml
+++ b/wagon-providers/wagon-scm/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-ssh-common-test/pom.xml b/wagon-providers/wagon-ssh-common-test/pom.xml
index b39667d..b9acbaf 100644
--- a/wagon-providers/wagon-ssh-common-test/pom.xml
+++ b/wagon-providers/wagon-ssh-common-test/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-ssh-common/pom.xml b/wagon-providers/wagon-ssh-common/pom.xml
index 88b8642..80fa887 100644
--- a/wagon-providers/wagon-ssh-common/pom.xml
+++ b/wagon-providers/wagon-ssh-common/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-ssh-external/pom.xml b/wagon-providers/wagon-ssh-external/pom.xml
index 8196bd2..a365782 100644
--- a/wagon-providers/wagon-ssh-external/pom.xml
+++ b/wagon-providers/wagon-ssh-external/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-ssh/pom.xml b/wagon-providers/wagon-ssh/pom.xml
index b103fd3..7e42c86 100644
--- a/wagon-providers/wagon-ssh/pom.xml
+++ b/wagon-providers/wagon-ssh/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-providers/wagon-webdav-jackrabbit/pom.xml b/wagon-providers/wagon-webdav-jackrabbit/pom.xml
index 2f7451c..3b9d498 100644
--- a/wagon-providers/wagon-webdav-jackrabbit/pom.xml
+++ b/wagon-providers/wagon-webdav-jackrabbit/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-providers</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-tcks/pom.xml b/wagon-tcks/pom.xml
index 4031cc3..c9f38f9 100644
--- a/wagon-tcks/pom.xml
+++ b/wagon-tcks/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/wagon-tcks/wagon-tck-http/pom.xml b/wagon-tcks/wagon-tck-http/pom.xml
index 92c91f0..cf8afff 100644
--- a/wagon-tcks/wagon-tck-http/pom.xml
+++ b/wagon-tcks/wagon-tck-http/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-tcks</artifactId>
- <version>3.4.4-SNAPSHOT</version>
+ <version>3.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[maven-wagon] 02/02: [WAGON-618] Remove HTTP file listing with JSoup
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch WAGON-616
in repository https://gitbox.apache.org/repos/asf/maven-wagon.git
commit 303f7ed8634a0c757c3ba6161e73a099a4a8d375
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Sat Dec 18 21:33:34 2021 +0100
[WAGON-618] Remove HTTP file listing with JSoup
---
.../java/org/apache/maven/wagon/WagonTestCase.java | 56 +-
.../apache/maven/wagon/http/HttpWagonTestCase.java | 79 ---
.../wagon/providers/http/LightweightHttpWagon.java | 46 --
.../wagon/shared/http/HtmlFileListParser.java | 164 -----
.../wagon/shared/http/HtmlFileListParserTest.java | 728 ---------------------
.../maven/wagon/providers/http/HttpWagon.java | 106 ---
.../wagon/providers/http/HttpWagonTimeoutTest.java | 34 +-
7 files changed, 37 insertions(+), 1176 deletions(-)
diff --git a/wagon-provider-test/src/main/java/org/apache/maven/wagon/WagonTestCase.java b/wagon-provider-test/src/main/java/org/apache/maven/wagon/WagonTestCase.java
index a1b9a45..937f37e 100644
--- a/wagon-provider-test/src/main/java/org/apache/maven/wagon/WagonTestCase.java
+++ b/wagon-provider-test/src/main/java/org/apache/maven/wagon/WagonTestCase.java
@@ -32,6 +32,7 @@ import org.apache.maven.wagon.resource.Resource;
import org.codehaus.plexus.PlexusTestCase;
import org.codehaus.plexus.util.FileUtils;
import org.easymock.IAnswer;
+import org.junit.Assume;
// CHECKSTYLE_OFF: AvoidStarImport
import static org.easymock.EasyMock.*;
@@ -768,30 +769,40 @@ public abstract class WagonTestCase
wagon.connect( testRepository, getAuthInfo() );
- List<String> list = wagon.getFileList( dirName );
- assertNotNull( "file list should not be null.", list );
- assertTrue( "file list should contain more items (actually contains '" + list + "').",
- list.size() >= filenames.length );
-
- for ( String filename : filenames )
+ try
{
- assertTrue( "Filename '" + filename + "' should be in list.", list.contains( filename ) );
- }
+ List<String> list = wagon.getFileList( dirName );
+ assertNotNull( "file list should not be null.", list );
+ assertTrue( "file list should contain more items (actually contains '" + list + "').",
+ list.size() >= filenames.length );
- // WAGON-250
- list = wagon.getFileList( "" );
- assertNotNull( "file list should not be null.", list );
- assertTrue( "file list should contain items (actually contains '" + list + "').", !list.isEmpty() );
- assertTrue( list.contains( "file-list/" ) );
- assertFalse( list.contains( "file-list" ) );
- assertFalse( list.contains( "." ) );
- assertFalse( list.contains( ".." ) );
- assertFalse( list.contains( "./" ) );
- assertFalse( list.contains( "../" ) );
+ for ( String filename : filenames )
+ {
+ assertTrue( "Filename '" + filename + "' should be in list.", list.contains( filename ) );
+ }
- wagon.disconnect();
+ // WAGON-250
+ list = wagon.getFileList( "" );
+ assertNotNull( "file list should not be null.", list );
+ assertTrue( "file list should contain items (actually contains '" + list + "').", !list.isEmpty() );
+ assertTrue( list.contains( "file-list/" ) );
+ assertFalse( list.contains( "file-list" ) );
+ assertFalse( list.contains( "." ) );
+ assertFalse( list.contains( ".." ) );
+ assertFalse( list.contains( "./" ) );
+ assertFalse( list.contains( "../" ) );
+ }
+ catch ( UnsupportedOperationException e )
+ {
+ // Some providers don't support this
+ Assume.assumeFalse( false );
+ }
+ finally
+ {
+ wagon.disconnect();
- tearDownWagonTestingFixtures();
+ tearDownWagonTestingFixtures();
+ }
}
/**
@@ -822,6 +833,11 @@ public abstract class WagonTestCase
{
// expected
}
+ catch ( UnsupportedOperationException e )
+ {
+ // Some providers don't support this
+ Assume.assumeFalse( false );
+ }
finally
{
wagon.disconnect();
diff --git a/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java b/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java
index f7b2341..19765a3 100644
--- a/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java
+++ b/wagon-provider-test/src/main/java/org/apache/maven/wagon/http/HttpWagonTestCase.java
@@ -153,18 +153,6 @@ public abstract class HttpWagonTestCase
server.stop();
}
- public void testWagonGetFileList()
- throws Exception
- {
- File dir = getRepositoryDirectory();
- FileUtils.deleteDirectory( dir );
-
- File f = new File( dir, "file-list" );
- f.mkdirs();
-
- super.testWagonGetFileList();
- }
-
public void testHttpHeaders()
throws Exception
{
@@ -318,73 +306,6 @@ public abstract class HttpWagonTestCase
}
}
- public void testList429()
- throws Exception
- {
- StreamingWagon wagon = (StreamingWagon) getWagon();
- try
- {
-
- Server server = new Server( );
- final AtomicBoolean called = new AtomicBoolean();
-
- AbstractHandler handler = new AbstractHandler()
- {
- public void handle( String target, Request baseRequest, HttpServletRequest request,
- HttpServletResponse response ) throws IOException, ServletException
- {
- if ( called.get() )
- {
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
- }
- else
- {
- called.set( true );
- response.setStatus( SC_TOO_MANY_REQUESTS );
- baseRequest.setHandled( true );
-
- }
- }
- };
-
- server.setHandler( handler );
- addConnector( server );
- server.start();
-
- wagon.connect( new Repository( "id", getRepositoryUrl( server ) ) );
-
- try
- {
- wagon.getFileList( "resource" );
- }
- finally
- {
- wagon.disconnect();
-
- server.stop();
- }
-
- }
- catch ( ResourceDoesNotExistException e )
- {
- assertTrue( true );
- }
- catch ( TransferFailedException e )
- {
- if ( wagon.getClass().getName().contains( "Lightweight" ) )
- {
- //we don't care about lightweight
- assertTrue( true );
- }
- else
- {
- fail();
- }
-
- }
- }
-
public void testGet500()
throws Exception
{
diff --git a/wagon-providers/wagon-http-lightweight/src/main/java/org/apache/maven/wagon/providers/http/LightweightHttpWagon.java b/wagon-providers/wagon-http-lightweight/src/main/java/org/apache/maven/wagon/providers/http/LightweightHttpWagon.java
index f8bb92a..4cd937b 100644
--- a/wagon-providers/wagon-http-lightweight/src/main/java/org/apache/maven/wagon/providers/http/LightweightHttpWagon.java
+++ b/wagon-providers/wagon-http-lightweight/src/main/java/org/apache/maven/wagon/providers/http/LightweightHttpWagon.java
@@ -19,7 +19,6 @@ package org.apache.maven.wagon.providers.http;
* under the License.
*/
-import org.apache.commons.io.IOUtils;
import org.apache.maven.wagon.ConnectionException;
import org.apache.maven.wagon.InputData;
import org.apache.maven.wagon.OutputData;
@@ -32,7 +31,6 @@ import org.apache.maven.wagon.events.TransferEvent;
import org.apache.maven.wagon.proxy.ProxyInfo;
import org.apache.maven.wagon.resource.Resource;
import org.apache.maven.wagon.shared.http.EncodingUtil;
-import org.apache.maven.wagon.shared.http.HtmlFileListParser;
import org.codehaus.plexus.util.Base64;
import java.io.FileNotFoundException;
@@ -384,50 +382,6 @@ public class LightweightHttpWagon
authenticator.resetWagon();
}
- public List<String> getFileList( String destinationDirectory )
- throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
- {
- InputData inputData = new InputData();
-
- if ( destinationDirectory.length() > 0 && !destinationDirectory.endsWith( "/" ) )
- {
- destinationDirectory += "/";
- }
-
- String url = buildUrl( new Resource( destinationDirectory ) );
-
- Resource resource = new Resource( destinationDirectory );
-
- inputData.setResource( resource );
-
- fillInputData( inputData );
-
- InputStream is = inputData.getInputStream();
-
- try
- {
-
- if ( is == null )
- {
- throw new TransferFailedException(
- url + " - Could not open input stream for resource: '" + resource + "'" );
- }
-
- final List<String> htmlFileList = HtmlFileListParser.parseFileList( url, is );
- is.close();
- is = null;
- return htmlFileList;
- }
- catch ( final IOException e )
- {
- throw new TransferFailedException( "Failure transferring " + resource.getName(), e );
- }
- finally
- {
- IOUtils.closeQuietly( is );
- }
- }
-
public boolean resourceExists( String resourceName )
throws TransferFailedException, AuthorizationException
{
diff --git a/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/HtmlFileListParser.java b/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/HtmlFileListParser.java
deleted file mode 100644
index e27696a..0000000
--- a/wagon-providers/wagon-http-shared/src/main/java/org/apache/maven/wagon/shared/http/HtmlFileListParser.java
+++ /dev/null
@@ -1,164 +0,0 @@
-package org.apache.maven.wagon.shared.http;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.apache.commons.io.IOUtils;
-import org.apache.maven.wagon.TransferFailedException;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Html File List Parser.
- */
-public class HtmlFileListParser
-{
- // Apache Fancy Index Sort Headers
- private static final Pattern APACHE_INDEX_SKIP = Pattern.compile( "\\?[CDMNS]=.*" );
-
- // URLs with excessive paths.
- private static final Pattern URLS_WITH_PATHS = Pattern.compile( "/[^/]*/" );
-
- // URLs that to a parent directory.
- private static final Pattern URLS_TO_PARENT = Pattern.compile( "\\.\\./" );
-
- // mailto urls
- private static final Pattern MAILTO_URLS = Pattern.compile( "mailto:.*" );
-
- private static final Pattern[] SKIPS =
- new Pattern[]{ APACHE_INDEX_SKIP, URLS_WITH_PATHS, URLS_TO_PARENT, MAILTO_URLS };
-
- /**
- * Fetches a raw HTML from a provided InputStream, parses it, and returns the file list.
- *
- * @param stream the input stream.
- * @return the file list.
- * @throws TransferFailedException if there was a problem fetching the raw html.
- */
- public static List<String> parseFileList( String baseurl, InputStream stream )
- throws TransferFailedException
- {
- try
- {
- URI baseURI = new URI( baseurl );
- // to make debugging easier, start with a string. This is assuming UTF-8, which might not be a safe
- // assumption.
- String content = IOUtils.toString( stream, "utf-8" );
- Document doc = Jsoup.parse( content, baseurl );
- Elements links = doc.select( "a[href]" );
- Set<String> results = new HashSet<String>();
- for ( Element link : links )
- {
- /*
- * The abs:href loses directories, so we deal with absolute paths ourselves below in cleanLink
- */
- String target = link.attr( "href" );
- if ( target != null )
- {
- String clean = cleanLink( baseURI, target );
- if ( isAcceptableLink( clean ) )
- {
- results.add( clean );
- }
- }
-
- }
-
- return new ArrayList<String>( results );
- }
- catch ( URISyntaxException e )
- {
- throw new TransferFailedException( "Unable to parse as base URI: " + baseurl, e );
- }
- catch ( IOException e )
- {
- throw new TransferFailedException( "I/O error reading HTML listing of artifacts: " + e.getMessage(), e );
- }
- }
-
- private static String cleanLink( URI baseURI, String link )
- {
- if ( link == null || link.length() == 0 )
- {
- return "";
- }
-
- String ret = link;
-
- try
- {
- URI linkuri = new URI( ret );
- if ( link.startsWith( "/" ) )
- {
- linkuri = baseURI.resolve( linkuri );
- }
- URI relativeURI = baseURI.relativize( linkuri ).normalize();
- ret = relativeURI.toASCIIString();
- if ( ret.startsWith( baseURI.getPath() ) )
- {
- ret = ret.substring( baseURI.getPath().length() );
- }
-
- ret = URLDecoder.decode( ret, "UTF-8" );
- }
- catch ( URISyntaxException e )
- {
- // ignore
- }
- catch ( UnsupportedEncodingException e )
- {
- // ignore
- }
-
- return ret;
- }
-
- private static boolean isAcceptableLink( String link )
- {
- if ( link == null || link.length() == 0 )
- {
- return false;
- }
-
- for ( Pattern pattern : SKIPS )
- {
- if ( pattern.matcher( link ).find() )
- {
- return false;
- }
- }
-
- return true;
- }
-
-}
\ No newline at end of file
diff --git a/wagon-providers/wagon-http-shared/src/test/java/org/apache/maven/wagon/shared/http/HtmlFileListParserTest.java b/wagon-providers/wagon-http-shared/src/test/java/org/apache/maven/wagon/shared/http/HtmlFileListParserTest.java
deleted file mode 100644
index 74df4d8..0000000
--- a/wagon-providers/wagon-http-shared/src/test/java/org/apache/maven/wagon/shared/http/HtmlFileListParserTest.java
+++ /dev/null
@@ -1,728 +0,0 @@
-package org.apache.maven.wagon.shared.http;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import org.apache.maven.wagon.TransferFailedException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Unit Tests for the HtmlFileListParser
- */
-public class HtmlFileListParserTest
- extends TestCase
-{
- private static Logger logger = LoggerFactory.getLogger( HtmlFileListParserTest.class );
-
- private void assertContainsExpected( List<String> links, String[] expected )
- {
- if ( expected.length != links.size() )
- {
- Collections.sort( links );
- for ( String link : links )
- {
- logger.info(" \"" + link + "\", ");
- }
- assertEquals( "Links to Expected size", expected.length, links.size() );
- }
-
- for ( String exp : expected )
- {
- assertTrue( "Should find [" + exp + "] in link list", links.contains( exp ) );
- }
- }
-
- private void assertNotContainingAvoided( List<String> links, String[] avoided )
- {
- for ( String a : avoided )
- {
- assertFalse( "Should not find [" + a + "] in link list", links.contains( a ) );
- }
- }
-
- private List<String> parseLinks( String url, String filename )
- throws TransferFailedException
- {
- InputStream is = this.getClass().getResourceAsStream( "/filelistings/" + filename );
- List<String> files = HtmlFileListParser.parseFileList( url, is );
-
- assertNotNull( "file list should not be null.", files );
- assertFalse( "file list should not be empty.", files.isEmpty() );
-
- /* Debug */
- if ( false )
- {
- for ( String file : files )
- {
- logger.info( "File: '" + file + "'" );
- }
- }
-
- return files;
- }
-
- /**
- * Example showing jetty directory browsing of commons-lang
- *
- * @throws TransferFailedException
- */
- public void testParseCommonsLang()
- throws TransferFailedException
- {
- List<String> links = parseLinks( "http://localhost/repository/commons-lang/commons-lang/2.3",
- "commons-lang.html" );
-
- String[] expected = new String[] {
- "commons-lang-2.3-javadoc.jar",
- "commons-lang-2.3-javadoc.jar.asc",
- "commons-lang-2.3-javadoc.jar.md5",
- "commons-lang-2.3-javadoc.jar.sha1",
- "commons-lang-2.3-sources.jar",
- "commons-lang-2.3-sources.jar.asc",
- "commons-lang-2.3-sources.jar.md5",
- "commons-lang-2.3-sources.jar.sha1",
- "commons-lang-2.3.jar",
- "commons-lang-2.3.jar.asc",
- "commons-lang-2.3.jar.md5",
- "commons-lang-2.3.jar.sha1",
- "commons-lang-2.3.pom",
- "commons-lang-2.3.pom.md5",
- "commons-lang-2.3.pom.sha1",
- "maven-metadata.xml",
- "maven-metadata.xml.md5",
- "maven-metadata.xml.sha1" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "../" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- public void testParseIbiblio()
- throws Exception
- {
- List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
- "ibiblio-wagon.html" );
-
- String[] expected = new String[] {
- "wagon-1.0-alpha-2.pom",
- "wagon-1.0-alpha-2.pom.asc",
- "wagon-1.0-alpha-2.pom.md5",
- "wagon-1.0-alpha-2.pom.sha1",
- "wagon-1.0-alpha-3-20050413.021234-4.pom",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
- "wagon-1.0-alpha-3-20050419.043745-5.pom",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
- "wagon-1.0-alpha-3-20050419.044035-6.pom",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
- "wagon-1.0-alpha-3-20050421.162738-7.pom",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
- "wagon-1.0-alpha-3-20050422.075233-8.pom",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
- "wagon-1.0-alpha-3-20050429.051847-9.pom",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
- "wagon-file/",
- "wagon-ftp/",
- "wagon-http-lightweight/",
- "wagon-http/",
- "wagon-lightweight-http/",
- "wagon-provider-api/",
- "wagon-provider-test/",
- "wagon-provider/",
- "wagon-providers-1.0-alpha-2.pom",
- "wagon-providers-1.0-alpha-2.pom.asc",
- "wagon-providers-1.0-alpha-2.pom.md5",
- "wagon-providers-1.0-alpha-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
- "wagon-providers/",
- "wagon-scm/",
- "wagon-ssh-external/",
- "wagon-ssh/",
- "wagon-webdav/",
- "wagon/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A", "?D=A", "?M=A", "?N=D", "?S=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Test of an html which is improperly formatted, and contains full host-specific paths to the resources.
- * @throws Exception
- */
- public void testParseJetty()
- throws Exception
- {
- List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
- "jetty-wagon.html" );
-
- String[] expected = new String[] {
- "wagon-1.0-alpha-2.pom",
- "wagon-1.0-alpha-2.pom.asc",
- "wagon-1.0-alpha-2.pom.md5",
- "wagon-1.0-alpha-2.pom.sha1",
- "wagon-1.0-alpha-3-20050413.021234-4.pom",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
- "wagon-1.0-alpha-3-20050419.043745-5.pom",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
- "wagon-1.0-alpha-3-20050419.044035-6.pom",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
- "wagon-1.0-alpha-3-20050421.162738-7.pom",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
- "wagon-1.0-alpha-3-20050422.075233-8.pom",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
- "wagon-1.0-alpha-3-20050429.051847-9.pom",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
- "wagon-file/",
- "wagon-ftp/",
- "wagon-http-lightweight/",
- "wagon-http/",
- "wagon-lightweight-http/",
- "wagon-provider-api/",
- "wagon-provider-test/",
- "wagon-provider/",
- "wagon-providers-1.0-alpha-2.pom",
- "wagon-providers-1.0-alpha-2.pom.asc",
- "wagon-providers-1.0-alpha-2.pom.md5",
- "wagon-providers-1.0-alpha-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
- "wagon-providers/",
- "wagon-scm/",
- "wagon-ssh-external/",
- "wagon-ssh/",
- "wagon-webdav/",
- "wagon/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Test of an html which is improperly formatted, contains full host-specific paths to the resources and uses non-normalized base URI
- * @throws Exception
- */
- public void testParseJettyWithNonNormalizedBaseURI()
- throws Exception
- {
- List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon//",
- "jetty-wagon.html" );
-
- String[] expected = new String[] {
- "wagon-1.0-alpha-2.pom",
- "wagon-1.0-alpha-2.pom.asc",
- "wagon-1.0-alpha-2.pom.md5",
- "wagon-1.0-alpha-2.pom.sha1",
- "wagon-1.0-alpha-3-20050413.021234-4.pom",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
- "wagon-1.0-alpha-3-20050419.043745-5.pom",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
- "wagon-1.0-alpha-3-20050419.044035-6.pom",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
- "wagon-1.0-alpha-3-20050421.162738-7.pom",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
- "wagon-1.0-alpha-3-20050422.075233-8.pom",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
- "wagon-1.0-alpha-3-20050429.051847-9.pom",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
- "wagon-file/",
- "wagon-ftp/",
- "wagon-http-lightweight/",
- "wagon-http/",
- "wagon-lightweight-http/",
- "wagon-provider-api/",
- "wagon-provider-test/",
- "wagon-provider/",
- "wagon-providers-1.0-alpha-2.pom",
- "wagon-providers-1.0-alpha-2.pom.asc",
- "wagon-providers-1.0-alpha-2.pom.md5",
- "wagon-providers-1.0-alpha-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
- "wagon-providers/",
- "wagon-scm/",
- "wagon-ssh-external/",
- "wagon-ssh/",
- "wagon-webdav/",
- "wagon/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Using repository.codehaus.org output as an example.
- * This is an example of an older RHEL installation of apache httpd with old fancy indexing output
- * This example tests how to detect directories properly.
- *
- * @throws TransferFailedException
- */
- public void testParseMevenIde()
- throws TransferFailedException
- {
- List<String> links = parseLinks( "http://repository.codehaus.org/org/codehaus/mevenide/", "mevenide.html" );
-
- String[] expected = new String[] {
- "apisupport/",
- "autoupdate/",
- "continuum-rpc/",
- "continuum/",
- "debugger-bridge/",
- "deployment-bridge/",
- "feature/",
- "grammar/",
- "ide-mojos/",
- "indexer/",
- "j2ee/",
- "junit/",
- "maven-metadata.xml",
- "maven-metadata.xml.md5",
- "maven-metadata.xml.sha1",
- "mevenide2-parent/",
- "nb-mvn-embedder/",
- "nb-project/",
- "nb-repo-browser/",
- "netbeans-debugger-plugin/",
- "netbeans-deploy-plugin/",
- "netbeans-libs/",
- "netbeans-nbmreload-plugin/",
- "netbeans-repository/",
- "netbeans-run-plugin/",
- "netbeans/",
- "persistence/",
- "plugin-bridges/",
- "plugins/",
- "reload-nbm-bridge/",
- "run-jar-bridge/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/org/codehaus/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- public void testParseMirror()
- throws Exception
- {
- List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
- "mirror-wagon.html" );
-
- String[] expected = new String[] {
- "wagon-1.0-alpha-2.pom",
- "wagon-1.0-alpha-2.pom.asc",
- "wagon-1.0-alpha-2.pom.md5",
- "wagon-1.0-alpha-2.pom.sha1",
- "wagon-1.0-alpha-3-20050413.021234-4.pom",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
- "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
- "wagon-1.0-alpha-3-20050419.043745-5.pom",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
- "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
- "wagon-1.0-alpha-3-20050419.044035-6.pom",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
- "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
- "wagon-1.0-alpha-3-20050421.162738-7.pom",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
- "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
- "wagon-1.0-alpha-3-20050422.075233-8.pom",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
- "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
- "wagon-1.0-alpha-3-20050429.051847-9.pom",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
- "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
- "wagon-file/",
- "wagon-ftp/",
- "wagon-http-lightweight/",
- "wagon-http/",
- "wagon-lightweight-http/",
- "wagon-provider-api/",
- "wagon-provider-test/",
- "wagon-provider/",
- "wagon-providers-1.0-alpha-2.pom",
- "wagon-providers-1.0-alpha-2.pom.asc",
- "wagon-providers-1.0-alpha-2.pom.md5",
- "wagon-providers-1.0-alpha-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
- "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
- "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
- "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
- "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
- "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
- "wagon-providers/",
- "wagon-scm/",
- "wagon-ssh-external/",
- "wagon-ssh/",
- "wagon-webdav/",
- "wagon/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] {
- "/org/codehaus/",
- "?C=S;O=A",
- "?C=D;O=A",
- "?C=M;O=A",
- "mailto:mirror.admin@mirror.com" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Example of output from repo1.maven.org
- * This example is of nekohtml specifically.
- *
- * @throws TransferFailedException
- */
- public void testParseNekoHtml()
- throws TransferFailedException
- {
- List<String> links = parseLinks( "http://repo1.maven.org//maven2/nekohtml/nekohtml/1.9.6/", "nekohtml.html" );
-
- String[] expected = new String[] {
- "nekohtml-1.9.6-javadoc.jar",
- "nekohtml-1.9.6-javadoc.jar.md5",
- "nekohtml-1.9.6-javadoc.jar.sha1",
- "nekohtml-1.9.6-sources.jar",
- "nekohtml-1.9.6-sources.jar.md5",
- "nekohtml-1.9.6-sources.jar.sha1",
- "nekohtml-1.9.6.jar",
- "nekohtml-1.9.6.jar.md5",
- "nekohtml-1.9.6.jar.sha1",
- "nekohtml-1.9.6.pom",
- "nekohtml-1.9.6.pom.md5",
- "nekohtml-1.9.6.pom.sha1" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/maven2/nekohtml/nekohtml/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Example of detecting directories on repo1.maven.org
- *
- * @throws TransferFailedException
- */
- public void testParseNetSourceforge()
- throws TransferFailedException
- {
- List<String> links = parseLinks( "http://repo1.maven.org/maven2/net/sf/", "net_sf.html" );
-
- String[] expected = new String[] {
- "a2j/",
- "aislib/",
- "alchim/",
- "antenna/",
- "apt-jelly/",
- "beanlib/",
- "bluecove/",
- "buildbox/",
- "click/",
- "clirr/",
- "datavision/",
- "dozer/",
- "dtddoc/",
- "dynpageplus/",
- "ehcache/",
- "ezmorph/",
- "falcon/",
- "grester/",
- "gwt-widget/",
- "hermesftp/",
- "hibernate/",
- "jcharts/",
- "jdatabaseimport/",
- "jeceira/",
- "jfcunit/",
- "jfig/",
- "jguard/",
- "jipcam/",
- "jlynx/",
- "jour/",
- "jpf/",
- "json-lib/",
- "jsptest/",
- "jsr107cache/",
- "jt400/",
- "jxls/",
- "kxml/",
- "ldaptemplate/",
- "locale4j/",
- "mapasuta/",
- "maven-har/",
- "maven-sar/",
- "opencsv/",
- "oval/",
- "proguard/",
- "qdwizard/",
- "resultsetmapper/",
- "retrotranslator/",
- "saxon/",
- "shadesdb/",
- "smc/",
- "speculoos/",
- "springlayout/",
- "stat-scm/",
- "statsvn/",
- "stax/",
- "struts/",
- "tacos/",
- "testextensions/",
- "webdav-servlet/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/maven2/net/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Another larger example of the directory link detection on repository.codehaus.org
- *
- * @throws TransferFailedException
- */
- public void testParseOrgCodehaus()
- throws TransferFailedException
- {
- List<String> links = parseLinks( "http://repository.codehaus.org/org/codehaus", "org.codehaus.html" );
-
- String[] expected = new String[] {
- "agilifier/",
- "benji/",
- "bruce/",
- "btm/",
- "cargo/",
- "castor-spring/",
- "castor/",
- "cozmos/",
- "dataforge/",
- "dimple/",
- "droolsdotnet/",
- "enunciate/",
- "fabric3/",
- "gant/",
- "grails-plugins/",
- "groovy/",
- "gsoc/",
- "guessencoding/",
- "gumtree/",
- "gwt-openlayers/",
- "haus/",
- "izpack/",
- "javasim/",
- "jedi/",
- "jequel/",
- "jet/",
- "jettison/",
- "jfdi/",
- "jikesrvm/",
- "jra/",
- "jremoting/",
- "jtestme/",
- "jtestr/",
- "labs-ng/",
- "logicabyss/",
- "marionette/",
- "mevenide/",
- "modello/",
- "mojo/",
- "mvel/",
- "mvflex/",
- "native-mojo/",
- "openim/",
- "plexus/",
- "polymap/",
- "prometheus/",
- "prophit/",
- "quaere/",
- "redback/",
- "rulessandpit/",
- "rvm/",
- "savana/",
- "scala-ide/",
- "senro/",
- "sonar/",
- "staxmate/",
- "stomp/",
- "svn4j/",
- "swiby/",
- "swizzle/",
- "sxc/",
- "testdox/",
- "wadi/",
- "waffle/",
- "woodstox/",
- "xdoclet/",
- "xfire/",
- "xharness/",
- "xsite/",
- "xstream/",
- "xwire/" };
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-
- /**
- * Test the output found from apache httpd with fancy indexing and dav module.
- * Using people.apache.org output as source material.
- *
- * @throws TransferFailedException
- */
- public void testParsePeopleApacheOrg()
- throws TransferFailedException
- {
- List<String> links = parseLinks(
- "http://people.apache.org/repo/m2-ibiblio-rsync-repository/org/apache/maven/archiva/archiva-plexus-runtime/1.0.1/",
- "org.apache.maven.html" );
-
- String[] expected = new String[] {
- "archiva-plexus-runtime-1.0.1-bin.tar.gz",
- "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc",
- "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc.md5",
- "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc.sha1",
- "archiva-plexus-runtime-1.0.1-bin.tar.gz.md5",
- "archiva-plexus-runtime-1.0.1-bin.tar.gz.sha1",
- "archiva-plexus-runtime-1.0.1-bin.zip",
- "archiva-plexus-runtime-1.0.1-bin.zip.asc",
- "archiva-plexus-runtime-1.0.1-bin.zip.asc.md5",
- "archiva-plexus-runtime-1.0.1-bin.zip.asc.sha1",
- "archiva-plexus-runtime-1.0.1-bin.zip.md5",
- "archiva-plexus-runtime-1.0.1-bin.zip.sha1",
- "archiva-plexus-runtime-1.0.1-sources.jar",
- "archiva-plexus-runtime-1.0.1-sources.jar.asc",
- "archiva-plexus-runtime-1.0.1-sources.jar.asc.md5",
- "archiva-plexus-runtime-1.0.1-sources.jar.asc.sha1",
- "archiva-plexus-runtime-1.0.1-sources.jar.md5",
- "archiva-plexus-runtime-1.0.1-sources.jar.sha1",
- "archiva-plexus-runtime-1.0.1.jar",
- "archiva-plexus-runtime-1.0.1.jar.asc",
- "archiva-plexus-runtime-1.0.1.jar.asc.md5",
- "archiva-plexus-runtime-1.0.1.jar.asc.sha1",
- "archiva-plexus-runtime-1.0.1.jar.md5",
- "archiva-plexus-runtime-1.0.1.jar.sha1",
- "archiva-plexus-runtime-1.0.1.pom",
- "archiva-plexus-runtime-1.0.1.pom.asc",
- "archiva-plexus-runtime-1.0.1.pom.asc.md5",
- "archiva-plexus-runtime-1.0.1.pom.asc.sha1",
- "archiva-plexus-runtime-1.0.1.pom.md5",
- "archiva-plexus-runtime-1.0.1.pom.sha1",
- "readme artifacts.txt"};
-
- assertContainsExpected( links, expected );
-
- String[] avoided = new String[] {
- "/repo/m2-ibiblio-rsync-repository/org/apache/maven/archiva/archiva-plexus-runtime/",
- "?C=S;O=A",
- "?C=D;O=A",
- "?C=M;O=A" };
-
- assertNotContainingAvoided( links, avoided );
- }
-}
diff --git a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java
index 0f3107a..ee65c99 100644
--- a/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java
+++ b/wagon-providers/wagon-http/src/main/java/org/apache/maven/wagon/providers/http/HttpWagon.java
@@ -19,26 +19,7 @@ package org.apache.maven.wagon.providers.http;
* under the License.
*/
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpException;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.util.EntityUtils;
-import org.apache.maven.wagon.ResourceDoesNotExistException;
-import org.apache.maven.wagon.TransferFailedException;
-import org.apache.maven.wagon.authorization.AuthorizationException;
import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon;
-import org.apache.maven.wagon.shared.http.HtmlFileListParser;
-import org.apache.maven.wagon.shared.http.HttpMessageUtils;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatResourceDoesNotExistMessage;
-import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatTransferDebugMessage;
-import static org.apache.maven.wagon.shared.http.HttpMessageUtils.formatTransferFailedMessage;
/**
* @author <a href="michal.maczka@dimatics.com">Michal Maczka</a>
@@ -47,91 +28,4 @@ public class HttpWagon
extends AbstractHttpClientWagon
{
- public List<String> getFileList( String destinationDirectory )
- throws AuthorizationException, ResourceDoesNotExistException, TransferFailedException
- {
- return getFileList( getInitialBackoffSeconds(), destinationDirectory );
- }
-
- private List<String> getFileList( int wait, String destinationDirectory )
- throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
- {
- if ( destinationDirectory.length() > 0 && !destinationDirectory.endsWith( "/" ) )
- {
- destinationDirectory += "/";
- }
-
- String url = getRepository().getUrl() + "/" + destinationDirectory;
-
- HttpGet getMethod = new HttpGet( url );
-
- try
- {
- CloseableHttpResponse response = execute( getMethod );
- try
- {
- String reasonPhrase = response.getStatusLine().getReasonPhrase();
- int statusCode = response.getStatusLine().getStatusCode();
-
- fireTransferDebug( formatTransferDebugMessage( url, statusCode, reasonPhrase, getProxyInfo() ) );
-
- switch ( statusCode )
- {
- case HttpStatus.SC_OK:
- break;
-
- // TODO Move 401/407 to AuthenticationException after WAGON-587
- case HttpStatus.SC_FORBIDDEN:
- case HttpStatus.SC_UNAUTHORIZED:
- case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
- EntityUtils.consumeQuietly( response.getEntity() );
- throw new AuthorizationException( HttpMessageUtils.formatAuthorizationMessage( url, statusCode,
- reasonPhrase, getProxyInfo() ) );
-
- case HttpStatus.SC_NOT_FOUND:
- case HttpStatus.SC_GONE:
- EntityUtils.consumeQuietly( response.getEntity() );
- throw new ResourceDoesNotExistException( formatResourceDoesNotExistMessage( url, statusCode,
- reasonPhrase, getProxyInfo() ) );
-
- case SC_TOO_MANY_REQUESTS:
- EntityUtils.consumeQuietly( response.getEntity() );
- return getFileList( backoff( wait, url ), destinationDirectory );
-
- //add more entries here
- default:
- EntityUtils.consumeQuietly( response.getEntity() );
- throw new TransferFailedException( formatTransferFailedMessage( url, statusCode, reasonPhrase,
- getProxyInfo() ) );
- }
- HttpEntity entity = response.getEntity();
- if ( entity != null )
- {
- return HtmlFileListParser.parseFileList( url, entity.getContent() );
- }
- else
- {
- return Collections.emptyList();
- }
-
- }
- finally
- {
- response.close();
- }
- }
- catch ( IOException e )
- {
- throw new TransferFailedException( "Could not read response body.", e );
- }
- catch ( HttpException e )
- {
- throw new TransferFailedException( "Could not read response body.", e );
- }
- catch ( InterruptedException e )
- {
- throw new TransferFailedException( "Unable to wait for resource.", e );
- }
- }
-
}
diff --git a/wagon-providers/wagon-http/src/test/java/org/apache/maven/wagon/providers/http/HttpWagonTimeoutTest.java b/wagon-providers/wagon-http/src/test/java/org/apache/maven/wagon/providers/http/HttpWagonTimeoutTest.java
index 553576f..71a8c9a 100644
--- a/wagon-providers/wagon-http/src/test/java/org/apache/maven/wagon/providers/http/HttpWagonTimeoutTest.java
+++ b/wagon-providers/wagon-http/src/test/java/org/apache/maven/wagon/providers/http/HttpWagonTimeoutTest.java
@@ -112,38 +112,6 @@ public class HttpWagonTimeoutTest
assertEquals( TransferFailedException.class, thrown.getClass() );
}
- public void testGetFileList()
- throws Exception
- {
- Exception thrown = null;
-
- try
- {
- Wagon wagon = getWagon();
- wagon.setReadTimeout( 1000 );
-
- Repository testRepository = new Repository();
- testRepository.setUrl( "http://localhost:" + getPort() );
-
- wagon.connect( testRepository );
-
- wagon.getFileList( "/timeoutfile" );
-
- wagon.disconnect();
- }
- catch ( Exception e )
- {
- thrown = e;
- }
- finally
- {
- stopServer();
- }
-
- assertNotNull( thrown );
- assertEquals( TransferFailedException.class, thrown.getClass() );
- }
-
public void testPutTimeout()
throws Exception
{
@@ -197,7 +165,7 @@ public class HttpWagonTimeoutTest
wagon.connect( testRepository );
long start = System.currentTimeMillis();
- wagon.getFileList( "/foobar" );
+ wagon.resourceExists( "/foobar" );
long end = System.currentTimeMillis();
wagon.disconnect();