You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ant.apache.org by ja...@apache.org on 2017/07/26 12:57:37 UTC
ant-ivy git commit: IVY-1498 Fix potential Ivy cache corruption
during retrieve task when symlink options come into play
Repository: ant-ivy
Updated Branches:
refs/heads/master 87c4e0656 -> 40708bdf0
IVY-1498 Fix potential Ivy cache corruption during retrieve task when symlink options come into play
Project: http://git-wip-us.apache.org/repos/asf/ant-ivy/repo
Commit: http://git-wip-us.apache.org/repos/asf/ant-ivy/commit/40708bdf
Tree: http://git-wip-us.apache.org/repos/asf/ant-ivy/tree/40708bdf
Diff: http://git-wip-us.apache.org/repos/asf/ant-ivy/diff/40708bdf
Branch: refs/heads/master
Commit: 40708bdf04dae6aa49bf8f3617f2efc16c83c5f8
Parents: 87c4e06
Author: Jaikiran Pai <ja...@apache.org>
Authored: Wed Jul 26 18:26:51 2017 +0530
Committer: Jaikiran Pai <ja...@apache.org>
Committed: Wed Jul 26 18:26:51 2017 +0530
----------------------------------------------------------------------
src/java/org/apache/ivy/util/FileUtil.java | 25 ++-
.../apache/ivy/core/retrieve/RetrieveTest.java | 162 +++++++++++++++----
.../1/org/foo-bar/ivys/ivy-2.3.4.xml | 22 +++
.../1/org/foo-bar/jars/foo-bar-1.2.3.jar | Bin 0 -> 470 bytes
.../1/org/foo-bar/jars/foo-bar-2.3.4.jar | Bin 0 -> 475 bytes
test/repositories/1/org/mod1/ivys/ivy-5.0.xml | 25 +++
test/repositories/1/org/mod1/jars/mod1-5.0.jar | 0
7 files changed, 197 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/src/java/org/apache/ivy/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/ivy/util/FileUtil.java b/src/java/org/apache/ivy/util/FileUtil.java
index 5446e9d..0ff62fa 100644
--- a/src/java/org/apache/ivy/util/FileUtil.java
+++ b/src/java/org/apache/ivy/util/FileUtil.java
@@ -17,6 +17,9 @@
*/
package org.apache.ivy.util;
+import org.apache.ivy.core.settings.TimeoutConstraint;
+import org.apache.ivy.util.url.URLHandlerRegistry;
+
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -29,6 +32,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -45,9 +49,6 @@ import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipInputStream;
-import org.apache.ivy.core.settings.TimeoutConstraint;
-import org.apache.ivy.util.url.URLHandlerRegistry;
-
/**
* Utility class used to deal with file related operations, like copy, full reading, symlink, ...
*/
@@ -174,7 +175,7 @@ public final class FileUtil {
return copy(src, dest, l, false);
}
- public static boolean prepareCopy(File src, File dest, boolean overwrite) throws IOException {
+ public static boolean prepareCopy(final File src, final File dest, final boolean overwrite) throws IOException {
if (src.isDirectory()) {
if (dest.exists()) {
if (!dest.isDirectory()) {
@@ -188,11 +189,25 @@ public final class FileUtil {
}
// else it is a file copy
if (dest.exists()) {
+ // If overwrite is specified as "true" and the dest file happens to be a
+ // symlink, we delete the "link" (a.k.a unlink it). This is for cases
+ // like https://issues.apache.org/jira/browse/IVY-1498 where not unlinking
+ // the existing symlink can lead to potentially overwriting the wrong "target" file
+ // TODO: This behaviour is intentionally hardcoded here for now, since I don't
+ // see a reason (yet) to expose it as a param of this method. If any use case arises
+ // we can have this behaviour decided by the callers of this method, by passing a value for this
+ // param
+ final boolean unlinkSymlinkIfOverwrite = true;
if (!dest.isFile()) {
throw new IOException("impossible to copy: destination is not a file: " + dest);
}
if (overwrite) {
- if (!dest.canWrite()) {
+ if (Files.isSymbolicLink(dest.toPath()) && unlinkSymlinkIfOverwrite) {
+ // unlink (a.k.a delete the symlink path)
+ dest.delete();
+ } else if (!dest.canWrite()) {
+ // if the file *isn't* "writable" (see javadoc of File.canWrite() on what that means)
+ // we delete it.
dest.delete();
} // if dest is writable, the copy will overwrite it without requiring a delete
} else {
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
----------------------------------------------------------------------
diff --git a/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java b/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
index 89a11fb..aae45d2 100644
--- a/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
+++ b/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
@@ -17,21 +17,6 @@
*/
package org.apache.ivy.core.retrieve;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ivy.Ivy;
import org.apache.ivy.TestHelper;
import org.apache.ivy.core.IvyPatternHelper;
@@ -55,6 +40,24 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class RetrieveTest {
private Ivy ivy;
@@ -250,23 +253,8 @@ public class RetrieveTest {
"jar", "default"));
}
- private void assertLink(String filename) throws IOException {
- // if the OS is known to support symlink, check that the file is a symlink,
- // otherwise just check the file exist.
-
- File file = new File(filename);
- assertTrue("The file " + filename + " doesn't exist", file.exists());
-
- String os = System.getProperty("os.name");
- if (os.equals("Linux") || os.equals("Solaris") || os.equals("FreeBSD")
- || os.equals("Mac OS X")) {
- // these OS should support symlink, so check that the file is actually a symlink.
- // this is done be checking that the canonical path is different from the absolute
- // path.
- File absFile = file.getAbsoluteFile();
- File canFile = file.getCanonicalFile();
- assertFalse("The file " + filename + " isn't a symlink", absFile.equals(canFile));
- }
+ private void assertLink(final String filename) throws IOException {
+ assertTrue(filename + " was expected to be a symlink", Files.isSymbolicLink(Paths.get(filename)));
}
@Test
@@ -409,6 +397,116 @@ public class RetrieveTest {
assertTrue("Retrieved artifact at " + dest.getAbsolutePath() + " is not a file", dest.isFile());
}
+ /**
+ * Tests that more than one retrieve, with one having symlink enabled and the other not, doesn't lead to repository
+ * cache corruption as reported in https://issues.apache.org/jira/browse/IVY-1498
+ * <p>
+ * The test does the following:
+ * - Resolves a module, let's call it "irrelevant-A" which has a dependency on "org:foo-bar:1.2.3"
+ * <p>
+ * - Invokes a retrieve RT1, with {@link RetrieveOptions#setMakeSymlinks(boolean) symlinks true}
+ * on that resolved module with a pattern that translates to a path "a/b/c/foo-bar.jar". When the RT1 retrieve is
+ * done, the path "a/b/c/foo-bar.jar" will be a symlink to a path "repo/cache/org/foo-bar/foo-bar-1.2.3.jar" in the
+ * Ivy cache.
+ * All fine so far.
+ * <p>
+ * - We then resolve another module, let's call it "irrelevant-B" which has a dependency on "org:foo-bar:2.3.4"
+ * <p>
+ * - Next, do a new retrieve RT2, on this newly resolved module with
+ * {@link RetrieveOptions#setMakeSymlinks(boolean) symlinks false} and {@link RetrieveOptions#setOverwriteMode(String) overwrite true}
+ * and with the same pattern as before), that translates to a path "a/b/c/foo-bar.jar".
+ * <p>
+ * When RT2 retrieve is done, we expect the path "a/b/c/foo-bar.jar" will *not* be a symlink and instead be an actual file that represents
+ * the org:foo-bar:2.3.4 artifact jar.
+ * <p>
+ * We also expect that this RT2 retrieve will not update/overwrite the file at path "repo/cache/org/foo-bar/foo-bar-1.2.3.jar"
+ * in the Ivy cache - the file that was the end target of the symlink generated by the previous RT1.
+ *
+ * @throws Exception
+ * @see <a href="https://issues.apache.org/jira/browse/IVY-1498">IVY-1498</a>
+ */
+ @Test
+ public void testRetrieveWithSymlinkToggling() throws Exception {
+ // org:mod1:2.0 depends on org:foo-bar:1.2.3
+ final ResolveReport resolve1Report = ivy.resolve(new File("test/repositories/1/org/mod1/ivys/ivy-2.0.xml").toURI().toURL(),
+ getResolveOptions(new String[]{"*"}));
+ assertNotNull("Resolve report isn't available", resolve1Report);
+ assertNotNull("Module descriptor missing in resolve report", resolve1Report.getModuleDescriptor());
+
+ // verify the downloaded dependency artifact in cache
+ final ModuleRevisionId fooBar123Mrid = ModuleRevisionId.newInstance("org", "foo-bar", "1.2.3");
+ final ArtifactDownloadReport[] downloadReports = resolve1Report.getArtifactsReports(fooBar123Mrid);
+ assertNotNull("No artifact download report found for the " + fooBar123Mrid + " dependency", downloadReports);
+ assertEquals("Unexpected number for artifact download reports for the " + fooBar123Mrid + " dependency", 1, downloadReports.length);
+ final File fooBar123ArtifactInCache = downloadReports[0].getLocalFile();
+ assertNotNull("Local file for " + fooBar123Mrid + " missing in download report", fooBar123ArtifactInCache);
+ assertTrue("Artifact file for " + fooBar123Mrid + " isn't a file, in cache at " + fooBar123ArtifactInCache,
+ fooBar123ArtifactInCache.isFile());
+ final byte[] fooBar123ArtifactContentsInCache = Files.readAllBytes(fooBar123ArtifactInCache.toPath());
+ assertTrue("Artifact content was empty at " + fooBar123ArtifactContentsInCache, fooBar123ArtifactContentsInCache.length > 0);
+
+
+ // now do a retrieve of the resolved module
+ final String retrievePattern = "build/test/retrieve/symlink-test/[module]/[artifact].[ext]";
+ ivy.retrieve(resolve1Report.getModuleDescriptor().getModuleRevisionId(),
+ getRetrieveOptions().setMakeSymlinks(true).setDestArtifactPattern(retrievePattern));
+ // we expect org:foo-bar:1.2.3 to have been retrieved
+ final Path retrievedArtifactSymlinkPath = Paths.get(IvyPatternHelper.substitute(retrievePattern, "org", "foo-bar",
+ "1.2.3", "foo-bar", "jar", "jar", "default"));
+ assertTrue("Artifact wasn't retrieved to " + retrievedArtifactSymlinkPath, Files.exists(retrievedArtifactSymlinkPath));
+ assertTrue("Artifact retrieved at " + retrievedArtifactSymlinkPath + " was expected to be a " +
+ "symlink", Files.isSymbolicLink(retrievedArtifactSymlinkPath));
+
+ // get hold of the contents of the retrieved artifact
+ final byte[] retrievedArtifactContent = Files.readAllBytes(retrievedArtifactSymlinkPath);
+ // compare it to the contents of org:foo-bar:1.2.3 artifact in repo cache. Should be the same
+ assertTrue("Unexpected content in the retrieved artifact at " + retrievedArtifactSymlinkPath,
+ Arrays.equals(fooBar123ArtifactContentsInCache, retrievedArtifactContent));
+
+ // let this retrieved artifact file stay and let's now try and resolve a module and then retrieve a different version
+ // of the same artifact, *without* symlinking enabled, which will end up at this exact location
+ // org:mod1:2.0 depends on org:foo-bar:1.2.3
+ final ResolveReport resolve2Report = ivy.resolve(new File("test/repositories/1/org/mod1/ivys/ivy-5.0.xml").toURI().toURL(),
+ getResolveOptions(new String[]{"*"}));
+ assertNotNull("Resolve report isn't available", resolve2Report);
+ assertNotNull("Module descriptor missing in resolve report", resolve2Report.getModuleDescriptor());
+
+ // verify the downloaded dependency artifact in cache
+ final ModuleRevisionId fooBar234Mrid = ModuleRevisionId.newInstance("org", "foo-bar", "2.3.4");
+ final ArtifactDownloadReport[] foobar234report = resolve2Report.getArtifactsReports(fooBar234Mrid);
+ assertNotNull("No artifact download report found for the " + fooBar234Mrid + " dependency", foobar234report);
+ assertEquals("Unexpected number for artifact download reports for the " + fooBar234Mrid + " dependency", 1, foobar234report.length);
+ final File foobar234InCache = foobar234report[0].getLocalFile();
+ assertNotNull("Local file for " + fooBar234Mrid + " missing in download report", foobar234InCache);
+ assertTrue("Artifact file for " + fooBar234Mrid + " isn't a file, in cache at " + foobar234InCache,
+ foobar234InCache.isFile());
+ final byte[] foobar234CacheFileContents = Files.readAllBytes(foobar234InCache.toPath());
+ assertTrue("Artifact content was empty at " + foobar234CacheFileContents, foobar234CacheFileContents.length > 0);
+
+ // do the retrieve with symlinks disabled
+ ivy.retrieve(resolve2Report.getModuleDescriptor().getModuleRevisionId(),
+ getRetrieveOptions().setMakeSymlinks(false).setDestArtifactPattern(retrievePattern));
+ // we expect org:foo-bar:2.3.4 to have been retrieved
+ final Path secondRetrieveArtifactPath = Paths.get(IvyPatternHelper.substitute(retrievePattern, "org", "foo-bar",
+ "2.3.4", "foo-bar", "jar", "jar", "default"));
+ assertTrue("Artifact wasn't retrieved to " + secondRetrieveArtifactPath, Files.exists(secondRetrieveArtifactPath));
+// assertFalse("Artifact retrieved at " + secondRetrieveArtifactPath + " wasn't expected to be a " +
+// "symlink", Files.isSymbolicLink(secondRetrieveArtifactPath));
+
+ // get hold of the contents of the retrieved artifact
+ final byte[] secondRetrievedArtifactContents = Files.readAllBytes(secondRetrieveArtifactPath);
+ // compare it to the contents of org:foo-bar:2.3.4 artifact in repo cache. Should be the same
+ assertTrue("Unexpected content in the retrieved artifact at " + secondRetrieveArtifactPath,
+ Arrays.equals(foobar234CacheFileContents, secondRetrievedArtifactContents));
+
+ // also make sure that this retrieved content hasn't messed up the other (unrelated) file, in the cache,
+ // that was previously symlinked to this retrieved path
+ assertTrue("A second retrieve of an artifact has corrupted an unrelated (previously symlinked) artifact in cache",
+ Arrays.equals(fooBar123ArtifactContentsInCache, Files.readAllBytes(Paths.get(fooBar123ArtifactInCache.getPath()))));
+
+
+ }
+
private RetrieveOptions getRetrieveOptions() {
return new RetrieveOptions();
}
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml
----------------------------------------------------------------------
diff --git a/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml b/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml
new file mode 100644
index 0000000..f49b7af
--- /dev/null
+++ b/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml
@@ -0,0 +1,22 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<ivy-module version="1.0">
+ <info organisation="org"
+ module="foo-bar"
+ revision="2.3.4"
+ />
+</ivy-module>
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar
----------------------------------------------------------------------
diff --git a/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar b/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar
index e69de29..d3cd0ae 100644
Binary files a/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar and b/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar differ
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar
----------------------------------------------------------------------
diff --git a/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar b/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar
new file mode 100644
index 0000000..26ae57a
Binary files /dev/null and b/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar differ
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/mod1/ivys/ivy-5.0.xml
----------------------------------------------------------------------
diff --git a/test/repositories/1/org/mod1/ivys/ivy-5.0.xml b/test/repositories/1/org/mod1/ivys/ivy-5.0.xml
new file mode 100644
index 0000000..417a213
--- /dev/null
+++ b/test/repositories/1/org/mod1/ivys/ivy-5.0.xml
@@ -0,0 +1,25 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<ivy-module version="1.0">
+ <info organisation="org"
+ module="mod1"
+ revision="5.0"
+ />
+ <dependencies>
+ <dependency org="org" name="foo-bar" rev="2.3.4" />
+ </dependencies>
+</ivy-module>
http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/mod1/jars/mod1-5.0.jar
----------------------------------------------------------------------
diff --git a/test/repositories/1/org/mod1/jars/mod1-5.0.jar b/test/repositories/1/org/mod1/jars/mod1-5.0.jar
new file mode 100644
index 0000000..e69de29