You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ant.apache.org by hi...@apache.org on 2012/08/15 19:32:47 UTC
svn commit: r1373534 - in /ant/ivy/core/trunk: ./ doc/use/
src/java/org/apache/ivy/ src/java/org/apache/ivy/ant/
src/java/org/apache/ivy/core/retrieve/ src/java/org/apache/ivy/util/
test/java/org/apache/ivy/core/retrieve/
Author: hibou
Date: Wed Aug 15 17:32:47 2012
New Revision: 1373534
URL: http://svn.apache.org/viewvc?rev=1373534&view=rev
Log:
IVY-1252 : symlinkmass feature based on symlink feature of ivy:retrieve (Thanks to Gene Smith)
Modified:
ant/ivy/core/trunk/CHANGES.txt
ant/ivy/core/trunk/doc/use/retrieve.html
ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java
ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java
ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java
ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java
ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java
ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
Modified: ant/ivy/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/CHANGES.txt?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/CHANGES.txt (original)
+++ ant/ivy/core/trunk/CHANGES.txt Wed Aug 15 17:32:47 2012
@@ -111,6 +111,7 @@ for detailed view of each issue, please
Ruslan Shevchenko
John Shields
Nihal Sinha
+ Gene Smith
Simon Steiner
Johan Stuyts
John Tinetti
@@ -141,6 +142,7 @@ for detailed view of each issue, please
- NEW: Support Conditional Setting of a Property (IVY-1367)
- NEW: Exposing some parent metadata (organisation, module, revision, branch) as properties (IVY-1288) (Thanks to Jean-Louis Boudart)
+- NEW: symlinkmass feature based on symlink feature of ivy:retrieve (IVY-1252) (Thanks to Gene Smith)
- IMPROVEMENT: add support for source bundles from p2 repositories
- IMPROVEMENT: add support for source URI from OBR repositories
Modified: ant/ivy/core/trunk/doc/use/retrieve.html
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/doc/use/retrieve.html?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/doc/use/retrieve.html (original)
+++ ant/ivy/core/trunk/doc/use/retrieve.html Wed Aug 15 17:32:47 2012
@@ -62,7 +62,17 @@ Possible values are:
<li><b>always</b></li> always overwrite the destination file
<li><b>never</b></li> never overwrite the destination file
</ul></td><td>No. Defaults to 'newer'.</td></tr>
- <tr><td>symlink</td><td>true to create symbolic links, false to copy the artifacts. The destination of the symbolic links depends on the value of the useOrigin attribute <span class="since">(since 2.0)</span></td><td>No. Defaults to false</td></tr>
+ <tr><td>symlink</td><td>true to create symbolic links, false to copy the artifacts.
+ The destination of the symbolic links depends on the value of the useOrigin attribute.
+ (requires "ln" to be a valid command, and to support the options -s and -f (works on UNIX/Linux, on other systems you may need to script "ln")
+ <span class="since">(since 2.0)</span></td><td>No. Defaults to false</td></tr>
+ <tr><td>symlinkmass</td><td>true to create symbolic links in mass, false to copy the artifacts.
+ "symlinkmass" overrides "symlink" if both are set to "true".
+ "symlinkmass" will create the same symbolic links "symlink" does, but with a single process call to "sh" with batched "ln" commands passed in as standard input (works on UNIX/Linux, on other systems you may need to script it)
+ Far large lists of resolved jars, this can be dramatically faster.
+ The destination of the symbolic links depends on the value of the useOrigin attribute.
+ The events "StartRetrieveArtifactEvent" and EndRetrieveEvent are NOT fired by this activity, because it is not clear when they should be called.
+ <span class="since">(since 2.4)</span></td><td>No. Defaults to false</td></tr>
<tr><td>settingsRef</td><td>A reference to the ivy settings that must be used by this task <span class="since">(since 2.0)</span></td><td>No, 'ivy.instance' is taken by default.</td></tr></tbody>
<tr><td>log</td><td>the log setting to use during the resolve and retrieve process. <span class="since">(since 2.0)</span><br/>
Available options are the same as for [[ant:resolve]] when used to trigger resolve automatically (see [[ant:postresolvetask]]), or the following for the retrieve process only:
Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/Main.java Wed Aug 15 17:32:47 2012
@@ -287,7 +287,8 @@ public final class Main {
.setUseOrigin(line.hasOption("useOrigin"))
.setDestIvyPattern(ivyPattern)
.setArtifactFilter(FilterHelper.getArtifactTypeFilter(line.getOptionValues("types")))
- .setMakeSymlinks(line.hasOption("symlink")));
+ .setMakeSymlinks(line.hasOption("symlink"))
+ .setMakeSymlinksInMass(line.hasOption("symlinkmass")));
}
if (line.hasOption("cachepath")) {
outputCachePath(ivy, cache, md, confs, line.getOptionValue("cachepath",
Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/ant/IvyRetrieve.java Wed Aug 15 17:32:47 2012
@@ -50,6 +50,8 @@ public class IvyRetrieve extends IvyPost
private boolean sync = false;
private boolean symlink = false;
+
+ private boolean symlinkmass = false;
private String overwriteMode = RetrieveOptions.OVERWRITEMODE_NEWER;
@@ -106,6 +108,7 @@ public class IvyRetrieve extends IvyPost
.setOverwriteMode(getOverwriteMode())
.setUseOrigin(isUseOrigin())
.setMakeSymlinks(symlink)
+ .setMakeSymlinksInMass(symlinkmass)
.setResolveId(getResolveId())
.setMapper(mapper == null ? null : new MapperAdapter(mapper)));
@@ -168,6 +171,13 @@ public class IvyRetrieve extends IvyPost
this.symlink = symlink;
}
+ /**
+ * Option to create symlinks in one mass action, instead of separately.
+ */
+ public void setSymlinkmass(boolean symlinkmass) {
+ this.symlinkmass = symlinkmass;
+ }
+
public void setOverwriteMode(String overwriteMode) {
if (!OVERWRITEMODE_VALUES.contains(overwriteMode)) {
throw new IllegalArgumentException("invalid overwriteMode value '" + overwriteMode + "'. "
Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveEngine.java Wed Aug 15 17:32:47 2012
@@ -110,6 +110,7 @@ public class RetrieveEngine {
}
try {
+ Map/*<File, File>*/ destToSrcMap = null;
Map artifactsToCopy = determineArtifactsToCopy(mrid, destFilePattern, options);
File fileRetrieveRoot = settings.resolveFile(
IvyPatternHelper.getTokenRoot(destFilePattern));
@@ -122,6 +123,12 @@ public class RetrieveEngine {
// for sync)
Collection targetIvysStructure = new HashSet(); // same for ivy files
+ if (options.isMakeSymlinksInMass()) {
+ // The HashMap is of "destToSrc" because src could go two places, but dest can only
+ // come from one
+ destToSrcMap = new HashMap();
+ }
+
// do retrieve
long totalCopiedSize = 0;
for (Iterator iter = artifactsToCopy.keySet().iterator(); iter.hasNext();) {
@@ -139,17 +146,29 @@ public class RetrieveEngine {
if (!settings.isCheckUpToDate() || !upToDate(archive, destFile, options)) {
Message.verbose("\t\tto " + destFile);
if (this.eventManager != null) {
- this.eventManager.fireIvyEvent(
- new StartRetrieveArtifactEvent(artifact, destFile));
+ // There is no unitary event for the mass sym linking.
+ // skip the event declaration.
+ if (!options.isMakeSymlinksInMass()) {
+ this.eventManager.fireIvyEvent(new StartRetrieveArtifactEvent(
+ artifact, destFile));
+ }
}
- if (options.isMakeSymlinks()) {
+ if (options.isMakeSymlinksInMass()) {
+ if (FileUtil.prepareCopy(archive, destFile, true)) {
+ destToSrcMap.put(destFile, archive);
+ }
+ } else if (options.isMakeSymlinks()) {
FileUtil.symlink(archive, destFile, null, true);
} else {
FileUtil.copy(archive, destFile, null, true);
}
if (this.eventManager != null) {
- this.eventManager.fireIvyEvent(
- new EndRetrieveArtifactEvent(artifact, destFile));
+ // There is no unitary event for the mass sym linking.
+ // skip the event declaration.
+ if (!options.isMakeSymlinksInMass()) {
+ this.eventManager.fireIvyEvent(new EndRetrieveArtifactEvent(
+ artifact, destFile));
+ }
}
totalCopiedSize += destFile.length();
report.addCopiedFile(destFile, artifact);
@@ -168,6 +187,11 @@ public class RetrieveEngine {
}
}
+ if (options.isMakeSymlinksInMass()) {
+ Message.verbose("\tMass symlinking " + destToSrcMap.size() + " files");
+ FileUtil.symlinkInMass(destToSrcMap, true);
+ }
+
if (options.isSync()) {
Message.verbose("\tsyncing...");
Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/core/retrieve/RetrieveOptions.java Wed Aug 15 17:32:47 2012
@@ -75,6 +75,12 @@ public class RetrieveOptions extends Log
private boolean makeSymlinks = false;
/**
+ * True if symbolic links should be created all at once, instead of one at a time. Works only on
+ * OS supporting with both "sh" (a shell) and "ln" (the link command).
+ */
+ private boolean makeSymlinksInMass = false;
+
+ /**
* The id used to store the resolve information.
*/
private String resolveId;
@@ -147,11 +153,20 @@ public class RetrieveOptions extends Log
return makeSymlinks;
}
+ public boolean isMakeSymlinksInMass() {
+ return makeSymlinksInMass;
+ }
+
public RetrieveOptions setMakeSymlinks(boolean makeSymlinks) {
this.makeSymlinks = makeSymlinks;
return this;
}
+ public RetrieveOptions setMakeSymlinksInMass(boolean makeSymlinksInMass) {
+ this.makeSymlinksInMass = makeSymlinksInMass;
+ return this;
+ }
+
public boolean isSync() {
return sync;
}
Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/FileUtil.java Wed Aug 15 17:32:47 2012
@@ -31,9 +31,13 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Stack;
import java.util.StringTokenizer;
+import java.util.regex.Pattern;
import org.apache.ivy.util.url.URLHandlerRegistry;
@@ -52,22 +56,75 @@ public final class FileUtil {
private static final byte[] EMPTY_BUFFER = new byte[0];
- public static void symlink(File src, File dest, CopyProgressListener l, boolean overwrite)
- throws IOException {
+ private static final Pattern ALLOWED_PATH_PATTERN = Pattern.compile("[\\w-./\\\\:~ ]+");
+
+ public static void symlinkInMass(Map/* <File, File> */destToSrcMap, boolean overwrite) throws IOException {
+
+ // This pattern could be more forgiving if somebody wanted it to be...
+ // ...but this should satisfy 99+% of all needs, without letting unsafe operations be done.
+ // If you paths is not supported, you then skip this mass option.
+ // NOTE: A space inside the path is allowed (I can't control other programmers who like them
+ // in their working directory names)...
+ // but trailing spaces on file names will be checked otherwise and refused.
try {
- if (dest.exists()) {
- if (!dest.isFile()) {
- throw new IOException("impossible to copy: destination is not a file: " + dest);
+ StringBuffer sb = new StringBuffer();
+
+ Iterator keyItr = destToSrcMap.entrySet().iterator();
+ while (keyItr.hasNext()) {
+ Entry/*<File, File>*/ entry = (Entry) keyItr.next();
+ File destFile = (File) entry.getKey();
+ File srcFile = (File) entry.getValue();
+ if (!ALLOWED_PATH_PATTERN.matcher(srcFile.getAbsolutePath()).matches()) {
+ throw new IOException("Unsafe file to 'mass' symlink: '"
+ + srcFile.getAbsolutePath() + "'");
}
- if (!overwrite) {
- Message.verbose(dest + " already exists, nothing done");
- return;
+ if (!ALLOWED_PATH_PATTERN.matcher(destFile.getAbsolutePath()).matches()) {
+ throw new IOException("Unsafe file to 'mass' symlink to: '"
+ + destFile.getAbsolutePath() + "'");
+ }
+
+ // Add to our buffer of commands
+ sb.append("ln -s -f \"" + srcFile.getAbsolutePath() + "\" \"" + destFile.getAbsolutePath() + "\";");
+ if (keyItr.hasNext()) {
+ sb.append("\n");
}
}
- if (dest.getParentFile() != null) {
- dest.getParentFile().mkdirs();
+
+ String commands = sb.toString();
+ // Run the buffer of commands we have built.
+ Runtime runtime = Runtime.getRuntime();
+ Message.verbose("executing \"sh\" of:\n\t" + commands.replace("\n", "\n\t"));
+ Process process = runtime.exec("sh");
+ OutputStream os = process.getOutputStream();
+ os.write(commands.getBytes("UTF-8"));
+ os.flush();
+ os.close();
+
+ if (process.waitFor() != 0) {
+ InputStream errorStream = process.getErrorStream();
+ InputStreamReader isr = new InputStreamReader(errorStream);
+ BufferedReader br = new BufferedReader(isr);
+
+ StringBuffer error = new StringBuffer();
+ String line;
+ while ((line = br.readLine()) != null) {
+ error.append(line);
+ error.append('\n');
+ }
+
+ throw new IOException("error running ln commands with 'sh':\n" + error);
}
+ } catch (InterruptedException x) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ public static void symlink(File src, File dest, CopyProgressListener l, boolean overwrite)
+ throws IOException {
+ if (!prepareCopy(src, dest, overwrite)) {
+ return;
+ }
+ try {
Runtime runtime = Runtime.getRuntime();
Message.verbose("executing 'ln -s -f " + src.getAbsolutePath() + " " + dest.getPath()
+ "'");
@@ -114,8 +171,7 @@ public final class FileUtil {
return copy(src, dest, l, false);
}
- public static boolean copy(File src, File dest, CopyProgressListener l, boolean overwrite)
- throws IOException {
+ public static boolean prepareCopy(File src, File dest, boolean overwrite) throws IOException {
if (dest.exists()) {
if (!dest.isFile()) {
throw new IOException("impossible to copy: destination is not a file: " + dest);
@@ -129,6 +185,17 @@ public final class FileUtil {
return false;
}
}
+ if (dest.getParentFile() != null) {
+ dest.getParentFile().mkdirs();
+ }
+ return true;
+ }
+
+ public static boolean copy(File src, File dest, CopyProgressListener l, boolean overwrite)
+ throws IOException {
+ if (!prepareCopy(src, dest, overwrite)) {
+ return false;
+ }
copy(new FileInputStream(src), dest, l);
long srcLen = src.length();
long destLen = dest.length();
Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java?rev=1373534&r1=1373533&r2=1373534&view=diff
==============================================================================
--- ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java (original)
+++ ant/ivy/core/trunk/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java Wed Aug 15 17:32:47 2012
@@ -195,17 +195,37 @@ public class RetrieveTest extends TestCa
ModuleDescriptor md = report.getModuleDescriptor();
assertNotNull(md);
+ getRetrieveOptions().setMakeSymlinks(true);
+
String pattern = "build/test/retrieve/[module]/[conf]/[artifact]-[revision].[ext]";
- ivy
- .retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions().setMakeSymlinks(
- true));
+ ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2", "jar",
"jar", "default"));
pattern = "build/test/retrieve/[module]/[conf]/[type]s/[artifact]-[revision].[ext]";
- ivy
- .retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions().setMakeSymlinks(
- true));
+ ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
+ assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2", "jar",
+ "jar", "default"));
+ }
+
+ public void testRetrieveWithSymlinksMass() throws Exception {
+ // mod1.1 depends on mod1.2
+ ResolveReport report = ivy.resolve(new File(
+ "test/repositories/1/org1/mod1.1/ivys/ivy-1.0.xml").toURL(),
+ getResolveOptions(new String[] {"*"}));
+ assertNotNull(report);
+ ModuleDescriptor md = report.getModuleDescriptor();
+ assertNotNull(md);
+
+ getRetrieveOptions().setMakeSymlinksInMass(true);
+
+ String pattern = "build/test/retrieve/[module]/[conf]/[artifact]-[revision].[ext]";
+ ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
+ assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2", "jar",
+ "jar", "default"));
+
+ pattern = "build/test/retrieve/[module]/[conf]/[type]s/[artifact]-[revision].[ext]";
+ ivy.retrieve(md.getModuleRevisionId(), pattern, getRetrieveOptions());
assertLink(IvyPatternHelper.substitute(pattern, "org1", "mod1.2", "2.0", "mod1.2", "jar",
"jar", "default"));
}