You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/11/12 22:17:10 UTC
svn commit: r1408463 - in /hadoop/common/branches/branch-trunk-win:
hadoop-dist/pom.xml hadoop-project-dist/pom.xml
Author: suresh
Date: Mon Nov 12 21:17:09 2012
New Revision: 1408463
URL: http://svn.apache.org/viewvc?rev=1408463&view=rev
Log:
HADOOP-9008. Building hadoop tarball fails on Windows. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-project-dist/pom.xml
Modified: hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml?rev=1408463&r1=1408462&r2=1408463&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml Mon Nov 12 21:17:09 2012
@@ -90,50 +90,6 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <echo file="${project.build.directory}/dist-layout-stitching.sh">
- run() {
- echo "\$ ${@}"
- "${@}"
- res=$?
- if [ $res != 0 ]; then
- echo
- echo "Failed!"
- echo
- exit $res
- fi
- }
-
- ROOT=`cd ${basedir}/..;pwd`
- echo
- echo "Current directory `pwd`"
- echo
- run rm -rf hadoop-${project.version}
- run mkdir hadoop-${project.version}
- run cd hadoop-${project.version}
- run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* .
- run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
- run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
- run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
- run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
- run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* .
- echo
- echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
- echo
- </echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-layout-stitching.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- <execution>
<id>tar</id>
<phase>package</phase>
<goals>
@@ -141,26 +97,61 @@
</goals>
<configuration>
<target if="tar">
- <echo file="${project.build.directory}/dist-tar-stitching.sh">
- run() {
- echo "\$ ${@}"
- "${@}"
- res=$?
- if [ $res != 0 ]; then
- echo
- echo "Failed!"
- echo
- exit $res
- fi
- }
+ <!-- This script preserves permissions and symlinks. -->
+ <!-- Python requires resetting indentation to far left. -->
+ <echo file="${project.build.directory}/dist-maketar.py">
+from os.path import abspath, basename, isdir, join
+import tarfile
+
+def make_file_filter(root, file_name_filter):
+ def filter_func(tar_info):
+ if tar_info.name == root:
+ # Always include root directory. Otherwise, tarfile.add assumes you are
+ # filtering out the whole directory and produces an empty tar.
+ return tar_info
+ if tar_info.isfile() or tar_info.issym():
+ # Include files and symlinks only if they match the specified name filter.
+ if file_name_filter(basename(tar_info.name)):
+ return tar_info
+ # Otherwise, exclude.
+ return None
+ return filter_func
+
+target_dirs = [
+ abspath(r"${basedir}/../hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}"),
+ abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}"),
+ abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}"),
+ abspath(r"${basedir}/../hadoop-yarn-project/target/hadoop-yarn-project-${project.version}"),
+ abspath(r"${basedir}/../hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}"),
+ abspath(r"${basedir}/../hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}")
+]
+base_name = "hadoop" + "-" + "${project.version}"
+dir_name = abspath(join(r"${project.build.directory}", base_name))
+tar_name = dir_name + ".tar.gz"
+
+with tarfile.open(tar_name, "w:gz") as tar:
+ for target_dir in target_dirs:
+ tar.add(target_dir, arcname=base_name)
+ native_dir = abspath(join(target_dir, "../native/target/usr/local/lib"))
+ if isdir(native_dir):
+ arc_name = base_name + "/lib/native"
+ tar.add(native_dir, arcname=arc_name,
+ filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
+ bin_dir = abspath(join(target_dir, "../bin"))
+ if isdir(bin_dir):
+ arc_name = base_name + "/bin"
+ tar.add(bin_dir, arcname=arc_name)
+ if "${bundle.snappy}" == "true":
+ arc_name = base_name + "/lib/native"
+ tar.add(r"${snappy.lib}", arcname=arc_name,
+ filter=make_file_filter(arc_name, lambda file: "snappy" in file))
- run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version}
- echo
- echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
- echo
+print
+print "Hadoop dist tar available at: " + tar_name
+print
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-tar-stitching.sh"/>
+ <exec executable="python" dir="${project.build.directory}" failonerror="true">
+ <arg value="dist-maketar.py" />
</exec>
</target>
</configuration>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-project-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project-dist/pom.xml?rev=1408463&r1=1408462&r2=1408463&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-project-dist/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-project-dist/pom.xml Mon Nov 12 21:17:09 2012
@@ -333,43 +333,6 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>pre-dist</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <!-- Using Unix script to preserve symlinks -->
- <echo file="${project.build.directory}/dist-copynativelibs.sh">
-
- which cygpath 2> /dev/null
- if [ $? = 1 ]; then
- BUILD_DIR="${project.build.directory}"
- else
- BUILD_DIR=`cygpath --unix '${project.build.directory}'`
- fi
- TAR='tar cf -'
- UNTAR='tar xfBp -'
- LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib"
- if [ -d $${LIB_DIR} ] ; then
- TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/lib/native"
- mkdir -p $${TARGET_DIR}
- cd $${LIB_DIR}
- $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR)
- if [ "${bundle.snappy}" = "true" ] ; then
- cd ${snappy.lib}
- $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
- fi
- fi
- </echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-copynativelibs.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- <execution>
<id>tar</id>
<phase>package</phase>
<goals>
@@ -377,20 +340,48 @@
</goals>
<configuration>
<target if="tar">
- <!-- Using Unix script to preserve symlinks -->
- <echo file="${project.build.directory}/dist-maketar.sh">
+ <!-- This script preserves permissions and symlinks. -->
+ <!-- Python requires resetting indentation to far left. -->
+ <echo file="${project.build.directory}/dist-maketar.py">
+from os.path import abspath, basename, isdir, join
+import tarfile
+
+def make_file_filter(root, file_name_filter):
+ def filter_func(tar_info):
+ if tar_info.name == root:
+ # Always include root directory. Otherwise, tarfile.add assumes you are
+ # filtering out the whole directory and produces an empty tar.
+ return tar_info
+ if tar_info.isfile() or tar_info.issym():
+ # Include files and symlinks only if they match the specified name filter.
+ if file_name_filter(basename(tar_info.name)):
+ return tar_info
+ # Otherwise, exclude.
+ return None
+ return filter_func
+
+base_name = "${project.artifactId}" + "-" + "${project.version}"
+dir_name = abspath(join(r"${project.build.directory}", base_name))
+tar_name = dir_name + ".tar.gz"
- which cygpath 2> /dev/null
- if [ $? = 1 ]; then
- BUILD_DIR="${project.build.directory}"
- else
- BUILD_DIR=`cygpath --unix '${project.build.directory}'`
- fi
- cd ${BUILD_DIR}
- tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+with tarfile.open(tar_name, "w:gz") as tar:
+ tar.add(dir_name, arcname=base_name)
+ native_dir = abspath(r"${project.build.directory}/native/target/usr/local/lib")
+ if isdir(native_dir):
+ arc_name = base_name + "/lib/native"
+ tar.add(native_dir, arcname=arc_name,
+ filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
+ if "${bundle.snappy}" == "true":
+ arc_name = base_name + "/lib/native"
+ tar.add(r"${snappy.lib}", arcname=arc_name,
+ filter=make_file_filter(arc_name, lambda file: "snappy" in file))
+ bin_dir = abspath(r"${project.build.directory}/bin")
+ if isdir(bin_dir):
+ arc_name = base_name + "/bin"
+ tar.add(bin_dir, arcname=arc_name)
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-maketar.sh"/>
+ <exec executable="python" dir="${project.build.directory}" failonerror="true">
+ <arg value="dist-maketar.py" />
</exec>
</target>
</configuration>