You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ac...@apache.org on 2017/04/01 23:03:38 UTC
incubator-systemml git commit: [SYSTEMML-1440] Automate Release
Artifact verification
Repository: incubator-systemml
Updated Branches:
refs/heads/master 648c62850 -> c04f7b0fb
[SYSTEMML-1440] Automate Release Artifact verification
Closes #448.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c04f7b0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c04f7b0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c04f7b0f
Branch: refs/heads/master
Commit: c04f7b0fb9007e306169df3ab9ff61c1fbd8b60e
Parents: 648c628
Author: Arvind Surve <ac...@yahoo.com>
Authored: Sat Apr 1 16:03:02 2017 -0700
Committer: Arvind Surve <ac...@yahoo.com>
Committed: Sat Apr 1 16:03:02 2017 -0700
----------------------------------------------------------------------
dev/release/artifact-verify.sh | 2 +-
.../sysml/validation/ValidateLicAndNotice.java | 299 ++++++++++++++++---
2 files changed, 257 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04f7b0f/dev/release/artifact-verify.sh
----------------------------------------------------------------------
diff --git a/dev/release/artifact-verify.sh b/dev/release/artifact-verify.sh
index bd2c380..fbe7b96 100755
--- a/dev/release/artifact-verify.sh
+++ b/dev/release/artifact-verify.sh
@@ -110,7 +110,7 @@ fi
if [[ "$COMPILE_CODE" == "true" ]]; then
echo "Compiling artifact utility..."
- javac -classpath ../../../../..//target/lib/commons-compress-1.4.1.jar:../../../../..//target/lib/commons-io-2.4.jar:. org/apache/sysml/validation/ValidateLicAndNotice.java
+ javac -Xlint:unchecked -classpath ../../../../..//target/lib/commons-compress-1.4.1.jar:../../../../..//target/lib/commons-io-2.4.jar:. org/apache/sysml/validation/ValidateLicAndNotice.java
cd "$ORIG_DIR" # Return to directoryt from it was called.
exit 0
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04f7b0f/dev/release/src/test/java/org/apache/sysml/validation/ValidateLicAndNotice.java
----------------------------------------------------------------------
diff --git a/dev/release/src/test/java/org/apache/sysml/validation/ValidateLicAndNotice.java b/dev/release/src/test/java/org/apache/sysml/validation/ValidateLicAndNotice.java
index 633f174..674b442 100644
--- a/dev/release/src/test/java/org/apache/sysml/validation/ValidateLicAndNotice.java
+++ b/dev/release/src/test/java/org/apache/sysml/validation/ValidateLicAndNotice.java
@@ -29,7 +29,11 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.StringTokenizer;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -49,12 +53,33 @@ public class ValidateLicAndNotice
{
//Return codes
public static final int SUCCESS = 0;
- public static final int NO_ZIP_TGZ = 1; // 0000 0000 0000 0001
- public static final int FILE_NOT_IN_LIC = 2; // 0000 0000 0000 0010
- public static final int FILE_NOT_IN_ZIP = 4; // 0000 0000 0000 0100
+ public static final int NO_ZIP_TGZ = 1; // 0000 0000 0000 0001
+ public static final int FILE_NOT_IN_LIC = 2; // 0000 0000 0000 0010
+ public static final int FILE_NOT_IN_ZIP = 4; // 0000 0000 0000 0100
+ public static final int FAILED_TO_EXTRACT = 8; // 0000 0000 0000 1000
+ public static final int LIC_NOT_EXIST = 16; // 0000 0000 0001 0000
public static final int FAILURE = 0xFFFF;
+ public static final boolean bSUCCESS = true;
+ public static final boolean bFAILURE = false;
+
+
+ //DEBUG PRINT CODE
+ public static final int DEBUG_PRINT_LEVEL = 3;
+
+ public static final int DEBUG_ERROR = 0;
+ public static final int DEBUG_WARNING = 1;
+ public static final int DEBUG_INFO = 2;
+ public static final int DEBUG_INFO2 = 3;
+ public static final int DEBUG_CODE = 4;
+
+
+ static final int BUFFER = 2048;
+
//String constants
+ public static final String SYSTEMML_NAME = "SystemML";
+ public static final String SYSTEMML_PACKAGE = "org/apache/sysml";
+
public static final String ZIP = "zip";
public static final String TGZ = "tgz";
public static final String LICENSE = "LICENSE";
@@ -64,13 +89,22 @@ public class ValidateLicAndNotice
public static final String LIB = "lib";
public static final String PDB = "pdb";
public static final String EXE = "exe";
+ public static final String CLASS = "class";
public static String[] fileTypes = {JAR, DLL, EXP, LIB, PDB, EXE};
- static final int BUFFER = 2048;
// Zip Distribution directory.
private String strDistroDir = "../../../target/release/incubator-systemml/target/";
+ public static final ArrayList<String[]> packageLicenses = new ArrayList<String[]> ();
+ static {
+ String[] strA1 = {"org/antlr", "ANTLR 4 Runtime (http://www.antlr.org/antlr4-runtime) org.antlr:antlr4-runtime:4.5.3"};
+ String[] strA2 = {"org/apache/wink/json4j","Apache Wink :: JSON4J (http://www.apache.org/wink/wink-json4j/) org.apache.wink:wink-json4j:1.4"};
+
+ packageLicenses.add(strA1);
+ packageLicenses.add(strA2);
+ }
+
public ValidateLicAndNotice() {
}
@@ -98,7 +132,7 @@ public class ValidateLicAndNotice
File distroRoot = new File( getDistroDir());
File libDirectory = distroRoot;
if (!libDirectory.exists()) {
- System.out.println("Distribution folder '" + libDirectory.getAbsoluteFile().toString() + "' does not exist.");
+ debugPrint(DEBUG_ERROR, "Distribution folder '" + libDirectory.getAbsoluteFile().toString() + "' does not exist.");
return NO_ZIP_TGZ;
}
@@ -108,38 +142,44 @@ public class ValidateLicAndNotice
List<String> zips = getZipsInDistro(libDirectory);
if(zips.size() == 0) {
- System.out.println("Can't find zip/tgz files in folder: " + libDirectory.getAbsoluteFile().toString());
+ debugPrint(DEBUG_ERROR, "Can't find zip/tgz files in folder: " + libDirectory.getAbsoluteFile().toString());
return NO_ZIP_TGZ;
}
for (String zipFile: zips)
{
retCodeForAllFileTypes = SUCCESS;
- System.out.println("======================================================================================");
- System.out.println("Validating zip file : " + zipFile + " ...");
+ debugPrint(DEBUG_INFO, "======================================================================================");
+ debugPrint(DEBUG_INFO, "Validating zip file : " + zipFile + " ...");
for (String fileType: fileTypes) {
retCode = SUCCESS;
List<String> filesAll = null;
- if (zipFile.endsWith("." + ZIP)) {
- ValidateLicAndNotice.extractFileFromZip(libDirectory + "/" + zipFile, LICENSE, outTempDir.getAbsolutePath());
- filesAll = getFilesFromZip(libDirectory + "/" + zipFile, fileType);
- } else if (zipFile.endsWith("." + TGZ)) {
- ValidateLicAndNotice.extractFileFromTGZ(libDirectory + "/" + zipFile, LICENSE, outTempDir.getAbsolutePath());
- filesAll = getFilesFromTGZ(libDirectory + "/" + zipFile, fileType);
- }
+ // Extract license only at first time in all filetypes validation for a given zip.
+ if(fileType == JAR)
+ if (!ValidateLicAndNotice.extractFile(libDirectory + "/" + zipFile, LICENSE, outTempDir.getAbsolutePath(), true))
+ return FAILED_TO_EXTRACT;
+
+ filesAll = getFiles(libDirectory + "/" + zipFile, fileType);
File licenseFile = new File(outTempDir, LICENSE);
List<String> files = new ArrayList<String>();
+ List<String> fileSysml = new ArrayList<String>();
for (String file : filesAll) {
- String strSysMLName = "SystemML";
- int sysmlLen = strSysMLName.length();
+ int sysmlLen = SYSTEMML_NAME.length();
String strBegPart = file.substring(0, sysmlLen);
- if (strBegPart.compareToIgnoreCase(strSysMLName) != 0)
+ if (strBegPart.compareToIgnoreCase(SYSTEMML_NAME) != 0)
files.add(file);
+ else
+ fileSysml.add(file);
}
+ // Validate shaded jar only one time.
+ if(fileType == JAR)
+ for (String file : fileSysml)
+ retCode += ValidateLicAndNotice.validateShadedLic(libDirectory + "/" + zipFile, file, outTempDir.getAbsolutePath());
+
List<String> bad2 = getLICENSEFilesNotInList(licenseFile, files, fileType);
if (bad2.size() > 0) {
System.err.println("Files in LICENSE but not in Distribution: " + bad2);
@@ -152,23 +192,127 @@ public class ValidateLicAndNotice
retCode += FILE_NOT_IN_LIC;
}
- if (bad1.size() > 0 || bad2.size() > 0) {
- System.out.println("ERROR: License validation failed for zip file " + zipFile + " with error code " + retCode + ", please validate file manually.");
+ if (retCode > SUCCESS) {
+ debugPrint(DEBUG_ERROR, "License validation of file types " + fileType + " failed for zip file " + zipFile + " with error code " + retCode + ", please validate file manually.");
retCodeForAllFileTypes = FAILURE;
}
}
if(retCodeForAllFileTypes == SUCCESS)
- System.out.println("Validation of zip file : " + zipFile + " completed successfully.");
+ debugPrint(DEBUG_INFO, "Validation of zip file : " + zipFile + " completed successfully.");
retCodeAll = retCode != SUCCESS?FAILURE:retCodeAll;
}
- System.out.println("======================================================================================");
+ debugPrint(DEBUG_INFO, "======================================================================================");
FileUtils.deleteDirectory(outTempDir);
return retCodeAll;
}
/**
+ * This will validate objects (class files) from jar file within a zip file.
+ *
+ * @param zipFileName is the name of zip file from which set of class packages will be returned.
+ * @param fileName is the name of the file within zip (jar) file to validate list of packages within.
+ * @param outTempDir is the temporary directory name.
+ * @return Success or Failure code
+ */
+ public static int validateShadedLic(String zipFileName, String file, String outTempDir) throws Exception
+ {
+
+ File outTempDir2 = new File (outTempDir + "/" + "2");
+ outTempDir2.mkdir();
+ if(!ValidateLicAndNotice.extractFile(zipFileName, file, outTempDir2.getAbsolutePath(), false))
+ return FAILED_TO_EXTRACT;
+ if(!ValidateLicAndNotice.extractFile(outTempDir2.getAbsolutePath()+"/"+file, LICENSE, outTempDir2.getAbsolutePath(), true))
+ return FAILED_TO_EXTRACT;
+
+ HashMap<String, Boolean> hashMapPackages = getPackagesFromZip(outTempDir2.getAbsolutePath() + "/" + file);
+ for (String packageName: hashMapPackages.keySet())
+ debugPrint(DEBUG_CODE, "Package: " + packageName + " Licensed: " + hashMapPackages.get(packageName));
+
+ int iRetCode = ValidateLicAndNotice.validatePackages(outTempDir2.getAbsolutePath()+"/"+LICENSE, hashMapPackages);
+
+ FileUtils.deleteDirectory(outTempDir2);
+ return iRetCode;
+ }
+
+ /**
+ * This will validate objects (class files) against license file.
+ *
+ * @param licenseFile is the name of the license file.
+ * @param hashMapPackages is the list of package names to be validated for license.
+ * @return Success or Failure code
+ */
+ public static int validatePackages(String licenseFile, HashMap<String, Boolean> hashMapPackages) throws Exception
+ {
+ int iRetCode = SUCCESS;
+ BufferedReader reader = new BufferedReader(new FileReader(licenseFile));
+ String line = null;
+ HashSet <String> packageValidLic = new HashSet<String>();
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ for(int i=0; i <packageLicenses.size(); ++i) {
+ if (line.contains(packageLicenses.get(i)[1])) {
+ packageValidLic.add(packageLicenses.get(i)[0]);
+ debugPrint(DEBUG_INFO2, "License for package " + packageLicenses.get(i)[0] + " exists.");
+ }
+ }
+ }
+
+ Iterator<Map.Entry<String, Boolean>> itPackages = hashMapPackages.entrySet().iterator();
+ while (itPackages.hasNext()) {
+ Map.Entry<String, Boolean> pairPackage = (Map.Entry<String, Boolean>) itPackages.next();
+ Iterator<String> itLicPackages = packageValidLic.iterator();
+ while (itLicPackages.hasNext()) {
+ if(((String)pairPackage.getKey()).startsWith((String)itLicPackages.next()))
+ pairPackage.setValue(Boolean.TRUE);
+ }
+ }
+
+ itPackages = hashMapPackages.entrySet().iterator();
+ while (itPackages.hasNext()) {
+ Map.Entry pairPackage = (Map.Entry) itPackages.next();
+ if(!(Boolean)pairPackage.getValue()) {
+ debugPrint(DEBUG_WARNING, "Could not validate license for package " + pairPackage.getKey() + ", please validate manually.");
+ iRetCode = LIC_NOT_EXIST;
+ }
+ }
+
+ return iRetCode;
+ }
+
+ /**
+ * This will return the set of packages from zip file.
+ *
+ * @param zipFileName is the name of zip file from which set of class packages will be returned.
+ * @return Returns set of packages for classes included in the zip file .
+ */
+ public static HashMap<String, Boolean> getPackagesFromZip (String zipFileName) throws Exception{
+ HashMap<String, Boolean> packages = new HashMap<String, Boolean>();
+ try {
+ ZipEntry entry;
+ ZipFile zipfile = new ZipFile(zipFileName);
+ Enumeration e = zipfile.entries();
+ while(e.hasMoreElements()) {
+ entry = (ZipEntry) e.nextElement();
+ if(! entry.getName().startsWith(SYSTEMML_PACKAGE) &&
+ entry.getName().endsWith("." + CLASS)) {
+ int iPos = entry.getName().lastIndexOf("/");
+ if (iPos > 0) {
+ String strPackageName = entry.getName().substring(0, iPos);
+ packages.put(strPackageName, Boolean.FALSE);
+ debugPrint(DEBUG_CODE, "Package found : " + strPackageName);
+ }
+ }
+ }
+ } catch(Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ return packages;
+ }
+
+ /**
* This will return the list of files in licsense files but not in list of files coming from zip/tgz file.
*
* @param licenseFile is the file against which contents of zip/tgz file gets compared.
@@ -299,14 +443,35 @@ public class ValidateLicAndNotice
}
/**
+ * This will return the file from zip/tgz file and store it in specified location.
+ *
+ * @param zipFileName is the name of zip/tgz file from which file to be extracted.
+ * @param fileName is the name of the file to be extracted.
+ * @param strDestLoc is the location where file will be extracted.
+ * @param bFirstDirLevel to indicate to get file from first directory level.
+ * @return Success or Failure
+ */
+ public static boolean extractFile(String zipFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) {
+ debugPrint(DEBUG_CODE, "Extracting " + fileName + " from jar/zip/tgz file " + zipFileName);
+ if (zipFileName.endsWith("." + ZIP) || zipFileName.endsWith("." + JAR))
+ return extractFileFromZip(zipFileName, fileName, strDestLoc, bFirstDirLevel);
+ else if (zipFileName.endsWith("." + TGZ))
+ return extractFileFromTGZ(zipFileName, fileName, strDestLoc, bFirstDirLevel);
+ return bFAILURE;
+ }
+
+
+ /**
* This will return the file from zip file and store it in specified location.
*
* @param zipFileName is the name of zip file from which file to be extracted.
* @param fileName is the name of the file to be extracted.
* @param strDestLoc is the location where file will be extracted.
- * @return
+ * @param bFirstDirLevel to indicate to get file from first directory level.
+ * @return Sucess or Failure
*/
- public static void extractFileFromZip (String zipFileName, String fileName, String strDestLoc) {
+ public static boolean extractFileFromZip (String zipFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) {
+ boolean bRetCode = bFAILURE;
try {
BufferedOutputStream bufOut = null;
BufferedInputStream bufIn = null;
@@ -315,13 +480,12 @@ public class ValidateLicAndNotice
Enumeration e = zipfile.entries();
while(e.hasMoreElements()) {
entry = (ZipEntry) e.nextElement();
-// System.out.println("Entry Name: " + entry);
if(! entry.getName().endsWith(fileName))
continue;
- //Get file at root (in single directory) level. This is for License in root location.
- if( entry.getName().indexOf('/') != entry.getName().lastIndexOf('/'))
+ //Get file at root (in single directory) level. This is for License in root location.
+ if( bFirstDirLevel &&
+ (entry.getName().indexOf('/') != entry.getName().lastIndexOf('/')))
continue;
-// System.out.println("License found : " + entry);
bufIn = new BufferedInputStream(zipfile.getInputStream(entry));
int count;
byte data[] = new byte[BUFFER];
@@ -334,11 +498,13 @@ public class ValidateLicAndNotice
bufOut.flush();
bufOut.close();
bufIn.close();
+ bRetCode = bSUCCESS;
break;
}
} catch(Exception e) {
e.printStackTrace();
}
+ return bRetCode;
}
/**
@@ -347,9 +513,13 @@ public class ValidateLicAndNotice
* @param tgzFileName is the name of tgz file from which file to be extracted.
* @param fileName is the name of the file to be extracted.
* @param strDestLoc is the location where file will be extracted.
- * @return
+ * @param bFirstDirLevel to indicate to get file from first directory level.
+ * @return Sucess or Failure
*/
- public static void extractFileFromTGZ (String tgzFileName, String fileName, String strDestLoc) {
+ public static boolean extractFileFromTGZ (String tgzFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) {
+
+ boolean bRetCode = bFAILURE;
+
TarArchiveInputStream tarIn = null;
try {
@@ -359,8 +529,8 @@ public class ValidateLicAndNotice
new BufferedInputStream(
new FileInputStream(tgzFileName))));
} catch(Exception e) {
- System.out.println("Exception in unzipping tar file: " + e);
- return;
+ debugPrint(DEBUG_ERROR, "Exception in unzipping tar file: " + e);
+ return bRetCode;
}
try {
@@ -368,15 +538,13 @@ public class ValidateLicAndNotice
BufferedInputStream bufIn = null;
TarArchiveEntry tarEntry = null;
while((tarEntry = tarIn.getNextTarEntry()) != null) {
-// System.out.println("Entry Name: " + tarEntry);
if(! tarEntry.getName().endsWith(fileName))
continue;
//Get file at root (in single directory) level. This is for License in root location.
- if( tarEntry.getName().indexOf('/') != tarEntry.getName().lastIndexOf('/'))
+ if( bFirstDirLevel &&
+ (tarEntry.getName().indexOf('/') != tarEntry.getName().lastIndexOf('/')))
continue;
-// System.out.println("License found : " + tarEentry);
bufIn = new BufferedInputStream (tarIn);
-// (zipfile.getInputStream(tarEntry));
int count;
byte data[] = new byte[BUFFER];
String strOutFileName = strDestLoc == null ? tarEntry.getName(): strDestLoc + "/" + fileName;
@@ -388,14 +556,30 @@ public class ValidateLicAndNotice
bufOut.flush();
bufOut.close();
bufIn.close();
+ bRetCode = bSUCCESS;
break;
}
} catch(Exception e) {
e.printStackTrace();
}
+ return bRetCode;
}
/**
+ * This will return the list of files from zip/tgz file.
+ *
+ * @param zipFileName is the name of zip file from which list of files with specified file extension will be returned.
+ * @param fileExt is the file extension to be used to get list of files to be returned.
+ * @return Returns list of files having specified extention from zip file .
+ */
+ public static List<String> getFiles (String zipFileName, String fileExt) {
+ if (zipFileName.endsWith("." + ZIP))
+ return getFilesFromZip (zipFileName, fileExt);
+ else if (zipFileName.endsWith("." + TGZ))
+ return getFilesFromTGZ (zipFileName, fileExt);
+ return null;
+ }
+ /**
* This will return the list of files from zip file.
*
* @param zipFileName is the name of zip file from which list of files with specified file extension will be returned.
@@ -410,14 +594,12 @@ public class ValidateLicAndNotice
Enumeration e = zipfile.entries();
while(e.hasMoreElements()) {
entry = (ZipEntry) e.nextElement();
-// System.out.println("Entry Name: " + entry);
if(entry.getName().endsWith("." + fileExt)) {
int iPos = entry.getName().lastIndexOf("/");
if (iPos == 0)
--iPos;
String strFileName = entry.getName().substring(iPos+1);
files.add(strFileName);
-// System.out.println("File found : " + strFileName);
}
}
} catch(Exception e) {
@@ -444,7 +626,7 @@ public class ValidateLicAndNotice
new BufferedInputStream(
new FileInputStream(tgzFileName))));
} catch(Exception e) {
- System.out.println("Exception in unzipping tar file: " + e);
+ debugPrint(DEBUG_ERROR, "Exception in unzipping tar file: " + e);
return null;
}
@@ -452,14 +634,12 @@ public class ValidateLicAndNotice
try {
TarArchiveEntry tarEntry = null;
while((tarEntry = tarIn.getNextTarEntry()) != null) {
-// System.out.println("Entry Name: " + entry);
if(tarEntry.getName().endsWith("." + fileExt)) {
int iPos = tarEntry.getName().lastIndexOf("/");
if (iPos == 0)
--iPos;
String strFileName = tarEntry.getName().substring(iPos+1);
files.add(strFileName);
-// System.out.println("File found : " + strFileName);
}
}
} catch(Exception e) {
@@ -484,11 +664,44 @@ public class ValidateLicAndNotice
try {
int retCode = valLic.validate();
- System.out.println("Return code = " + retCode);
+ debugPrint(DEBUG_INFO, "Return code = " + retCode);
}
catch (Exception e) {
- System.out.println("Error while validating license in zip/tgz file." + e);
+ debugPrint(DEBUG_ERROR, "Error while validating license in zip/tgz file." + e);
}
}
+ /**
+ * This will be used to output on console in consistent and controlled based on DEBUG flag.
+ *
+ * @param debugLevel is the debuglevel message user wants to prrint message.
+ * @param message is the message to be displayed.
+ * @return
+ */
+ public static void debugPrint(int debugLevel, String message) {
+ String displayMessage = "";
+ switch (debugLevel) {
+ case DEBUG_ERROR:
+ displayMessage = "ERROR: " + message;
+ break;
+ case DEBUG_WARNING:
+ displayMessage = "WARNING: " + message;
+ break;
+ case DEBUG_INFO:
+ displayMessage = "INFO: " + message;
+ break;
+ case DEBUG_INFO2:
+ displayMessage = "INFO2: " + message;
+ break;
+ case DEBUG_CODE:
+ displayMessage = "DEBUG: " + message;
+ break;
+ default:
+ break;
+ }
+
+ if(debugLevel <= DEBUG_PRINT_LEVEL)
+ System.out.println(displayMessage);
+ }
+
}