You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/10 17:42:09 UTC
[04/21] hadoop git commit: HADOOP-12194. Support for incremental
generation in the protoc plugin.
HADOOP-12194. Support for incremental generation in the protoc plugin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/625d7ed9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/625d7ed9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/625d7ed9
Branch: refs/heads/HADOOP-12111
Commit: 625d7ed9eb65f0df204b506ce92c11803fbce273
Parents: fc6182d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Jul 8 11:09:43 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Jul 8 11:09:43 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-maven-plugins/pom.xml | 8 +
.../hadoop/maven/plugin/protoc/ProtocMojo.java | 188 +++++++++++++++++--
3 files changed, 185 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/625d7ed9/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8ab109d..6cc6b71 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12172. FsShell mkdir -p makes an unnecessary check for the existence
of the parent. (cnauroth)
+ HADOOP-12194. Support for incremental generation in the protoc plugin.
+ (wang)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
http://git-wip-us.apache.org/repos/asf/hadoop/blob/625d7ed9/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index b48b9ac..b39c22b 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -47,6 +47,14 @@
<version>${maven.plugin-tools.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/625d7ed9/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
index 465b713..b9be33e 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
@@ -22,11 +22,21 @@ import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
-
+import java.util.Map;
+import java.util.zip.CRC32;
@Mojo(name="protoc", defaultPhase = LifecyclePhase.GENERATE_SOURCES)
public class ProtocMojo extends AbstractMojo {
@@ -49,6 +59,118 @@ public class ProtocMojo extends AbstractMojo {
@Parameter(required=true)
private String protocVersion;
+ @Parameter(defaultValue =
+ "${project.build.directory}/hadoop-maven-plugins-protoc-checksums.json")
+ private String checksumPath;
+
+ /**
+ * Compares include and source file checksums against previously computed
+ * checksums stored in a json file in the build directory.
+ */
+ public class ChecksumComparator {
+
+ private final Map<String, Long> storedChecksums;
+ private final Map<String, Long> computedChecksums;
+
+ private final File checksumFile;
+
+ ChecksumComparator(String checksumPath) throws IOException {
+ checksumFile = new File(checksumPath);
+ // Read in the checksums
+ if (checksumFile.exists()) {
+ ObjectMapper mapper = new ObjectMapper();
+ storedChecksums = mapper
+ .readValue(checksumFile, new TypeReference<Map<String, Long>>() {
+ });
+ } else {
+ storedChecksums = new HashMap<>(0);
+ }
+ computedChecksums = new HashMap<>();
+ }
+
+ public boolean hasChanged(File file) throws IOException {
+ if (!file.exists()) {
+ throw new FileNotFoundException(
+ "Specified protoc include or source does not exist: " + file);
+ }
+ if (file.isDirectory()) {
+ return hasDirectoryChanged(file);
+ } else if (file.isFile()) {
+ return hasFileChanged(file);
+ } else {
+ throw new IOException("Not a file or directory: " + file);
+ }
+ }
+
+ private boolean hasDirectoryChanged(File directory) throws IOException {
+ File[] listing = directory.listFiles();
+ boolean changed = false;
+ // Do not exit early, since we need to compute and save checksums
+ // for each file within the directory.
+ for (File f : listing) {
+ if (f.isDirectory()) {
+ if (hasDirectoryChanged(f)) {
+ changed = true;
+ }
+ } else if (f.isFile()) {
+ if (hasFileChanged(f)) {
+ changed = true;
+ }
+ } else {
+ getLog().debug("Skipping entry that is not a file or directory: "
+ + f);
+ }
+ }
+ return changed;
+ }
+
+ private boolean hasFileChanged(File file) throws IOException {
+ long computedCsum = computeChecksum(file);
+
+ // Return if the generated csum matches the stored csum
+ Long storedCsum = storedChecksums.get(file.getCanonicalPath());
+ if (storedCsum == null || storedCsum.longValue() != computedCsum) {
+ // It has changed.
+ return true;
+ }
+ return false;
+ }
+
+ private long computeChecksum(File file) throws IOException {
+ // If we've already computed the csum, reuse the computed value
+ final String canonicalPath = file.getCanonicalPath();
+ if (computedChecksums.containsKey(canonicalPath)) {
+ return computedChecksums.get(canonicalPath);
+ }
+ // Compute the csum for the file
+ CRC32 crc = new CRC32();
+ byte[] buffer = new byte[1024*64];
+ try (BufferedInputStream in =
+ new BufferedInputStream(new FileInputStream(file))) {
+ while (true) {
+ int read = in.read(buffer);
+ if (read <= 0) {
+ break;
+ }
+ crc.update(buffer, 0, read);
+ }
+ }
+ // Save it in the generated map and return
+ final long computedCsum = crc.getValue();
+ computedChecksums.put(canonicalPath, computedCsum);
+ return crc.getValue();
+ }
+
+ public void writeChecksums() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ try (BufferedOutputStream out = new BufferedOutputStream(
+ new FileOutputStream(checksumFile))) {
+ mapper.writeValue(out, computedChecksums);
+ getLog().info("Wrote protoc checksums to file " + checksumFile);
+ }
+ }
+ }
+
public void execute() throws MojoExecutionException {
try {
List<String> command = new ArrayList<String>();
@@ -58,7 +180,7 @@ public class ProtocMojo extends AbstractMojo {
List<String> out = new ArrayList<String>();
if (exec.run(command, out) == 127) {
getLog().error("protoc, not found at: " + protocCommand);
- throw new MojoExecutionException("protoc failure");
+ throw new MojoExecutionException("protoc failure");
} else {
if (out.isEmpty()) {
getLog().error("stdout: " + out);
@@ -67,36 +189,74 @@ public class ProtocMojo extends AbstractMojo {
} else {
if (!out.get(0).endsWith(protocVersion)) {
throw new MojoExecutionException(
- "protoc version is '" + out.get(0) + "', expected version is '"
- + protocVersion + "'");
+ "protoc version is '" + out.get(0) + "', expected version is '"
+ + protocVersion + "'");
}
}
}
if (!output.mkdirs()) {
if (!output.exists()) {
- throw new MojoExecutionException("Could not create directory: " +
- output);
+ throw new MojoExecutionException(
+ "Could not create directory: " + output);
}
}
+
+ // Whether the import or source protoc files have changed.
+ ChecksumComparator comparator = new ChecksumComparator(checksumPath);
+ boolean importsChanged = false;
+
command = new ArrayList<String>();
command.add(protocCommand);
command.add("--java_out=" + output.getCanonicalPath());
if (imports != null) {
for (File i : imports) {
+ if (comparator.hasChanged(i)) {
+ importsChanged = true;
+ }
command.add("-I" + i.getCanonicalPath());
}
}
+ // Filter to generate classes for just the changed source files.
+ List<File> changedSources = new ArrayList<>();
+ boolean sourcesChanged = false;
for (File f : FileSetUtils.convertFileSetToFiles(source)) {
- command.add(f.getCanonicalPath());
+ // Need to recompile if the source has changed, or if any import has
+ // changed.
+ if (comparator.hasChanged(f) || importsChanged) {
+ sourcesChanged = true;
+ changedSources.add(f);
+ command.add(f.getCanonicalPath());
+ }
}
- exec = new Exec(this);
- out = new ArrayList<String>();
- if (exec.run(command, out) != 0) {
- getLog().error("protoc compiler error");
- for (String s : out) {
- getLog().error(s);
+
+ if (!sourcesChanged && !importsChanged) {
+ getLog().info("No changes detected in protoc files, skipping "
+ + "generation.");
+ } else {
+ if (getLog().isDebugEnabled()) {
+ StringBuilder b = new StringBuilder();
+ b.append("Generating classes for the following protoc files: [");
+ String prefix = "";
+ for (File f : changedSources) {
+ b.append(prefix);
+ b.append(f.toString());
+ prefix = ", ";
+ }
+ b.append("]");
+ getLog().debug(b.toString());
}
- throw new MojoExecutionException("protoc failure");
+
+ exec = new Exec(this);
+ out = new ArrayList<String>();
+ if (exec.run(command, out) != 0) {
+ getLog().error("protoc compiler error");
+ for (String s : out) {
+ getLog().error(s);
+ }
+ throw new MojoExecutionException("protoc failure");
+ }
+ // Write the new checksum file on success.
+ comparator.writeChecksums();
}
} catch (Throwable ex) {
throw new MojoExecutionException(ex.toString(), ex);