You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2019/01/22 05:06:48 UTC

[kafka] branch trunk updated: KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)

This is an automated email from the ASF dual-hosted git repository.

ijuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new fb0db76  KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)
fb0db76 is described below

commit fb0db7602abfd54137677d582a7781e4f790d7f9
Author: Stanislav Kozlovski <st...@outlook.com>
AuthorDate: Tue Jan 22 05:06:37 2019 +0000

    KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)
    
    The presence of the buildSrc subproject is causing problems when we try
    to run installAll, jarAll, and the other "all" targets. It's easier
    just to make the generator code a regular subproject and use the
    JavaExec gradle task to run the code. This also makes it more
    straightforward to run the generator unit tests.
    
    Reviewers: David Arthur <mu...@gmail.com>, Ismael Juma <is...@juma.me.uk>
    
    Co-authored-by: Colin P. Mccabe <cm...@confluent.io>
    Co-authored-by: Stanislav Kozlovski <st...@outlook.com>
---
 build.gradle                                       | 36 +++++++++---
 buildSrc/build.gradle                              | 26 ---------
 .../org/apache/kafka/task/ProcessMessagesTask.java | 68 ----------------------
 .../kafka/message/ApiMessageFactoryGenerator.java  |  0
 .../java/org/apache/kafka/message/CodeBuffer.java  |  0
 .../java/org/apache/kafka/message/FieldSpec.java   |  0
 .../java/org/apache/kafka/message/FieldType.java   |  0
 .../org/apache/kafka/message/HeaderGenerator.java  |  0
 .../apache/kafka/message/MessageDataGenerator.java |  0
 .../org/apache/kafka/message/MessageGenerator.java | 62 +++++++++-----------
 .../java/org/apache/kafka/message/MessageSpec.java |  0
 .../org/apache/kafka/message/MessageSpecType.java  |  0
 .../org/apache/kafka/message/SchemaGenerator.java  |  0
 .../java/org/apache/kafka/message/StructSpec.java  |  0
 .../java/org/apache/kafka/message/Versions.java    |  0
 .../apache/kafka/message/MessageGeneratorTest.java |  0
 .../org/apache/kafka/message/VersionsTest.java     |  0
 settings.gradle                                    |  2 +-
 18 files changed, 57 insertions(+), 137 deletions(-)

diff --git a/build.gradle b/build.gradle
index 49955ca..58d554f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -249,10 +249,11 @@ subprojects {
       exceptionFormat = testExceptionFormat
     }
 
-    useJUnit {
-      excludeCategories 'org.apache.kafka.test.IntegrationTest'
+    if (it.project.name != 'generator') {
+      useJUnit {
+        excludeCategories 'org.apache.kafka.test.IntegrationTest'
+      }
     }
-    
   }
 
   jar {
@@ -823,6 +824,23 @@ project(':examples') {
   }
 }
 
+project(':generator') {
+  dependencies {
+    compile libs.jacksonDatabind
+    compile libs.jacksonJDK8Datatypes
+    compile libs.jacksonJaxrsJsonProvider
+    testCompile libs.junit
+  }
+
+  integrationTest {
+    enabled = false
+  }
+
+  javadoc {
+    enabled = false
+  }
+}
+
 project(':clients') {
   archivesBaseName = "kafka-clients"
 
@@ -899,9 +917,12 @@ project(':clients') {
     delete "$buildDir/kafka/"
   }
 
-  task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) {
-    inputDirectory = file("src/main/resources/common/message")
-    outputDirectory = file("src/generated/java/org/apache/kafka/common/message")
+  task processMessages(type:JavaExec) {
+    main = "org.apache.kafka.message.MessageGenerator"
+    classpath = project(':generator').sourceSets.main.runtimeClasspath
+    args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
+    inputs.dir("src/main/resources/common/message")
+    outputs.dir("src/generated/java/org/apache/kafka/common/message")
   }
 
   sourceSets {
@@ -912,8 +933,7 @@ project(':clients') {
     }
     test {
       java {
-        srcDirs = ["src/generated/java", "src/test/java",
-            "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"]
+        srcDirs = ["src/generated/java", "src/test/java"]
       }
     }
   }
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
deleted file mode 100644
index 23a1fc4..0000000
--- a/buildSrc/build.gradle
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-repositories {
-  mavenCentral()
-}
-
-dependencies {
-  compile "com.fasterxml.jackson.core:jackson-databind:2.9.6"
-  compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6"
-  compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6"
-}
-
-test.enabled=false
diff --git a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java b/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java
deleted file mode 100644
index dd85a54..0000000
--- a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.task;
-
-import org.apache.kafka.message.MessageGenerator;
-import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputFiles;
-import org.gradle.api.tasks.TaskAction;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- * A gradle task which processes a directory full of JSON files into an output directory.
- */
-public class ProcessMessagesTask extends DefaultTask {
-    /**
-     * The directory where we should read the input JSON from.
-     */
-    public File inputDirectory;
-
-    /**
-     * The directory that we should write output JSON to.
-     */
-    public File outputDirectory;
-
-    @InputDirectory
-    public File getInputDirectory() {
-        return inputDirectory;
-    }
-
-    /**
-     * Define the task outputs.
-     *
-     * Gradle consults this to see if the task is up-to-date.
-     */
-    @OutputFiles
-    public Map<String, File> getOutputFiles() throws Exception {
-        return MessageGenerator.getOutputFiles(
-            outputDirectory.toString(), inputDirectory.toString());
-    }
-
-    @TaskAction
-    public void run() {
-        try {
-            MessageGenerator.processDirectories(
-                outputDirectory.toString(), inputDirectory.toString());
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java b/generator/src/main/java/org/apache/kafka/message/CodeBuffer.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java
rename to generator/src/main/java/org/apache/kafka/message/CodeBuffer.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java b/generator/src/main/java/org/apache/kafka/message/FieldSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java
rename to generator/src/main/java/org/apache/kafka/message/FieldSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java b/generator/src/main/java/org/apache/kafka/message/FieldType.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/FieldType.java
rename to generator/src/main/java/org/apache/kafka/message/FieldType.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
similarity index 76%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index 2abc704..74f9880 100644
--- a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 
 import java.io.BufferedWriter;
-import java.io.File;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
-import java.util.Map;
 
 /**
  * The Kafka message generator.
@@ -94,40 +92,30 @@ public final class MessageGenerator {
         JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
     }
 
-    public static Map<String, File> getOutputFiles(String outputDir, String inputDir) throws Exception {
-        HashMap<String, File> outputFiles = new HashMap<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
-            String jsonName = inputPath.getFileName().toString();
-            String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java";
-            File outputFile = new File(outputDir, javaName);
-            outputFiles.put(outputFile.toString(), outputFile);
-        }
-        File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA);
-        outputFiles.put(factoryFile.toString(), factoryFile);
-        return outputFiles;
-    }
-
     public static void processDirectories(String outputDir, String inputDir) throws Exception {
         Files.createDirectories(Paths.get(outputDir));
         int numProcessed = 0;
         ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator();
         HashSet<String> outputFileNames = new HashSet<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
-            try {
-                MessageSpec spec = JSON_SERDE.
-                    readValue(inputPath.toFile(), MessageSpec.class);
-                String javaName = spec.generatedClassName() + JAVA_SUFFIX;
-                outputFileNames.add(javaName);
-                Path outputPath = Paths.get(outputDir, javaName);
-                try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
-                    MessageDataGenerator generator = new MessageDataGenerator();
-                    generator.generate(spec);
-                    generator.write(writer);
+        try (DirectoryStream<Path> directoryStream = Files
+                .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+            for (Path inputPath : directoryStream) {
+                try {
+                    MessageSpec spec = JSON_SERDE.
+                        readValue(inputPath.toFile(), MessageSpec.class);
+                    String javaName = spec.generatedClassName() + JAVA_SUFFIX;
+                    outputFileNames.add(javaName);
+                    Path outputPath = Paths.get(outputDir, javaName);
+                    try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
+                        MessageDataGenerator generator = new MessageDataGenerator();
+                        generator.generate(spec);
+                        generator.write(writer);
+                    }
+                    numProcessed++;
+                    messageFactoryGenerator.registerMessageType(spec);
+                } catch (Exception e) {
+                    throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
                 }
-                numProcessed++;
-                messageFactoryGenerator.registerMessageType(spec);
-            } catch (Exception e) {
-                throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
             }
         }
         Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA);
@@ -137,9 +125,15 @@ public final class MessageGenerator {
             messageFactoryGenerator.write(writer);
         }
         numProcessed++;
-        for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) {
-            if (!outputFileNames.contains(outputPath.getFileName().toString())) {
-                Files.delete(outputPath);
+        try (DirectoryStream<Path> directoryStream = Files.
+                newDirectoryStream(Paths.get(outputDir))) {
+            for (Path outputPath : directoryStream) {
+                Path fileName = outputPath.getFileName();
+                if (fileName != null) {
+                    if (!outputFileNames.contains(fileName.toString())) {
+                        Files.delete(outputPath);
+                    }
+                }
             }
         }
         System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java
rename to generator/src/main/java/org/apache/kafka/message/MessageSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java b/generator/src/main/java/org/apache/kafka/message/MessageSpecType.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java
rename to generator/src/main/java/org/apache/kafka/message/MessageSpecType.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/generator/src/main/java/org/apache/kafka/message/StructSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java
rename to generator/src/main/java/org/apache/kafka/message/StructSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/generator/src/main/java/org/apache/kafka/message/Versions.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/Versions.java
rename to generator/src/main/java/org/apache/kafka/message/Versions.java
diff --git a/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
similarity index 100%
rename from clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
rename to generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
diff --git a/clients/src/test/java/org/apache/kafka/message/VersionsTest.java b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
similarity index 100%
rename from clients/src/test/java/org/apache/kafka/message/VersionsTest.java
rename to generator/src/test/java/org/apache/kafka/message/VersionsTest.java
diff --git a/settings.gradle b/settings.gradle
index a74df21..5ae78e4 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal
         'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
         'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20',
         'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
-        'connect:basic-auth-extension', 'jmh-benchmarks'
+        'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'