You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2019/01/22 05:06:48 UTC
[kafka] branch trunk updated: KAFKA-7844: Use regular subproject
for generator to fix *All targets (#6182)
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new fb0db76 KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)
fb0db76 is described below
commit fb0db7602abfd54137677d582a7781e4f790d7f9
Author: Stanislav Kozlovski <st...@outlook.com>
AuthorDate: Tue Jan 22 05:06:37 2019 +0000
KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)
The presence of the buildSrc subproject is causing problems when we try
to run installAll, jarAll, and the other "all" targets. It's easier
just to make the generator code a regular subproject and use the
JavaExec gradle task to run the code. This also makes it more
straightforward to run the generator unit tests.
Reviewers: David Arthur <mu...@gmail.com>, Ismael Juma <is...@juma.me.uk>
Co-authored-by: Colin P. Mccabe <cm...@confluent.io>
Co-authored-by: Stanislav Kozlovski <st...@outlook.com>
---
build.gradle | 36 +++++++++---
buildSrc/build.gradle | 26 ---------
.../org/apache/kafka/task/ProcessMessagesTask.java | 68 ----------------------
.../kafka/message/ApiMessageFactoryGenerator.java | 0
.../java/org/apache/kafka/message/CodeBuffer.java | 0
.../java/org/apache/kafka/message/FieldSpec.java | 0
.../java/org/apache/kafka/message/FieldType.java | 0
.../org/apache/kafka/message/HeaderGenerator.java | 0
.../apache/kafka/message/MessageDataGenerator.java | 0
.../org/apache/kafka/message/MessageGenerator.java | 62 +++++++++-----------
.../java/org/apache/kafka/message/MessageSpec.java | 0
.../org/apache/kafka/message/MessageSpecType.java | 0
.../org/apache/kafka/message/SchemaGenerator.java | 0
.../java/org/apache/kafka/message/StructSpec.java | 0
.../java/org/apache/kafka/message/Versions.java | 0
.../apache/kafka/message/MessageGeneratorTest.java | 0
.../org/apache/kafka/message/VersionsTest.java | 0
settings.gradle | 2 +-
18 files changed, 57 insertions(+), 137 deletions(-)
diff --git a/build.gradle b/build.gradle
index 49955ca..58d554f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -249,10 +249,11 @@ subprojects {
exceptionFormat = testExceptionFormat
}
- useJUnit {
- excludeCategories 'org.apache.kafka.test.IntegrationTest'
+ if (it.project.name != 'generator') {
+ useJUnit {
+ excludeCategories 'org.apache.kafka.test.IntegrationTest'
+ }
}
-
}
jar {
@@ -823,6 +824,23 @@ project(':examples') {
}
}
+project(':generator') {
+ dependencies {
+ compile libs.jacksonDatabind
+ compile libs.jacksonJDK8Datatypes
+ compile libs.jacksonJaxrsJsonProvider
+ testCompile libs.junit
+ }
+
+ integrationTest {
+ enabled = false
+ }
+
+ javadoc {
+ enabled = false
+ }
+}
+
project(':clients') {
archivesBaseName = "kafka-clients"
@@ -899,9 +917,12 @@ project(':clients') {
delete "$buildDir/kafka/"
}
- task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) {
- inputDirectory = file("src/main/resources/common/message")
- outputDirectory = file("src/generated/java/org/apache/kafka/common/message")
+ task processMessages(type:JavaExec) {
+ main = "org.apache.kafka.message.MessageGenerator"
+ classpath = project(':generator').sourceSets.main.runtimeClasspath
+ args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
+ inputs.dir("src/main/resources/common/message")
+ outputs.dir("src/generated/java/org/apache/kafka/common/message")
}
sourceSets {
@@ -912,8 +933,7 @@ project(':clients') {
}
test {
java {
- srcDirs = ["src/generated/java", "src/test/java",
- "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"]
+ srcDirs = ["src/generated/java", "src/test/java"]
}
}
}
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
deleted file mode 100644
index 23a1fc4..0000000
--- a/buildSrc/build.gradle
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-repositories {
- mavenCentral()
-}
-
-dependencies {
- compile "com.fasterxml.jackson.core:jackson-databind:2.9.6"
- compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6"
- compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6"
-}
-
-test.enabled=false
diff --git a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java b/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java
deleted file mode 100644
index dd85a54..0000000
--- a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.task;
-
-import org.apache.kafka.message.MessageGenerator;
-import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputFiles;
-import org.gradle.api.tasks.TaskAction;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- * A gradle task which processes a directory full of JSON files into an output directory.
- */
-public class ProcessMessagesTask extends DefaultTask {
- /**
- * The directory where we should read the input JSON from.
- */
- public File inputDirectory;
-
- /**
- * The directory that we should write output JSON to.
- */
- public File outputDirectory;
-
- @InputDirectory
- public File getInputDirectory() {
- return inputDirectory;
- }
-
- /**
- * Define the task outputs.
- *
- * Gradle consults this to see if the task is up-to-date.
- */
- @OutputFiles
- public Map<String, File> getOutputFiles() throws Exception {
- return MessageGenerator.getOutputFiles(
- outputDirectory.toString(), inputDirectory.toString());
- }
-
- @TaskAction
- public void run() {
- try {
- MessageGenerator.processDirectories(
- outputDirectory.toString(), inputDirectory.toString());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java b/generator/src/main/java/org/apache/kafka/message/CodeBuffer.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java
rename to generator/src/main/java/org/apache/kafka/message/CodeBuffer.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java b/generator/src/main/java/org/apache/kafka/message/FieldSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java
rename to generator/src/main/java/org/apache/kafka/message/FieldSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java b/generator/src/main/java/org/apache/kafka/message/FieldType.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/FieldType.java
rename to generator/src/main/java/org/apache/kafka/message/FieldType.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
similarity index 76%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index 2abc704..74f9880 100644
--- a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import java.io.BufferedWriter;
-import java.io.File;
+import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
-import java.util.Map;
/**
* The Kafka message generator.
@@ -94,40 +92,30 @@ public final class MessageGenerator {
JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
}
- public static Map<String, File> getOutputFiles(String outputDir, String inputDir) throws Exception {
- HashMap<String, File> outputFiles = new HashMap<>();
- for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
- String jsonName = inputPath.getFileName().toString();
- String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java";
- File outputFile = new File(outputDir, javaName);
- outputFiles.put(outputFile.toString(), outputFile);
- }
- File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA);
- outputFiles.put(factoryFile.toString(), factoryFile);
- return outputFiles;
- }
-
public static void processDirectories(String outputDir, String inputDir) throws Exception {
Files.createDirectories(Paths.get(outputDir));
int numProcessed = 0;
ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator();
HashSet<String> outputFileNames = new HashSet<>();
- for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
- try {
- MessageSpec spec = JSON_SERDE.
- readValue(inputPath.toFile(), MessageSpec.class);
- String javaName = spec.generatedClassName() + JAVA_SUFFIX;
- outputFileNames.add(javaName);
- Path outputPath = Paths.get(outputDir, javaName);
- try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
- MessageDataGenerator generator = new MessageDataGenerator();
- generator.generate(spec);
- generator.write(writer);
+ try (DirectoryStream<Path> directoryStream = Files
+ .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+ for (Path inputPath : directoryStream) {
+ try {
+ MessageSpec spec = JSON_SERDE.
+ readValue(inputPath.toFile(), MessageSpec.class);
+ String javaName = spec.generatedClassName() + JAVA_SUFFIX;
+ outputFileNames.add(javaName);
+ Path outputPath = Paths.get(outputDir, javaName);
+ try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
+ MessageDataGenerator generator = new MessageDataGenerator();
+ generator.generate(spec);
+ generator.write(writer);
+ }
+ numProcessed++;
+ messageFactoryGenerator.registerMessageType(spec);
+ } catch (Exception e) {
+ throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
}
- numProcessed++;
- messageFactoryGenerator.registerMessageType(spec);
- } catch (Exception e) {
- throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
}
}
Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA);
@@ -137,9 +125,15 @@ public final class MessageGenerator {
messageFactoryGenerator.write(writer);
}
numProcessed++;
- for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) {
- if (!outputFileNames.contains(outputPath.getFileName().toString())) {
- Files.delete(outputPath);
+ try (DirectoryStream<Path> directoryStream = Files.
+ newDirectoryStream(Paths.get(outputDir))) {
+ for (Path outputPath : directoryStream) {
+ Path fileName = outputPath.getFileName();
+ if (fileName != null) {
+ if (!outputFileNames.contains(fileName.toString())) {
+ Files.delete(outputPath);
+ }
+ }
}
}
System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java
rename to generator/src/main/java/org/apache/kafka/message/MessageSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java b/generator/src/main/java/org/apache/kafka/message/MessageSpecType.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java
rename to generator/src/main/java/org/apache/kafka/message/MessageSpecType.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java
rename to generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/generator/src/main/java/org/apache/kafka/message/StructSpec.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java
rename to generator/src/main/java/org/apache/kafka/message/StructSpec.java
diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/generator/src/main/java/org/apache/kafka/message/Versions.java
similarity index 100%
rename from buildSrc/src/main/java/org/apache/kafka/message/Versions.java
rename to generator/src/main/java/org/apache/kafka/message/Versions.java
diff --git a/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
similarity index 100%
rename from clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
rename to generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
diff --git a/clients/src/test/java/org/apache/kafka/message/VersionsTest.java b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
similarity index 100%
rename from clients/src/test/java/org/apache/kafka/message/VersionsTest.java
rename to generator/src/test/java/org/apache/kafka/message/VersionsTest.java
diff --git a/settings.gradle b/settings.gradle
index a74df21..5ae78e4 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal
'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20',
'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
- 'connect:basic-auth-extension', 'jmh-benchmarks'
+ 'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'