You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2019/01/29 22:15:47 UTC
[kudu] branch master updated: KUDU-2411: Add OS/Arch detection to
binary extract
This is an automated email from the ASF dual-hosted git repository.
mpercy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new ebe6265 KUDU-2411: Add OS/Arch detection to binary extract
ebe6265 is described below
commit ebe62653101e670a4d4c8358ec9fa7a1824c4420
Author: Brian McDevitt <br...@phdata.io>
AuthorDate: Mon Dec 31 11:57:43 2018 -0600
KUDU-2411: Add OS/Arch detection to binary extract
OS/Arch detection ensures that the runtime has the appropriate Kudu
binaries to execute the MiniKuduCluster. Currently this limited to
OS: osx|linux and Arch: x86_64.
Change-Id: I9752914a426dd1572610f891dad4d4778d04f79c
Reviewed-on: http://gerrit.cloudera.org:8080/12141
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>
---
java/gradle/dependencies.gradle | 2 ++
java/kudu-test-utils/build.gradle | 1 +
.../kudu/test/cluster/KuduBinaryJarExtractor.java | 11 +++---
.../kudu/test/cluster/KuduBinaryLocator.java | 17 +++++++--
.../test/cluster/TestKuduBinaryJarExtractor.java | 42 ++++++++++++++++++----
.../META-INF/apache-kudu-test-binary.properties | 18 ----------
.../src/test/resources/log4j.properties | 3 ++
7 files changed, 62 insertions(+), 32 deletions(-)
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 19d8e45..eafd537 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -48,6 +48,7 @@ versions += [
mockito : "2.23.4",
murmur : "1.0.0",
netty : "3.10.6.Final",
+ osdetector : "1.6.1",
parquet : "1.10.0",
pmd : "5.8.1",
protobuf : "3.6.1",
@@ -95,6 +96,7 @@ libs += [
mockitoCore : "org.mockito:mockito-core:$versions.mockito",
murmur : "com.sangupta:murmur:$versions.murmur",
netty : "io.netty:netty:$versions.netty",
+ osdetector : "com.google.gradle:osdetector-gradle-plugin:$versions.osdetector",
parquetHadoop : "org.apache.parquet:parquet-hadoop:$versions.parquet",
protobufJava : "com.google.protobuf:protobuf-java:$versions.protobuf",
protobufJavaUtil : "com.google.protobuf:protobuf-java-util:$versions.protobuf",
diff --git a/java/kudu-test-utils/build.gradle b/java/kudu-test-utils/build.gradle
index 934f86d..ebe5c24 100644
--- a/java/kudu-test-utils/build.gradle
+++ b/java/kudu-test-utils/build.gradle
@@ -20,6 +20,7 @@ apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
compile project(path: ":kudu-client")
compile libs.guava
+ compile libs.osdetector
compileUnshaded libs.junit
compileUnshaded libs.slf4jApi
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
index 006bfbd..bb5f84a 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryJarExtractor.java
@@ -17,6 +17,7 @@
package org.apache.kudu.test.cluster;
+import com.google.gradle.osdetector.OsDetector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +49,7 @@ public class KuduBinaryJarExtractor {
private static final Logger LOG = LoggerFactory.getLogger(KuduBinaryJarExtractor.class);
private static final String KUDU_TEST_BIN_PROPS_PATH =
"META-INF/apache-kudu-test-binary.properties";
+ private static final OsDetector DETECTOR = new OsDetector();
public KuduBinaryJarExtractor() {}
@@ -60,14 +62,13 @@ public class KuduBinaryJarExtractor {
private static Properties getBinaryProps() throws IOException {
Enumeration<URL> resources = getCurrentClassLoader().getResources(KUDU_TEST_BIN_PROPS_PATH);
- //TODO: normalize osName
- //TODO: check for matching architecture
while (resources.hasMoreElements()) {
URL url = resources.nextElement();
try {
- Properties binaryProps = loadBinaryProps(url);
- if (binaryProps != null && !binaryProps.isEmpty()) {
- return binaryProps;
+ Properties props = loadBinaryProps(url);
+ if (DETECTOR.getOs().equals(props.getProperty("artifact.os")) &&
+ DETECTOR.getArch().equals(props.getProperty("artifact.arch"))) {
+ return props;
}
} catch (IOException ex) {
LOG.warn("Unable to parse properties file from Kudu binary artifact", ex);
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
index 2f02fad..d1102a2 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/cluster/KuduBinaryLocator.java
@@ -18,6 +18,7 @@
package org.apache.kudu.test.cluster;
import com.google.common.io.CharStreams;
+import org.apache.kudu.test.TempDirUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
@@ -55,6 +56,17 @@ public class KuduBinaryLocator {
return kuduBinDirProp;
}
+ try {
+ KuduBinaryJarExtractor extractor = new KuduBinaryJarExtractor();
+ if (extractor.isKuduBinaryJarOnClasspath()) {
+ String testTmpDir = TempDirUtils.getTempDirectory("kudu-binary-jar").toString();
+ LOG.info("Using Kudu binary jar directory: {}", testTmpDir);
+ return extractor.extractKuduBinary(testTmpDir);
+ }
+ } catch (IOException ex) {
+ LOG.warn("Unable to extract a Kudu binary jar", ex);
+ }
+
// If the `kudu` binary is found on the PATH using `which kudu`, use its parent directory.
try {
Runtime runtime = Runtime.getRuntime();
@@ -72,9 +84,8 @@ public class KuduBinaryLocator {
throw new RuntimeException("Error while locating kudu binary", ex);
}
- throw new RuntimeException("Could not locate the kudu binary directory. " +
- "Set the system variable " + KUDU_BIN_DIR_PROP +
- " or ensure the `kudu` binary is on your path.");
+ throw new RuntimeException("Set the system variable " + KUDU_BIN_DIR_PROP + " or add the Kudu" +
+ " binary test jar to your classpath or ensure the `kudu` binary is on your path.");
}
/**
diff --git a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
index 91f623b..696eabe 100644
--- a/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
+++ b/java/kudu-test-utils/src/test/java/org/apache/kudu/test/cluster/TestKuduBinaryJarExtractor.java
@@ -22,6 +22,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -37,6 +38,7 @@ import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -46,11 +48,12 @@ public class TestKuduBinaryJarExtractor {
private static final Logger LOG = LoggerFactory.getLogger(TestKuduBinaryJarExtractor.class);
- private Path createKuduBinaryJar() throws IOException, URISyntaxException {
- Path tempDir = Files.createTempDirectory("fake-kudu-binary-jar");
+ private Path createKuduBinaryJar(final String os) throws IOException, URISyntaxException {
+ String baseName = "fake-" + os + "-kudu-binary";
+ Path tempDir = Files.createTempDirectory(baseName);
// convert the filename to a URI
- final Path path = Paths.get(tempDir.toString(), "fake-kudu-binary.jar");
+ final Path path = Paths.get(tempDir.toString(), baseName + ".jar");
LOG.info("Creating fake kudu binary jar at {}", path.toString());
final URI uri = URI.create("jar:file:" + path.toUri().getPath());
@@ -78,16 +81,35 @@ public class TestKuduBinaryJarExtractor {
final Path dirToCreate = zipFs.getPath(root.toString(),
src.relativize(dir).toString());
if (Files.notExists(dirToCreate)) {
- System.out.printf("Creating directory %s\n", dirToCreate);
+ LOG.debug("Creating directory {}", dirToCreate);
Files.createDirectories(dirToCreate);
}
return FileVisitResult.CONTINUE;
}
});
+
+ Path metaInf = zipFs.getPath(root.toString(), "META-INF");
+ Files.createDirectory(metaInf);
+ // Customize the properties file to enable positive and negative test scenarios.
+ Path propsPath = zipFs.getPath(metaInf.toString(), "apache-kudu-test-binary.properties");
+ OutputStream propsOutputStream = Files.newOutputStream(propsPath);
+ writeProperties(os, propsOutputStream);
+ propsOutputStream.close();
+
zipFs.close();
return path;
}
+ private static void writeProperties(String os, OutputStream out) throws IOException {
+ Properties properties = new Properties();
+ properties.setProperty("format.version", "1");
+ properties.setProperty("artifact.version", "1.9.0-SNAPSHOT");
+ properties.setProperty("artifact.prefix", "apache-kudu-1.9.0-SNAHSHOT");
+ properties.setProperty("artifact.os", os);
+ properties.setProperty("artifact.arch", "x86_64");
+ properties.store(out, "test");
+ }
+
/**
* Create a ClassLoader. The parent of the ClassLoader will be the current thread context
* ClassLoader, if not set, or the ClassLoader that loaded this test class if not.
@@ -104,7 +126,7 @@ public class TestKuduBinaryJarExtractor {
@Test
public void testExtractJar() throws IOException, URISyntaxException {
- Path binaryJar = createKuduBinaryJar();
+ Path binaryJar = createKuduBinaryJar("osx");
Path extractedBinDir =
KuduBinaryJarExtractor.extractJar(binaryJar,
@@ -120,9 +142,17 @@ public class TestKuduBinaryJarExtractor {
public void testIsKuduBinaryJarOnClasspath() throws IOException, URISyntaxException {
KuduBinaryJarExtractor extractor = new KuduBinaryJarExtractor();
assertFalse(extractor.isKuduBinaryJarOnClasspath());
- Path binaryJar = createKuduBinaryJar();
+
+ boolean isOsX = System.getProperty("os.name").replaceAll("\\s", "").equalsIgnoreCase("macosx");
+
+ Path binaryJar = createKuduBinaryJar(isOsX ? "linux" : "osx");
ClassLoader childLoader = createChildClassLoader(new URL[] { binaryJar.toUri().toURL() });
Thread.currentThread().setContextClassLoader(childLoader);
+ assertFalse(extractor.isKuduBinaryJarOnClasspath());
+
+ binaryJar = createKuduBinaryJar(!isOsX ? "linux" : "osx");
+ childLoader = createChildClassLoader(new URL[] { binaryJar.toUri().toURL() });
+ Thread.currentThread().setContextClassLoader(childLoader);
assertTrue(extractor.isKuduBinaryJarOnClasspath());
}
}
diff --git a/java/kudu-test-utils/src/test/resources/fake-kudu-binary/META-INF/apache-kudu-test-binary.properties b/java/kudu-test-utils/src/test/resources/fake-kudu-binary/META-INF/apache-kudu-test-binary.properties
deleted file mode 100644
index a8efb44..0000000
--- a/java/kudu-test-utils/src/test/resources/fake-kudu-binary/META-INF/apache-kudu-test-binary.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-artifact.prefix=apache-kudu-1.9.0-SNAPSHOT
diff --git a/java/kudu-test-utils/src/test/resources/log4j.properties b/java/kudu-test-utils/src/test/resources/log4j.properties
index 52b9b75..8103224 100644
--- a/java/kudu-test-utils/src/test/resources/log4j.properties
+++ b/java/kudu-test-utils/src/test/resources/log4j.properties
@@ -21,3 +21,6 @@ log4j.appender.out.layout = org.apache.log4j.PatternLayout
log4j.appender.out.layout.ConversionPattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
log4j.logger.org.apache.kudu = INFO
+
+# This will log the detected operating system and CPU architecture.
+log4j.logger.com.google.gradle = INFO