You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/01/12 17:39:38 UTC
[1/3] incubator-slider git commit: SLIDER-749 jenkins on windows
failing: add some tests for file read/write
Repository: incubator-slider
Updated Branches:
refs/heads/develop 4c9268b9a -> 2ad763cfb
SLIDER-749 jenkins on windows failing: add some tests for file read/write
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9c0793d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9c0793d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9c0793d9
Branch: refs/heads/develop
Commit: 9c0793d9c232c3e1d52be4bb914a619c54901e22
Parents: 4c9268b
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 12 16:36:17 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jan 12 16:38:49 2015 +0000
----------------------------------------------------------------------
.../slider/other/TestLocalDirStatus.groovy | 87 ++++++++++++++
.../org/apache/slider/tools/TestUtility.java | 114 ++++++++++++++++++-
2 files changed, 199 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c0793d9/slider-core/src/test/groovy/org/apache/slider/other/TestLocalDirStatus.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/other/TestLocalDirStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/other/TestLocalDirStatus.groovy
new file mode 100644
index 0000000..6d4f15d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/other/TestLocalDirStatus.groovy
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.other
+
+import groovy.transform.CompileStatic
+import org.apache.slider.test.SliderTestUtils
+import org.apache.slider.tools.TestUtility
+import org.junit.Test
+
+/**
+ * This test exists to diagnose local FS permissions
+ */
+@CompileStatic
+class TestLocalDirStatus extends SliderTestUtils {
+
+
+ public static final int SIZE = 0x100000
+
+ @Test
+ public void testTempDir() throws Throwable {
+ File tmpf = File.createTempFile("testl",".bin")
+ createAndReadFile(tmpf, SIZE)
+ tmpf.delete()
+ assert !tmpf.exists()
+ }
+
+ @Test
+ public void testTargetDir() throws Throwable {
+ File target = new File("target").absoluteFile
+ assert target.exists()
+ File tmpf = File.createTempFile("testl", ".bin", target)
+ createAndReadFile(tmpf, SIZE)
+ tmpf.delete()
+ assert !tmpf.exists()
+ }
+
+ protected void createAndReadFile(File path, int len) {
+ byte[] dataset = TestUtility.dataset(len, 32, 128)
+ writeFile(path, dataset)
+ assert path.exists()
+ assert path.length() == len
+ def persisted = readFile(path)
+ TestUtility.compareByteArrays(dataset, persisted, len)
+ }
+
+ protected void writeFile(File path, byte[] dataset) {
+ def out = new FileOutputStream(path)
+ try {
+ out.write(dataset)
+ out.flush()
+ } finally {
+ out.close()
+ }
+ }
+
+ protected byte[] readFile(File path) {
+ assert path.absoluteFile.exists()
+ assert path.absoluteFile.isFile()
+ int len = (int)path.length()
+ byte[] dataset = new byte[len]
+ def ins = new FileInputStream(path)
+ try {
+ ins.read(dataset)
+ } finally {
+ ins.close()
+ }
+ return dataset
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c0793d9/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java b/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
index a8b14ac..0717ca1 100644
--- a/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
+++ b/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
@@ -19,6 +19,7 @@ package org.apache.slider.tools;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,7 +29,11 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-/** Various utility methods */
+/**
+ * Various utility methods
+ * Byte comparison methods are from
+ * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code>
+ */
public class TestUtility {
protected static final Logger log =
LoggerFactory.getLogger(TestUtility.class);
@@ -49,7 +54,7 @@ public class TestUtility {
public static void zipDir(String zipFile, String dir) throws IOException {
File dirObj = new File(dir);
ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
- log.info("Creating : " + zipFile);
+ log.info("Creating : {}", zipFile);
try {
addDir(dirObj, out, "");
} finally {
@@ -68,4 +73,109 @@ public class TestUtility {
return zipFileName;
}
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ Assert.assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ log.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ log.warn(line);
+ }
+ Assert.fail(message);
+ }
+ }
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
}
[3/3] incubator-slider git commit: Merge branch
'feature/SLIDER-749_jenkins_on_windows_failing' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-749_jenkins_on_windows_failing' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2ad763cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2ad763cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2ad763cf
Branch: refs/heads/develop
Commit: 2ad763cfb890bfd992006850fc423ae0d75f1f41
Parents: 4c9268b 4cec507
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 12 16:39:29 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jan 12 16:39:29 2015 +0000
----------------------------------------------------------------------
.../services/security/CertificateManager.java | 25 ++--
.../slider/other/TestLocalDirStatus.groovy | 87 ++++++++++++++
.../web/rest/agent/TestAMAgentWebServices.java | 37 +++---
.../security/TestCertificateManager.java | 4 +-
.../org/apache/slider/tools/TestUtility.java | 114 ++++++++++++++++++-
5 files changed, 232 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-slider git commit: SLIDER-750
CertificateManager.generateServerCertificate to throw exceptions
Posted by st...@apache.org.
SLIDER-750 CertificateManager.generateServerCertificate to throw exceptions
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4cec507d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4cec507d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4cec507d
Branch: refs/heads/develop
Commit: 4cec507d4b86258723a3b60cd1bde60fcaede935
Parents: 9c0793d
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jan 12 16:39:15 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jan 12 16:39:15 2015 +0000
----------------------------------------------------------------------
.../services/security/CertificateManager.java | 25 ++++++-------
.../web/rest/agent/TestAMAgentWebServices.java | 37 +++++++++++---------
.../security/TestCertificateManager.java | 4 +--
3 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4cec507d/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 257f8f9..71ae8d3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -63,7 +63,7 @@ public class CertificateManager {
/**
* Verify that root certificate exists, generate it otherwise.
*/
- public void initialize(MapOperations compOperations) {
+ public void initialize(MapOperations compOperations) throws SliderException {
SecurityUtils.initializeSecurityParameters(compOperations);
LOG.info("Initialization of root certificate");
@@ -195,7 +195,7 @@ public class CertificateManager {
}
}
- private void generateServerCertificate(){
+ private void generateServerCertificate() throws SliderException {
LOG.info("Generation of server certificate");
String srvrKstrDir = SecurityUtils.getSecurityDir();
@@ -208,22 +208,17 @@ public class CertificateManager {
Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName};
- try {
- String command = MessageFormat.format(GEN_SRVR_KEY,scriptArgs);
- runCommand(command);
-
- command = MessageFormat.format(GEN_SRVR_REQ,scriptArgs);
- runCommand(command);
+ String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs);
+ runCommand(command);
- command = MessageFormat.format(SIGN_SRVR_CRT,scriptArgs);
- runCommand(command);
+ command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs);
+ runCommand(command);
- command = MessageFormat.format(EXPRT_KSTR,scriptArgs);
- runCommand(command);
- } catch (SliderException e) {
- LOG.error("Error generating the server certificate", e);
- }
+ command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs);
+ runCommand(command);
+ command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
+ runCommand(command);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4cec507d/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index f2a7569..12ca949 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
@@ -49,6 +50,7 @@ import org.apache.slider.server.services.security.SecurityUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,17 +58,13 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.io.File;
import java.net.URI;
-//import java.nio.file.FileVisitResult;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
-//import java.nio.file.SimpleFileVisitor;
-//import java.nio.file.attribute.BasicFileAttributes;
import static org.junit.Assert.assertEquals;
public class TestAMAgentWebServices {
+ static CertificateManager certificateManager;
+
static {
//for localhost testing only
javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
@@ -81,15 +79,7 @@ public class TestAMAgentWebServices {
}
});
- MapOperations configMap = new MapOperations();
- SecurityUtils.initializeSecurityParameters(configMap, true);
- CertificateManager certificateManager = new CertificateManager();
- certificateManager.initialize(configMap);
- String keystoreFile = SecurityUtils.getSecurityDir() + File.separator + SliderKeys.KEYSTORE_FILE_NAME;
- String password = SecurityUtils.getKeystorePass();
- System.setProperty("javax.net.ssl.trustStore", keystoreFile);
- System.setProperty("javax.net.ssl.trustStorePassword", password);
- System.setProperty("javax.net.ssl.trustStoreType", "PKCS12");
+
}
protected static final Logger log =
@@ -108,6 +98,21 @@ public class TestAMAgentWebServices {
private AgentWebApp webApp;
private String base_url;
+ @BeforeClass
+ public static void setupClass() throws SliderException {
+ MapOperations configMap = new MapOperations();
+ SecurityUtils.initializeSecurityParameters(configMap, true);
+ certificateManager = new CertificateManager();
+ certificateManager.initialize(configMap);
+ String keystoreFile = SecurityUtils.getSecurityDir() + File.separator +
+ SliderKeys.KEYSTORE_FILE_NAME;
+ String password = SecurityUtils.getKeystorePass();
+ System.setProperty("javax.net.ssl.trustStore", keystoreFile);
+ System.setProperty("javax.net.ssl.trustStorePassword", password);
+ System.setProperty("javax.net.ssl.trustStoreType", "PKCS12");
+ }
+
+
@Before
public void setUp() throws Exception {
YarnConfiguration conf = SliderUtils.createConfiguration();
@@ -133,7 +138,7 @@ public class TestAMAgentWebServices {
historyPath,
null, null, new SimpleReleaseSelector());
} catch (Exception e) {
- log.error("Failed to set up app {}", e);
+ log.error("Failed to set up app {}", e, e);
}
ProviderAppState providerAppState = new ProviderAppState("undefined",
appState);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4cec507d/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
index 97a3f74..816a34b 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
@@ -49,13 +49,13 @@ public class TestCertificateManager {
@Test
public void testServerCertificateGenerated() throws Exception {
File serverCrt = new File(secDir, SliderKeys.CRT_FILE_NAME);
- Assert.assertTrue(serverCrt.exists());
+ Assert.assertTrue("Server CRD does not exist:" + serverCrt,serverCrt.exists());
}
@Test
public void testKeystoreGenerated() throws Exception {
File keystore = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- Assert.assertTrue(keystore.exists());
+ Assert.assertTrue("Keystore does not exist: " + keystore, keystore.exists());
}
}