You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2019/04/04 18:39:32 UTC
[asterixdb] branch master updated: [NO ISSUE] Fix logging for
ncservice-based tests
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7974819 [NO ISSUE] Fix logging for ncservice-based tests
7974819 is described below
commit 797481957cfccffa01ebdea7f2ecb86c19c077ac
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Apr 4 12:29:56 2019 -0400
[NO ISSUE] Fix logging for ncservice-based tests
Change-Id: Id264aede0f62558ad6e34355047c623a1d594692
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3311
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@uci.edu>
---
.../api/http/servlet/VersionApiServletTest.java | 18 ++----
.../apache/asterix/test/runtime/HDFSCluster.java | 17 ++---
.../asterix/test/server/AbstractExecutionIT.java | 16 ++---
.../asterix/test/server/NCServiceExecutionIT.java | 72 ++++++++++++++--------
.../apache/asterix/test/server/NcLifecycleIT.java | 36 +++++------
.../org/apache/asterix/test/server/RecoveryIT.java | 36 +++++------
.../apache/asterix/test/server/ReplicationIT.java | 36 +++++------
.../test/resources/NCServiceExecutionIT/cc.conf | 2 +-
.../resources/NCServiceExecutionIT/ncservice1.conf | 3 +-
.../resources/NCServiceExecutionIT/ncservice2.conf | 3 +-
.../src/test/resources/NcLifecycleIT/cc.conf | 1 +
.../test/resources/NcLifecycleIT/ncservice1.conf | 3 +-
.../test/resources/NcLifecycleIT/ncservice2.conf | 3 +-
.../src/test/resources/ReplicationIT/cc.conf | 1 +
.../test/resources/ReplicationIT/ncservice1.conf | 3 +-
.../test/resources/ReplicationIT/ncservice2.conf | 3 +-
.../control/cc/CCLogConfigurationFactory.java | 12 ++--
.../hyracks/control/common/NodeControllerData.java | 2 +-
.../common/controllers/NodeRegistration.java | 4 +-
.../src/main/resources/log4j2.xml | 0
.../control/nc/NCLogConfigurationFactory.java | 21 ++++---
.../org/apache/hyracks/http/server/HttpServer.java | 6 --
.../org/apache/hyracks/http/server/WebManager.java | 6 ++
hyracks-fullstack/hyracks/hyracks-server/pom.xml | 4 --
.../apache/hyracks/test/server/NCServiceIT.java | 13 +---
.../test/server/process/HyracksCCProcess.java | 8 +--
.../server/process/HyracksNCServiceProcess.java | 10 ++-
.../test/server/process/HyracksServerProcess.java | 45 ++++++++++----
.../test/server/process/HyracksVirtualCluster.java | 32 ----------
.../src/test/resources/NCServiceIT/cc.conf | 1 +
.../src/test/resources/NCServiceIT/nc-blue.conf | 1 +
.../src/test/resources/NCServiceIT/nc-red.conf | 1 +
.../java/org/apache/hyracks/util/PidHelper.java | 14 +----
.../java/org/apache/hyracks/util/trace/Event.java | 8 +--
.../java/org/apache/hyracks/util/trace/Tracer.java | 2 +-
35 files changed, 210 insertions(+), 233 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
index 976dbc3..8dcfa26 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java
@@ -33,13 +33,10 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.asterix.api.http.server.VersionApiServlet;
import org.apache.asterix.common.config.BuildProperties;
import org.apache.asterix.runtime.utils.CcApplicationContext;
-import org.apache.asterix.test.runtime.SqlppExecutionTest;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -47,19 +44,12 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpMethod;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
public class VersionApiServletTest {
-
- @Before
- public void setup() throws Exception {
- // Starts test asterixdb cluster.
- SqlppExecutionTest.setUp();
- }
-
- @After
- public void teardown() throws Exception {
- // Tears down the asterixdb cluster.
- SqlppExecutionTest.tearDown();
+ static {
+ // bootstrap netty with log4j2 logging
+ io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
}
@Test
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
index a89304a..c0b4841 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
@@ -59,17 +59,18 @@ public class HDFSCluster {
/**
* Instantiates the (Mini) DFS Cluster with the configured number of datanodes.
- * Post instantiation, data is laoded to HDFS.
+ * Post instantiation, data is loaded to HDFS.
* Called prior to running the Runtime test suite.
*/
public void setup() throws Exception {
- setup("");
+ setup(new File("."));
}
- public void setup(String basePath) throws Exception {
- conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/core-site.xml"));
- conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
- conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
+ public void setup(File basePath) throws Exception {
+ File hadoopConfDir = new File(basePath, PATH_TO_HADOOP_CONF);
+ conf.addResource(new Path(new File(hadoopConfDir, "core-site.xml").getPath()));
+ conf.addResource(new Path(new File(hadoopConfDir, "mapred-site.xml").getPath()));
+ conf.addResource(new Path(new File(hadoopConfDir, "hdfs-site.xml").getPath()));
cleanupLocal();
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR);
MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
@@ -81,10 +82,10 @@ public class HDFSCluster {
loadData(basePath);
}
- private void loadData(String localDataRoot) throws IOException {
+ private void loadData(File localDataRoot) throws IOException {
Path destDir = new Path(HDFS_PATH);
dfs.mkdirs(destDir);
- File srcDir = new File(localDataRoot + DATA_PATH);
+ File srcDir = new File(localDataRoot, DATA_PATH);
if (srcDir.exists()) {
File[] listOfFiles = srcDir.listFiles();
for (File srcFile : listOfFiles) {
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
index 1a2f9ba..35bf7e4 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
@@ -14,12 +14,15 @@
*/
package org.apache.asterix.test.server;
+import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME;
+import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Objects;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
@@ -29,7 +32,6 @@ import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.asterix.testframework.context.TestFileContext;
import org.apache.asterix.testframework.xml.TestCase.CompilationUnit;
import org.apache.asterix.testframework.xml.TestGroup;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codehaus.plexus.util.FileUtils;
@@ -75,20 +77,18 @@ public abstract class AbstractExecutionIT {
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
- File externalTestsJar =
- new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator))
- .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0];
+ File externalTestsJar = Objects.requireNonNull(new File(joinPath("..", "asterix-external-data", "target"))
+ .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar")))[0];
- FileUtils.copyFile(externalTestsJar,
- new File(NCServiceExecutionIT.APP_HOME + "/repo", externalTestsJar.getName()));
+ FileUtils.copyFile(externalTestsJar, new File(APP_HOME, joinPath("repo", externalTestsJar.getName())));
NCServiceExecutionIT.setUp();
FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "data")),
- new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/data"));
+ new File(ASTERIX_APP_DIR, joinPath("clusters", "local", "working_dir", "data")));
FileUtils.copyDirectoryStructure(new File(joinPath("..", "asterix-app", "target", "data")),
- new File(NCServiceExecutionIT.ASTERIX_APP_DIR + "/clusters/local/working_dir/target/data"));
+ new File(ASTERIX_APP_DIR, joinPath("clusters", "local", "working_dir", "target", "data")));
// Set the node resolver to be the identity resolver that expects node names
// to be node controller ids; a valid assumption in test environment.
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
index 64506f5..d64dcb5 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
@@ -18,19 +18,22 @@
*/
package org.apache.asterix.test.server;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.runtime.HDFSCluster;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.asterix.testframework.xml.TestGroup;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.test.server.process.HyracksCCProcess;
import org.apache.hyracks.test.server.process.HyracksNCServiceProcess;
import org.apache.hyracks.test.server.process.HyracksVirtualCluster;
@@ -55,46 +58,64 @@ public class NCServiceExecutionIT {
// Important paths and files for this test.
// The "target" subdirectory of asterix-server. All outputs go here.
- public static final String TARGET_DIR =
- StringUtils.join(new String[] { "../asterix-server/target" }, File.separator);
+ public static final String TARGET_DIR = joinPath("..", "asterix-server", "target");
// Directory where the NCs create and store all data, as configured by
// src/test/resources/NCServiceExecutionIT/cc.conf.
- public static final String INSTANCE_DIR = StringUtils.join(new String[] { TARGET_DIR, "tmp" }, File.separator);
+ public static final String INSTANCE_DIR = joinPath(TARGET_DIR, "tmp");
// The log directory, where all CC, NCService, and NC logs are written. CC and
// NCService logs are configured on the HyracksVirtualCluster below. NC logs
// are configured in src/test/resources/NCServiceExecutionIT/ncservice*.conf.
- public static final String LOG_DIR =
- StringUtils.join(new String[] { TARGET_DIR, "failsafe-reports" }, File.separator);
+ public static final String LOG_DIR = joinPath(TARGET_DIR, "failsafe-reports");
// Directory where *.conf files are located.
- public static final String CONF_DIR =
- StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NCServiceExecutionIT" }, File.separator);
+ public static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "NCServiceExecutionIT");
// The app.home specified for HyracksVirtualCluster. The NCService expects
// to find the NC startup script in ${app.home}/bin.
- public static final String APP_HOME = StringUtils.join(new String[] { TARGET_DIR, "appassembler" }, File.separator);
+ public static final File APP_HOME = new File(TARGET_DIR, "appassembler");
// Path to the asterix-app directory. This is used as the current working
// directory for the CC and NCService processes, which allows relative file
// paths in "load" statements in test queries to find the right data. It is
// also used for HDFSCluster.
- public static final String ASTERIX_APP_DIR = StringUtils.join(new String[] { "..", "asterix-app" }, File.separator);
+ public static final File ASTERIX_APP_DIR = new File(joinPath("..", "asterix-app"));
// Path to the actual AQL test files, which we borrow from asterix-app. This is
// passed to TestExecutor.
- protected static final String TESTS_DIR =
- StringUtils.join(new String[] { ASTERIX_APP_DIR, "src", "test", "resources", "runtimets" }, File.separator);
+ protected static final File TESTS_DIR =
+ new File(ASTERIX_APP_DIR, joinPath("src", "test", "resources", "runtimets"));
// Path that actual results are written to. We create and clean this directory
// here, and also pass it to TestExecutor which writes the test output there.
- public static final String ACTUAL_RESULTS_DIR =
- StringUtils.join(new String[] { TARGET_DIR, "ittest" }, File.separator);
+ public static final File ACTUAL_RESULTS_DIR = new File(TARGET_DIR, "ittest");
private static final Logger LOGGER = LogManager.getLogger();
private static boolean startHdfs;
+ @AfterClass
+ public static void checkLogFiles() {
+ NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, NCServiceExecutionIT.class.getSimpleName()),
+ "asterix_nc1", "asterix_nc2");
+ }
+
+ public static void checkLogFiles(File logsPath, String... ncNames) {
+ LOGGER.info("verifying server log files");
+ Assert.assertTrue("logs dir (" + logsPath + ") missing!", logsPath.exists());
+ List<String> allFiles = new ArrayList<>();
+ Stream.of(ncNames).map(NCServiceExecutionIT::addNcFiles).forEach(allFiles::addAll);
+ allFiles.add("access.log");
+ allFiles.add("cc.log");
+ allFiles.forEach(file -> {
+ Assert.assertTrue(file + " missing!", new File(logsPath, file).exists());
+ });
+ }
+
+ private static List<String> addNcFiles(String ncName) {
+ return Arrays.asList("access-" + ncName + ".log", "nc-" + ncName + ".log", "trace-" + ncName + ".log");
+ }
+
enum KillCommand {
CC,
NC1,
@@ -126,8 +147,7 @@ public class NCServiceExecutionIT {
public static void setUp(boolean startHdfs) throws Exception {
NCServiceExecutionIT.startHdfs = startHdfs;
// Create actual-results output directory.
- File outDir = new File(ACTUAL_RESULTS_DIR);
- outDir.mkdirs();
+ ACTUAL_RESULTS_DIR.mkdirs();
// Remove any instance data from previous runs.
File instanceDir = new File(INSTANCE_DIR);
@@ -136,17 +156,16 @@ public class NCServiceExecutionIT {
}
if (startHdfs) {
- // HDFSCluster requires the input directory to end with a file separator.
- HDFSCluster.getInstance().setup(ASTERIX_APP_DIR + File.separator);
+ HDFSCluster.getInstance().setup(ASTERIX_APP_DIR);
}
- cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR));
- nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log"));
+ cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR);
+ nc1 = cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null);
- nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log"));
+ nc2 = cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null);
// Start CC
- cc = cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "asterixcc.out.log"));
+ cc = cluster.start(new File(CONF_DIR, "cc.conf"), null);
testExecutor.waitForClusterActive(30, TimeUnit.SECONDS);
clusterActive = true;
@@ -154,10 +173,9 @@ public class NCServiceExecutionIT {
@AfterClass
public static void tearDown() throws Exception {
- File outdir = new File(ACTUAL_RESULTS_DIR);
- File[] files = outdir.listFiles();
+ File[] files = ACTUAL_RESULTS_DIR.listFiles();
if (files == null || files.length == 0) {
- outdir.delete();
+ ACTUAL_RESULTS_DIR.delete();
}
cluster.stop();
if (startHdfs) {
@@ -176,7 +194,7 @@ public class NCServiceExecutionIT {
Collection<Object[]> testArgs = new ArrayList<>();
Random random = getRandom();
TestCaseContext.Builder b = new TestCaseContext.Builder();
- for (TestCaseContext ctx : b.build(new File(TESTS_DIR))) {
+ for (TestCaseContext ctx : b.build(TESTS_DIR)) {
if (!skip(ctx)) {
testArgs.add(new Object[] { ctx, ctx, null });
}
@@ -234,7 +252,7 @@ public class NCServiceExecutionIT {
@Test
public void test() throws Exception {
if (tcCtx != null) {
- testExecutor.executeTest(ACTUAL_RESULTS_DIR, tcCtx, null, false);
+ testExecutor.executeTest(ACTUAL_RESULTS_DIR.getPath(), tcCtx, null, false);
testExecutor.cleanup(tcCtx.toString(), badTestCases);
} else {
switch (killType) {
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java
index 4ae9d96..a31e36f 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NcLifecycleIT.java
@@ -18,11 +18,8 @@
*/
package org.apache.asterix.test.server;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.INSTANCE_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.LOG_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.TARGET_DIR;
+import static org.apache.asterix.test.server.NCServiceExecutionIT.*;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
import java.util.ArrayList;
@@ -33,12 +30,11 @@ import org.apache.asterix.test.base.RetainLogsRule;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.test.server.process.HyracksVirtualCluster;
-import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -49,13 +45,11 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class NcLifecycleIT {
- private static final String PATH_BASE =
- FileUtil.joinPath("src", "test", "resources", "integrationts", "NcLifecycle");
- private static final String CONF_DIR =
- StringUtils.join(new String[] { TARGET_DIR, "test-classes", "NcLifecycleIT" }, File.separator);
- private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest");
+ private static final String PATH_BASE = joinPath("src", "test", "resources", "integrationts", "NcLifecycle");
+ private static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "NcLifecycleIT");
+ private static final String PATH_ACTUAL = joinPath("target", "ittest");
private static final Logger LOGGER = LogManager.getLogger();
- private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath();
+ private static final File reportPath = new File("target", "failsafe-reports");
private static final TestExecutor testExecutor = new TestExecutor();
private static HyracksVirtualCluster cluster;
@@ -66,7 +60,7 @@ public class NcLifecycleIT {
}
@Rule
- public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this);
+ public TestRule retainLogs = new RetainLogsRule(ASTERIX_APP_DIR, reportPath, this);
@Before
public void before() throws Exception {
@@ -76,12 +70,12 @@ public class NcLifecycleIT {
FileUtils.deleteDirectory(instanceDir);
}
- cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR));
- cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log"));
- cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log"));
+ cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR);
+ cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null);
+ cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null);
// Start CC
- cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log"));
+ cluster.start(new File(CONF_DIR, "cc.conf"), null);
LOGGER.info("Instance created.");
testExecutor.waitForClusterActive(30, TimeUnit.SECONDS);
LOGGER.info("Instance is in ACTIVE state.");
@@ -94,6 +88,12 @@ public class NcLifecycleIT {
LOGGER.info("Instance destroyed.");
}
+ @AfterClass
+ public static void checkLogFiles() {
+ NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, NcLifecycleIT.class.getSimpleName()), "asterix_nc1",
+ "asterix_nc2");
+ }
+
@Test
public void test() throws Exception {
testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false);
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java
index 9123a5d..214eb5a 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/RecoveryIT.java
@@ -18,17 +18,19 @@
*/
package org.apache.asterix.test.server;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
+
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
+import java.util.Objects;
import org.apache.asterix.test.base.RetainLogsRule;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.runtime.HDFSCluster;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.AfterClass;
@@ -44,9 +46,9 @@ import org.junit.runners.Parameterized.Parameters;
public class RecoveryIT {
private static final Logger LOGGER = LogManager.getLogger();
- private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator;
+ private static final String PATH_ACTUAL = joinPath("target", "rttest");
private static final String PATH_BASE = "src/test/resources/transactionts/";
- private static final String HDFS_BASE = "../asterix-app/";
+ private static final File HDFS_BASE = new File("..", "asterix-app");
private TestCaseContext tcCtx;
private static File asterixInstallerPath;
private static File installerTargetPath;
@@ -68,17 +70,17 @@ public class RecoveryIT {
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
- File externalTestsJar =
- new File(StringUtils.join(new String[] { "..", "asterix-external-data", "target" }, File.separator))
- .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar"))[0];
+ File externalTestsJar = Objects.requireNonNull(new File(joinPath("..", "asterix-external-data", "target"))
+ .listFiles((dir, name) -> name.matches("asterix-external-data-.*-tests.jar")))[0];
asterixInstallerPath = new File(System.getProperty("user.dir"));
installerTargetPath = new File(new File(asterixInstallerPath.getParentFile(), "asterix-server"), "target");
reportPath = new File(installerTargetPath, "failsafe-reports").getAbsolutePath();
- ncServiceSubDirName =
- installerTargetPath.list((dir, name) -> name.matches("asterix-server.*binary-assembly"))[0];
+ ncServiceSubDirName = Objects.requireNonNull(
+ installerTargetPath.list((dir, name) -> name.matches("asterix-server.*binary-assembly")))[0];
ncServiceSubPath = new File(installerTargetPath, ncServiceSubDirName).getAbsolutePath();
- ncServiceHomeDirName = new File(ncServiceSubPath).list(((dir, name) -> name.matches("apache-asterixdb.*")))[0];
+ ncServiceHomeDirName = Objects.requireNonNull(
+ new File(ncServiceSubPath).list(((dir, name) -> name.matches("apache-asterixdb.*"))))[0];
ncServiceHomePath = new File(ncServiceSubPath, ncServiceHomeDirName).getAbsolutePath();
LOGGER.info("NCSERVICE_HOME=" + ncServiceHomePath);
@@ -89,14 +91,12 @@ public class RecoveryIT {
env = pb.environment();
env.put("NCSERVICE_HOME", ncServiceHomePath);
env.put("JAVA_HOME", System.getProperty("java.home"));
- scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator
- + "resources" + File.separator + "transactionts" + File.separator + "scripts";
+ scriptHomePath =
+ joinPath(asterixInstallerPath.getPath(), "src", "test", "resources", "transactionts", "scripts");
env.put("SCRIPT_HOME", scriptHomePath);
- TestExecutor.executeScript(pb,
- scriptHomePath + File.separator + "setup_teardown" + File.separator + "configure_and_validate.sh");
- TestExecutor.executeScript(pb,
- scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh");
+ TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "configure_and_validate.sh"));
+ TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "stop_and_delete.sh"));
HDFSCluster.getInstance().setup(HDFS_BASE);
}
@@ -104,11 +104,9 @@ public class RecoveryIT {
public static void tearDown() throws Exception {
File outdir = new File(PATH_ACTUAL);
FileUtils.deleteDirectory(outdir);
- File dataCopyDir =
- new File(ncServiceHomePath + File.separator + ".." + File.separator + ".." + File.separator + "data");
+ File dataCopyDir = new File(joinPath(ncServiceHomePath, "..", "..", "data"));
FileUtils.deleteDirectory(dataCopyDir);
- TestExecutor.executeScript(pb,
- scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh");
+ TestExecutor.executeScript(pb, joinPath(scriptHomePath, "setup_teardown", "stop_and_delete.sh"));
HDFSCluster.getInstance().cleanup();
}
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java
index 0f058bc..1de71c4 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/ReplicationIT.java
@@ -18,11 +18,8 @@
*/
package org.apache.asterix.test.server;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.APP_HOME;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.ASTERIX_APP_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.INSTANCE_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.LOG_DIR;
-import static org.apache.asterix.test.server.NCServiceExecutionIT.TARGET_DIR;
+import static org.apache.asterix.test.server.NCServiceExecutionIT.*;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
import java.net.InetAddress;
@@ -37,12 +34,11 @@ import org.apache.asterix.test.base.RetainLogsRule;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.test.server.process.HyracksVirtualCluster;
-import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -53,13 +49,11 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class ReplicationIT {
- private static final String PATH_BASE =
- FileUtil.joinPath("src", "test", "resources", "integrationts", "replication");
- private static final String CONF_DIR =
- StringUtils.join(new String[] { TARGET_DIR, "test-classes", "ReplicationIT" }, File.separator);
- private static final String PATH_ACTUAL = FileUtil.joinPath("target", "ittest");
+ private static final String PATH_BASE = joinPath("src", "test", "resources", "integrationts", "replication");
+ private static final String CONF_DIR = joinPath(TARGET_DIR, "test-classes", "ReplicationIT");
+ private static final String PATH_ACTUAL = joinPath("target", "ittest");
private static final Logger LOGGER = LogManager.getLogger();
- private static String reportPath = new File(FileUtil.joinPath("target", "failsafe-reports")).getAbsolutePath();
+ private static File reportPath = new File("target", "failsafe-reports");
private static final TestExecutor testExecutor = new TestExecutor();
private static HyracksVirtualCluster cluster;
@@ -82,7 +76,7 @@ public class ReplicationIT {
}
@Rule
- public TestRule retainLogs = new RetainLogsRule(NCServiceExecutionIT.ASTERIX_APP_DIR, reportPath, this);
+ public TestRule retainLogs = new RetainLogsRule(ASTERIX_APP_DIR, reportPath, this);
@Before
public void before() throws Exception {
@@ -92,12 +86,12 @@ public class ReplicationIT {
FileUtils.deleteDirectory(instanceDir);
}
- cluster = new HyracksVirtualCluster(new File(APP_HOME), new File(ASTERIX_APP_DIR));
- cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), new File(LOG_DIR, "ncservice1.log"));
- cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), new File(LOG_DIR, "ncservice2.log"));
+ cluster = new HyracksVirtualCluster(APP_HOME, ASTERIX_APP_DIR);
+ cluster.addNCService(new File(CONF_DIR, "ncservice1.conf"), null);
+ cluster.addNCService(new File(CONF_DIR, "ncservice2.conf"), null);
// Start CC
- cluster.start(new File(CONF_DIR, "cc.conf"), new File(LOG_DIR, "cc.log"));
+ cluster.start(new File(CONF_DIR, "cc.conf"), null);
LOGGER.info("Instance created.");
testExecutor.waitForClusterActive(30, TimeUnit.SECONDS);
LOGGER.info("Instance is in ACTIVE state.");
@@ -110,6 +104,12 @@ public class ReplicationIT {
LOGGER.info("Instance destroyed.");
}
+ @AfterClass
+ public static void checkLogFiles() {
+ NCServiceExecutionIT.checkLogFiles(new File(TARGET_DIR, ReplicationIT.class.getSimpleName()), "asterix_nc1",
+ "asterix_nc2");
+ }
+
@Test
public void test() throws Exception {
testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false);
diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
index 2a92c5d..1c6d574 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/cc.conf
@@ -46,5 +46,5 @@ heartbeat.period=2000
heartbeat.max.misses=25
[common]
-log.dir = ../asterix-server/target/failsafe-reports/
+log.dir = ../asterix-server/target/NCServiceExecutionIT
log.level = INFO
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf
index ba10142..e178cfa 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice1.conf
@@ -16,5 +16,4 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
-
+logdir=-
diff --git a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf
index 2036584..43aecf0 100644
--- a/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf
+++ b/asterixdb/asterix-server/src/test/resources/NCServiceExecutionIT/ncservice2.conf
@@ -16,6 +16,5 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
+logdir=-
port=9091
-
diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf
index dc80e56..a91622b 100644
--- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/cc.conf
@@ -46,3 +46,4 @@ heartbeat.max.misses=25
[common]
log.level = INFO
+log.dir=../asterix-server/target/NcLifecycleIT
diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf
index ba10142..e178cfa 100644
--- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf
+++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice1.conf
@@ -16,5 +16,4 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
-
+logdir=-
diff --git a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf
index 2036584..43aecf0 100644
--- a/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf
+++ b/asterixdb/asterix-server/src/test/resources/NcLifecycleIT/ncservice2.conf
@@ -16,6 +16,5 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
+logdir=-
port=9091
-
diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf
index 19e951f..b88cbd4 100644
--- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf
+++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/cc.conf
@@ -47,6 +47,7 @@ heartbeat.max.misses=25
[common]
log.level = INFO
+log.dir=../asterix-server/target/ReplicationIT
replication.enabled=true
replication.strategy=all
replication.factor=2
diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf
index ba10142..e178cfa 100644
--- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf
+++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice1.conf
@@ -16,5 +16,4 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
-
+logdir=-
diff --git a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf
index 2036584..43aecf0 100644
--- a/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf
+++ b/asterixdb/asterix-server/src/test/resources/ReplicationIT/ncservice2.conf
@@ -16,6 +16,5 @@
; under the License.
[ncservice]
-logdir=../asterix-server/target/failsafe-reports
+logdir=-
port=9091
-
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
index 22dea9f..a635084 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
@@ -23,6 +23,8 @@ import java.net.URI;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -34,6 +36,7 @@ import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
public class CCLogConfigurationFactory extends ConfigurationFactory {
+ private static final Logger LOGGER = LogManager.getLogger();
private CCConfig config;
public CCLogConfigurationFactory(CCConfig config) {
@@ -42,6 +45,8 @@ public class CCLogConfigurationFactory extends ConfigurationFactory {
public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
File logDir = new File(config.getLogDir());
+ File ccLog = new File(logDir, "cc.log");
+ LOGGER.warn("logs are being redirected to: {}", ccLog::getAbsolutePath);
builder.setStatusLevel(Level.WARN);
builder.setConfigurationName("RollingBuilder");
// create a rolling file appender
@@ -50,10 +55,9 @@ public class CCLogConfigurationFactory extends ConfigurationFactory {
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
- AppenderComponentBuilder defaultRoll =
- builder.newAppender("default", "RollingFile").addAttribute("fileName", new File(logDir, "cc.log"))
- .addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz")).add(defaultLayout)
- .addComponent(triggeringPolicy);
+ AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile")
+ .addAttribute("fileName", ccLog).addAttribute("filePattern", new File(logDir, "cc-%d{MM-dd-yy}.log.gz"))
+ .add(defaultLayout).addComponent(triggeringPolicy);
builder.add(defaultRoll);
// create the new logger
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
index 24a3e57..c37acab 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
@@ -79,7 +79,7 @@ public class NodeControllerData {
private final Map<String, String> systemProperties;
- private final long pid;
+ private final int pid;
private final HeartbeatSchema hbSchema;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
index 474bc0a..437b001 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
@@ -75,7 +75,7 @@ public final class NodeRegistration implements Serializable {
private final NetworkAddress messagingPort;
- private final long pid;
+ private final int pid;
private final NodeCapacity capacity;
@@ -190,7 +190,7 @@ public final class NodeRegistration implements Serializable {
return messagingPort;
}
- public long getPid() {
+ public int getPid() {
return pid;
}
}
diff --git a/asterixdb/asterix-app/src/main/resources/log4j2.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
similarity index 100%
rename from asterixdb/asterix-app/src/main/resources/log4j2.xml
rename to hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
index a657d6b..fad6b3e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
@@ -23,6 +23,8 @@ import java.net.URI;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -34,6 +36,7 @@ import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
public class NCLogConfigurationFactory extends ConfigurationFactory {
+ private static final Logger LOGGER = LogManager.getLogger();
private NCConfig config;
public NCLogConfigurationFactory(NCConfig config) {
@@ -43,6 +46,8 @@ public class NCLogConfigurationFactory extends ConfigurationFactory {
public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
String nodeId = config.getNodeId();
File logDir = new File(config.getLogDir());
+ File ncLog = new File(logDir, "nc-" + nodeId + ".log");
+ LOGGER.warn("logs are being redirected to: {}", ncLog::getAbsolutePath);
builder.setStatusLevel(Level.WARN);
builder.setConfigurationName("RollingBuilder");
// create a rolling file appender
@@ -51,11 +56,11 @@ public class NCLogConfigurationFactory extends ConfigurationFactory {
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
- AppenderComponentBuilder defaultRoll = builder.newAppender("default", "RollingFile")
- .addAttribute("fileName", new File(logDir, "nc-" + nodeId + ".log").getAbsolutePath())
- .addAttribute("filePattern",
- new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath())
- .add(defaultLayout).addComponent(triggeringPolicy);
+ AppenderComponentBuilder defaultRoll =
+ builder.newAppender("default", "RollingFile").addAttribute("fileName", ncLog.getAbsolutePath())
+ .addAttribute("filePattern",
+ new File(logDir, "nc-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").getAbsolutePath())
+ .add(defaultLayout).addComponent(triggeringPolicy);
builder.add(defaultRoll);
// create the new logger
@@ -74,9 +79,9 @@ public class NCLogConfigurationFactory extends ConfigurationFactory {
LayoutComponentBuilder traceLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m,%n")
.addAttribute("header", "[").addAttribute("footer", "]");
AppenderComponentBuilder traceRoll = builder.newAppender("trace", "RollingFile")
- .addAttribute("fileName", logDir + "trace-" + nodeId + ".log")
- .addAttribute("filePattern", logDir + "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz").add(traceLayout)
- .addComponent(triggeringPolicy);
+ .addAttribute("fileName", new File(logDir, "trace-" + nodeId + ".log"))
+ .addAttribute("filePattern", new File(logDir, "trace-" + nodeId + "-%d{MM-dd-yy-ss}.log.gz"))
+ .add(traceLayout).addComponent(triggeringPolicy);
builder.add(traceRoll);
builder.add(builder.newLogger("org.apache.hyracks.util.trace.Tracer.Traces", Level.forName("TRACER", 570))
.add(builder.newAppenderRef("trace")).addAttribute("additivity", false));
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 94a6440..152f6c7 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -50,8 +50,6 @@ import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
-import io.netty.util.internal.logging.InternalLoggerFactory;
-import io.netty.util.internal.logging.Log4J2LoggerFactory;
public class HttpServer {
// Constants
@@ -84,10 +82,6 @@ public class HttpServer {
private Throwable cause;
private HttpServerConfig config;
- static {
- InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
- }
-
public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
this(bossGroup, workerGroup, new InetSocketAddress(port), config, null);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
index 55741e4..b8347ae 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/WebManager.java
@@ -24,12 +24,18 @@ import java.util.List;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
public class WebManager {
private final List<HttpServer> servers;
private final EventLoopGroup bosses;
private final EventLoopGroup workers;
+ static {
+ // bootstrap netty with log4j2 logging
+ io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
+ }
+
/**
* Create a web manager with number of bosses = 1
* and number of workers = MultithreadEventLoopGroup.DEFAULT_EVENT_LOOP_THREADS
diff --git a/hyracks-fullstack/hyracks/hyracks-server/pom.xml b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
index 7e566d6..6c5639f 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-server/pom.xml
@@ -185,9 +185,5 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
index 5cbc5b4..cd0359f 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/NCServiceIT.java
@@ -21,11 +21,8 @@ package org.apache.hyracks.test.server;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
import java.util.Iterator;
-import org.apache.commons.io.FileUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
@@ -58,13 +55,9 @@ public class NCServiceIT {
@BeforeClass
public static void setUp() throws Exception {
cluster = new HyracksVirtualCluster(new File(APP_HOME), null);
- File tempConf = new File(TARGET_DIR, "cc.conf");
- FileUtils.copyFile(new File(RESOURCE_DIR, "cc.conf"), tempConf);
- Files.write(tempConf.toPath(), ("log.dir: " + LOG_DIR).getBytes(), StandardOpenOption.APPEND);
- File log4jPath = new File(FileUtil.joinPath("..", "..", "src", "test", "resources", "log4j2-hyracks-test.xml"));
- cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), new File(LOG_DIR, "nc-red.log"), log4jPath);
- cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), new File(LOG_DIR, "nc-blue.log"), log4jPath);
+ cluster.addNCService(new File(RESOURCE_DIR, "nc-red.conf"), null);
+ cluster.addNCService(new File(RESOURCE_DIR, "nc-blue.conf"), null);
try {
Thread.sleep(2000);
@@ -72,7 +65,7 @@ public class NCServiceIT {
}
// Start CC
- cluster.start(tempConf, new File(LOG_DIR, "cc.log"), log4jPath);
+ cluster.start(new File(RESOURCE_DIR, "cc.conf"), null);
try {
Thread.sleep(10000);
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
index a79d033..0b529fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksCCProcess.java
@@ -25,18 +25,14 @@ import org.apache.hyracks.control.cc.CCDriver;
public class HyracksCCProcess extends HyracksServerProcess {
- public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+ HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir) {
+ super(" cc");
this.configFile = configFile;
this.logFile = logFile;
this.appHome = appHome;
this.workingDir = workingDir;
}
- public HyracksCCProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) {
- this(configFile, logFile, appHome, workingDir);
- args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath());
- }
-
@Override
protected String getMainClassName() {
return CCDriver.class.getName();
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
index d0e0244..958eb9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksNCServiceProcess.java
@@ -20,23 +20,21 @@ package org.apache.hyracks.test.server.process;
import java.io.File;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hyracks.control.nc.service.NCService;
public class HyracksNCServiceProcess extends HyracksServerProcess {
+ private static final AtomicInteger ncServiceCounter = new AtomicInteger();
- public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) {
+ HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir) {
+ super("nc" + ncServiceCounter.incrementAndGet());
this.configFile = configFile;
this.logFile = logFile;
this.appHome = appHome;
this.workingDir = workingDir;
}
- public HyracksNCServiceProcess(File configFile, File logFile, File appHome, File workingDir, File log4jPath) {
- this(configFile, logFile, appHome, workingDir);
- args.add("-Dlog4j.configurationFile=file://" + log4jPath.getAbsolutePath());
- }
-
@Override
protected String getMainClassName() {
return NCService.class.getName();
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
index a8c363b..59ab4f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksServerProcess.java
@@ -18,9 +18,12 @@
*/
package org.apache.hyracks.test.server.process;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -32,14 +35,21 @@ import org.apache.logging.log4j.Logger;
abstract class HyracksServerProcess {
private static final Logger LOGGER = LogManager.getLogger();
+ protected final String processName;
protected Process process;
+ protected Thread pipeThread;
protected File configFile = null;
protected File logFile = null;
protected File appHome = null;
protected File workingDir = null;
protected List<String> args = new ArrayList<>();
+ protected HyracksServerProcess(String processName) {
+ this.processName = processName;
+ }
+
public void start() throws IOException {
+
String[] cmd = buildCommand();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Starting command: " + Arrays.toString(cmd));
@@ -47,34 +57,45 @@ abstract class HyracksServerProcess {
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.redirectErrorStream(true);
+ pb.directory(workingDir);
if (logFile != null) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Logging to: " + logFile.getCanonicalPath());
- }
+ LOGGER.info("Logging to: " + logFile.getCanonicalPath());
logFile.getParentFile().mkdirs();
try (FileWriter writer = new FileWriter(logFile, true)) {
writer.write("---------------------\n");
}
pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile));
+ process = pb.start();
} else {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Logfile not set, subprocess will output to stdout");
- }
+ pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
+ process = pb.start();
+ pipeThread = new Thread(() -> {
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ System.out.println(processName + ": " + line);
+ }
+ } catch (IOException e) {
+ LOGGER.debug("exception reading process pipe", e);
+ }
+ });
+ pipeThread.start();
}
- pb.directory(workingDir);
- process = pb.start();
}
public void stop() {
process.destroy();
try {
boolean success = process.waitFor(30, TimeUnit.SECONDS);
- if (LOGGER.isWarnEnabled()) {
- LOGGER.warn("Killing unresponsive NC Process");
- }
if (!success) {
+ LOGGER.warn("Killing unresponsive NC Process");
process.destroyForcibly();
}
+ if (pipeThread != null) {
+ pipeThread.interrupt();
+ pipeThread.join();
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -106,7 +127,7 @@ abstract class HyracksServerProcess {
cList.add(configFile.getAbsolutePath());
}
addCmdLineArgs(cList);
- return cList.toArray(new String[cList.size()]);
+ return cList.toArray(new String[0]);
}
protected void addJvmArgs(List<String> cList) {
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
index 6c77628..062d429 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/java/org/apache/hyracks/test/server/process/HyracksVirtualCluster.java
@@ -65,22 +65,6 @@ public class HyracksVirtualCluster {
}
/**
- * Creates and starts an NCService.
- *
- * @param configFile
- * - full path to an ncservice.conf. May be null to accept all defaults.
- * @throws IOException
- * - if there are errors starting the process.
- */
- public HyracksNCServiceProcess addNCService(File configFile, File logFile, File log4jConfig) throws IOException {
- HyracksNCServiceProcess proc =
- new HyracksNCServiceProcess(configFile, logFile, appHome, workingDir, log4jConfig);
- proc.start();
- ncProcs.add(proc);
- return proc;
- }
-
- /**
* Starts the CC, initializing the cluster. Expects that any NCs referenced
* in the cluster configuration have already been started with addNCService().
*
@@ -97,22 +81,6 @@ public class HyracksVirtualCluster {
}
/**
- * Starts the CC, initializing the cluster. Expects that any NCs referenced
- * in the cluster configuration have already been started with addNCService().
- *
- * @param ccConfigFile
- * - full path to a cluster conf file. May be null to accept all
- * defaults, although this is seldom useful since there are no NCs.
- * @throws IOException
- * - if there are errors starting the process.
- */
- public HyracksCCProcess start(File ccConfigFile, File logFile, File log4jConfig) throws IOException {
- ccProc = new HyracksCCProcess(ccConfigFile, logFile, appHome, workingDir, log4jConfig);
- ccProc.start();
- return ccProc;
- }
-
- /**
* Stops all processes in the cluster.
* QQQ Someday this should probably do a graceful stop of NCs rather than
* killing the NCService.
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
index 9c80c7d..9b1a1cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/cc.conf
@@ -30,3 +30,4 @@ address = 127.0.0.1
console.listen.port = 12345
[common]
+log.dir=target/NCServiceIT
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
index 6eb38dd..baccd46 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-blue.conf
@@ -18,3 +18,4 @@
[ncservice]
address=127.0.0.1
port=9091
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
index 286bd32..7616b37 100644
--- a/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
+++ b/hyracks-fullstack/hyracks/hyracks-server/src/test/resources/NCServiceIT/nc-red.conf
@@ -18,3 +18,4 @@
[ncservice]
address=127.0.0.1
port=9090
+logdir=-
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
index 46e77e3..5a8edbd 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
@@ -21,7 +21,6 @@ package org.apache.hyracks.util;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.apache.logging.log4j.Level;
@@ -35,20 +34,11 @@ public class PidHelper {
private PidHelper() {
}
- public static long getPid() {
+ public static int getPid() {
return getPid(ManagementFactory.getRuntimeMXBean());
}
- public static long getPid(RuntimeMXBean runtimeMXBean) {
- // TODO: replace with direct invoke of getPid() once compatibility is at JDK 10 or higher
- try {
- Method getPidMethod = runtimeMXBean.getClass().getMethod("getPid");
- return (Long) getPidMethod.invoke(runtimeMXBean);
- } catch (NoSuchMethodException e) {
- LOGGER.debug("ignoring exception trying to find getPid() (expected pre-JDK 10)", e);
- } catch (IllegalAccessException | InvocationTargetException e) {
- LOGGER.debug("ignoring exception trying to execute getPid()", e);
- }
+ public static int getPid(RuntimeMXBean runtimeMXBean) {
try {
Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm");
jvmField.setAccessible(true);
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
index b744198..b5fe3d3 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Event.java
@@ -28,14 +28,14 @@ final class Event {
public final long cat;
public final ITracer.Phase ph;
public final long ts;
- public final long pid;
+ public final int pid;
public final long tid;
public final ITracer.Scope scope;
public final String args;
public final TraceCategoryRegistry registry;
- private Event(String name, long cat, ITracer.Phase ph, long ts, long pid, long tid, ITracer.Scope scope,
- String args, TraceCategoryRegistry registry) {
+ private Event(String name, long cat, ITracer.Phase ph, long ts, int pid, long tid, ITracer.Scope scope, String args,
+ TraceCategoryRegistry registry) {
this.name = name;
this.cat = cat;
this.ph = ph;
@@ -51,7 +51,7 @@ final class Event {
return (System.nanoTime() - NANOTIME_DELTA_TO_EPOCH) / 1000;
}
- public static Event create(String name, long cat, ITracer.Phase ph, long pid, long tid, ITracer.Scope scope,
+ public static Event create(String name, long cat, ITracer.Phase ph, int pid, long tid, ITracer.Scope scope,
String args, TraceCategoryRegistry registry) {
return new Event(name, cat, ph, timestamp(), pid, tid, scope, args, registry);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
index c1c38cf..9019fdf 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
@@ -44,7 +44,7 @@ public class Tracer implements ITracer {
private long categories;
private final TraceCategoryRegistry registry;
- private static final long pid = PidHelper.getPid();
+ private static final int pid = PidHelper.getPid();
public Tracer(String name, long categories, TraceCategoryRegistry registry) {
final String traceLoggerName = Tracer.class.getName() + ".Traces." + name;