You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by co...@apache.org on 2016/02/22 07:26:01 UTC
sqoop git commit: SQOOP-2832: Sqoop2: Precommit: Create log files for
individual tests (Jarek Jarcec Cecho via Colin Ma)
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 87515c56e -> 9aec8f965
SQOOP-2832: Sqoop2: Precommit: Create log files for individual tests
(Jarek Jarcec Cecho via Colin Ma)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9aec8f96
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9aec8f96
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9aec8f96
Branch: refs/heads/sqoop2
Commit: 9aec8f965eacde9be04fd40eb53a664873bee4df
Parents: 87515c5
Author: Colin Ma <co...@apache.org>
Authored: Mon Feb 22 15:24:43 2016 +0800
Committer: Colin Ma <co...@apache.org>
Committed: Mon Feb 22 15:24:43 2016 +0800
----------------------------------------------------------------------
test/pom.xml | 1 +
.../test/testng/ReconfigureLogListener.java | 118 +++++++++++++++++++
.../resources/connector-loading-tests-suite.xml | 1 +
test/src/test/resources/hive-tests-suite.xml | 1 +
.../test/resources/integration-tests-suite.xml | 1 +
.../resources/new-integration-tests-suite.xml | 1 +
test/src/test/resources/shell-tests-suite.xml | 1 +
test/src/test/resources/tools-tests-suite.xml | 1 +
test/src/test/resources/upgrade-tests-suite.xml | 1 +
9 files changed, 126 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 134bca1..1e88b34 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -209,6 +209,7 @@ limitations under the License.
</excludes>
<systemPropertyVariables>
<sqoop.integration.tmpdir>${project.build.directory}</sqoop.integration.tmpdir>
+ <sqoop.integration.log>${project.build.directory}/surefire-reports</sqoop.integration.log>
</systemPropertyVariables>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java b/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java
new file mode 100644
index 0000000..52e9bb9
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testng/ReconfigureLogListener.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testng;
+
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.testng.ITestResult;
+import org.testng.TestListenerAdapter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Sqoop is running as much tests as possible inside one suite to safe time starting
+ * miniclusters which is time consuming exercise (~40 seconds per single test class).
+ * That however means that we have one output log file that recently grown to more then
+ * 1GB in side and hence the usability has decreased.
+ *
+ * This listener will intercept each test and will reconfigure log4j to log directly
+ * into files rather then to console (that would be forwarded by maven surefire plugin to
+ * the normal log file). Each test will get it's own file which is easier for human to
+ * read.
+ *
+ * We're using a counter to order log files per execution order rather then per name as
+ * we can't guarantee log isolation entirely (e.g. some information relevant to test N
+ * can be in log file for test N-1). It's easier to open previous log if you immediately
+ * know what is the previous log.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings({"ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD"})
+public class ReconfigureLogListener extends TestListenerAdapter {
+
+ /**
+ * Directory into which we should put all log files.
+ */
+ private static final String TEST_LOG_DIRECTORY = System.getProperty("sqoop.integration.log");
+
+ /**
+ * We're purposely configuring some of the loggers into a different level then DEBUG
+ * as they are more spamming the log then being actually helpful. As with all logs,
+ * it can happen that they might be needed at some point, so rather then hardcoding
+ * the rules in code, we've chosen path to expose ability to override them on the
+ * command line.
+ */
+ private static final String TEST_LOGGERS = System.getProperty("sqoop.integration.log.loggers",
+ "org.eclipse.jetty=INFO," +
+ "org.apache.directory=INFO," +
+ "org.apache.hadoop.ipc.Server=INFO," +
+ "org.apache.hadoop.hdfs=INFO," +
+ "org.apache.hadoop.security.SaslInputStream=INFO," +
+ "org.apache.hadoop.security.SaslRpcClient=INFO," +
+ "org.apache.hadoop.ipc.Client=INFO," +
+ "org.apache.hadoop.conf.Configuration=INFO"
+ );
+
+ // Parsed and cached variant of TEST_LOGGERS
+ private static Map<String, Level> loggerConfiguration;
+ static {
+ loggerConfiguration = new HashMap<>();
+ for(String rule : TEST_LOGGERS.split(",")) {
+ String []split = rule.split("=");
+ if(split.length != 2) {
+ throw new RuntimeException("Incorrect rule, expected logger=level: " + rule);
+ }
+
+ loggerConfiguration.put(split[0].trim(), Level.toLevel(split[1].trim()));
+ }
+ }
+
+ /**
+ * Counter is increased for each test execution
+ */
+ private static int counter = 0;
+
+ /**
+ * On every test start, we'll start logging into different file
+ */
+ @Override
+ public void onTestStart(ITestResult tr) {
+ // Reset log4j configuration on every test run
+ Logger.getRootLogger().getLoggerRepository().resetConfiguration();
+
+ // Get test # so that our resulting files are sorted by execution and not by name
+ int ourCounter = counter++;
+
+ // Usual File appender
+ FileAppender appender = new FileAppender();
+ appender.setName("Sqoop test dynamic logger");
+ appender.setFile(TEST_LOG_DIRECTORY + "/" + String.format("%05d", ourCounter) + "_" + tr.getTestClass().getName() + "." + tr.getName() + ".txt");
+ appender.setLayout(new PatternLayout("%d{ISO8601} [%t] %-5p %c %x - %m%n"));
+ appender.setImmediateFlush(true);
+ appender.setThreshold(Level.DEBUG);
+ appender.activateOptions();
+
+ // Different levels for various not-so important loggers
+ Logger.getRootLogger().addAppender(appender);
+ for(Map.Entry<String, Level> entry : loggerConfiguration.entrySet()) {
+ Logger.getLogger(entry.getKey()).setLevel(entry.getValue());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/connector-loading-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/connector-loading-tests-suite.xml b/test/src/test/resources/connector-loading-tests-suite.xml
index 02c1df3..c03fb4f 100644
--- a/test/src/test/resources/connector-loading-tests-suite.xml
+++ b/test/src/test/resources/connector-loading-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="ConnectorLoadingTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/hive-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/hive-tests-suite.xml b/test/src/test/resources/hive-tests-suite.xml
index 3e2b328..c65db04 100644
--- a/test/src/test/resources/hive-tests-suite.xml
+++ b/test/src/test/resources/hive-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="HiveTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/integration-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/integration-tests-suite.xml b/test/src/test/resources/integration-tests-suite.xml
index 73e0a77..91e47e4 100644
--- a/test/src/test/resources/integration-tests-suite.xml
+++ b/test/src/test/resources/integration-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="ConnectorTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/new-integration-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/new-integration-tests-suite.xml b/test/src/test/resources/new-integration-tests-suite.xml
index 96a1320..b470b62 100644
--- a/test/src/test/resources/new-integration-tests-suite.xml
+++ b/test/src/test/resources/new-integration-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="ServerTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/shell-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/shell-tests-suite.xml b/test/src/test/resources/shell-tests-suite.xml
index e2c0b09..565d2fd 100644
--- a/test/src/test/resources/shell-tests-suite.xml
+++ b/test/src/test/resources/shell-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="ShellTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/tools-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/tools-tests-suite.xml b/test/src/test/resources/tools-tests-suite.xml
index fee2121..ceb192e 100644
--- a/test/src/test/resources/tools-tests-suite.xml
+++ b/test/src/test/resources/tools-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="ToolsTests">
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9aec8f96/test/src/test/resources/upgrade-tests-suite.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/upgrade-tests-suite.xml b/test/src/test/resources/upgrade-tests-suite.xml
index 3318e67..f97a2d2 100644
--- a/test/src/test/resources/upgrade-tests-suite.xml
+++ b/test/src/test/resources/upgrade-tests-suite.xml
@@ -22,6 +22,7 @@ limitations under the License.
<listeners>
<listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
+ <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
</listeners>
<test name="RepositoryTests">