You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ka...@apache.org on 2011/11/14 10:16:05 UTC
svn commit: r1201640 - in /incubator/oozie/trunk: ./ core/
core/src/test/java/org/apache/oozie/test/ core/src/test/resources/
docs/src/site/twiki/
Author: kamrul
Date: Mon Nov 14 09:16:04 2011
New Revision: 1201640
URL: http://svn.apache.org/viewvc?rev=1201640&view=rev
Log:
OOZIE-557 Simplify/normalize testing configuration when using different databases. (Alejandro via Mohammad)
Modified:
incubator/oozie/trunk/core/pom.xml
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
incubator/oozie/trunk/core/src/test/resources/mysql-oozie-site.xml
incubator/oozie/trunk/core/src/test/resources/postgresql-oozie-site.xml
incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki
incubator/oozie/trunk/pom.xml
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/pom.xml?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/core/pom.xml (original)
+++ incubator/oozie/trunk/core/pom.xml Mon Nov 14 09:16:04 2011
@@ -465,7 +465,8 @@
<activation>
<activeByDefault>false</activeByDefault>
<property>
- <name>mysql</name>
+ <name>oozie.test.db</name>
+ <value>mysql</value>
</property>
</activation>
<dependencies>
@@ -485,7 +486,8 @@
<activation>
<activeByDefault>false</activeByDefault>
<property>
- <name>oracle</name>
+ <name>oozie.test.db</name>
+ <value>oracle</value>
</property>
</activation>
<dependencies>
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java Mon Nov 14 09:16:04 2011
@@ -48,6 +48,7 @@ import org.apache.oozie.CoordinatorJobBe
import org.apache.oozie.SLAEventBean;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.store.CoordinatorStore;
@@ -79,21 +80,29 @@ public abstract class XTestCase extends
private String hadoopVersion;
protected XLog log = new XLog(LogFactory.getLog(getClass()));
+ private static File OOZIE_SRC_DIR = null;
private static final String OOZIE_TEST_PROPERTIES = "oozie.test.properties";
public static float WAITFOR_RATIO = Float.parseFloat(System.getProperty("oozie.test.waitfor.ratio", "1"));
static {
try {
- // by default uses 'test.properties'
- String testPropsFile = System.getProperty(OOZIE_TEST_PROPERTIES, "test.properties");
- File file = new File(testPropsFile);
- // the testPropsFile is looked at 'oozie-main' project level
- // this trick here is to work both with Maven (runs the testcases from module directory)
- // and IDEs (run testcases from 'oozie-main' project directory).
- if (!file.exists()) {
- file = new File(file.getAbsoluteFile().getParentFile().getParentFile(), testPropsFile);
+ OOZIE_SRC_DIR = new File("core").getAbsoluteFile();
+ if (!OOZIE_SRC_DIR.exists()) {
+ OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile().getParentFile();
+ OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core");
+ }
+ if (!OOZIE_SRC_DIR.exists()) {
+ System.err.println();
+ System.err.println("Could not determine project root directory");
+ System.err.println();
+ System.exit(-1);
}
+ OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile();
+
+ String testPropsFile = System.getProperty(OOZIE_TEST_PROPERTIES, "test.properties");
+ File file = (testPropsFile.startsWith("/"))
+ ? new File(testPropsFile) : new File(OOZIE_SRC_DIR, testPropsFile);
if (file.exists()) {
System.out.println();
System.out.println("*********************************************************************************");
@@ -193,8 +202,7 @@ public abstract class XTestCase extends
@Override
protected void setUp() throws Exception {
super.setUp();
- String baseDir = System.getProperty(OOZIE_TEST_DIR, "/tmp");
- hadoopVersion = System.getProperty(HADOOP_VERSION, "0.20.0");
+ String baseDir = System.getProperty(OOZIE_TEST_DIR, new File("target/test-data").getAbsolutePath());
String msg = null;
if (!baseDir.startsWith("/")) {
msg = XLog.format("System property [{0}]=[{1}] must be set to an absolute path", OOZIE_TEST_DIR, baseDir);
@@ -205,8 +213,18 @@ public abstract class XTestCase extends
}
}
if (msg != null) {
- throw new Error(msg);
+ System.err.println();
+ System.err.println(msg);
+ System.exit(-1);
+ }
+ File f = new File(baseDir);
+ f.mkdirs();
+ if (!f.exists() || !f.isDirectory()) {
+ System.err.println();
+ System.err.println(XLog.format("Could not create test dir [{0}]", baseDir));
+ System.exit(-1);
}
+ hadoopVersion = System.getProperty(HADOOP_VERSION, "0.20.0");
sysProps = new HashMap<String, String>();
testCaseDir = createTestCaseDir(this, true);
@@ -215,27 +233,23 @@ public abstract class XTestCase extends
Services.setOozieHome();
testCaseConfDir = createTestCaseSubDir("conf");
- //setting up custom Oozie site for testing if avail
- String customOozieSite = System.getProperty("oozie.test.config.file", "");
- if (!customOozieSite.equals("")) {
- if (!customOozieSite.startsWith("/")) {
- System.err.println();
- System.err.println(XLog.format(
- "Custom configuration file must be an absolute path [{0}]", customOozieSite));
- System.err.println();
- System.exit(-1);
- }
- File source = new File(customOozieSite);
- if (!source.exists()) {
- System.err.println();
- System.err.println(XLog.format(
- "Custom configuration file for testing does no exist [{0}]", customOozieSite));
- System.err.println();
- System.exit(-1);
- }
- File target = new File(testCaseConfDir, "oozie-site.xml");
- IOUtils.copyStream(new FileInputStream(source), new FileOutputStream(target));
+ // load test Oozie site
+ String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb");
+ String defaultOozieSize =
+ new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieTestDB + "-oozie-site.xml").getAbsolutePath();
+ String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSize);
+ File source = (customOozieSite.startsWith("/"))
+ ? new File(customOozieSite) : new File(OOZIE_SRC_DIR, customOozieSite);
+ source = source.getAbsoluteFile();
+ if (!source.exists()) {
+ System.err.println();
+ System.err.println(XLog.format("Custom configuration file for testing does no exist [{0}]",
+ source.getAbsolutePath()));
+ System.err.println();
+ System.exit(-1);
}
+ File target = new File(testCaseConfDir, "oozie-site.xml");
+ IOUtils.copyStream(new FileInputStream(source), new FileOutputStream(target));
if (System.getProperty("oozielocal.log") == null) {
setSystemProperty("oozielocal.log", "/tmp/oozielocal.log");
@@ -250,16 +264,10 @@ public abstract class XTestCase extends
System.setProperty("oozie.services.ext", "org.apache.oozie.service.HadoopAccessorService");
}
- if (System.getProperty("oozie.test.db", "hsqldb").equals("hsqldb")) {
- setSystemProperty("oozie.service.JPAService.jdbc.driver", "org.hsqldb.jdbcDriver");
- setSystemProperty("oozie.service.JPAService.jdbc.url", "jdbc:hsqldb:mem:oozie-db;create=true");
- }
- if (System.getProperty("oozie.test.db", "hsqldb").equals("derby")) {
- delete(new File(baseDir, "oozie-derby"));
- setSystemProperty("oozie.service.JPAService.jdbc.driver", "org.apache.derby.jdbc.EmbeddedDriver");
- setSystemProperty("oozie.service.JPAService.jdbc.url", "jdbc:derby:" + baseDir +
- "/oozie-derby;create=true");
+ if (System.getProperty("oozie.test.db.host") == null) {
+ System.setProperty("oozie.test.db.host", "localhost");
}
+ setSystemProperty(ConfigurationService.OOZIE_DATA_DIR, testCaseDir);
}
/**
@@ -353,8 +361,8 @@ public abstract class XTestCase extends
*/
private String getTestCaseDirInternal(TestCase testCase) {
ParamChecker.notNull(testCase, "testCase");
- File dir = new File(System.getProperty(OOZIE_TEST_DIR, "/tmp"));
- dir = new File(dir, "oozietests");
+ File dir = new File(System.getProperty(OOZIE_TEST_DIR, "target/test-data"));
+ dir = new File(dir, "oozietests").getAbsoluteFile();
dir = new File(dir, testCase.getClass().getName());
dir = new File(dir, testCase.getName());
return dir.getAbsolutePath();
Modified: incubator/oozie/trunk/core/src/test/resources/mysql-oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/mysql-oozie-site.xml?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/mysql-oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/test/resources/mysql-oozie-site.xml Mon Nov 14 09:16:04 2011
@@ -19,22 +19,26 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
- <name>oozie.service.DataSourceService.jdbc.driver</name>
+ <name>oozie.service.JPAService.jdbc.driver</name>
<value>com.mysql.jdbc.Driver</value>
<description>JDBC driver class.</description>
</property>
<property>
- <name>oozie.service.DataSourceService.jdbc.url</name>
- <value>jdbc:mysql://localhost:3306</value>
+ <name>oozie.test.db.port</name>
+ <value>3306</value>
+ </property>
+ <property>
+ <name>oozie.service.JPAService.jdbc.url</name>
+ <value>jdbc:mysql://${oozie.test.db.host}:${oozie.test.db.port}/oozie</value>
<description>JDBC URL.</description>
</property>
<property>
- <name>oozie.service.DataSourceService.jdbc.username</name>
+ <name>oozie.service.JPAService.jdbc.username</name>
<value>oozie</value>
<description>DB user name.</description>
</property>
<property>
- <name>oozie.service.DataSourceService.jdbc.password</name>
+ <name>oozie.service.JPAService.jdbc.password</name>
<value>oozie</value>
<description>
DB user password. IMPORTANT: if password is emtpy leave a 1 space string, the service trims the
Modified: incubator/oozie/trunk/core/src/test/resources/postgresql-oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/postgresql-oozie-site.xml?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/postgresql-oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/test/resources/postgresql-oozie-site.xml Mon Nov 14 09:16:04 2011
@@ -1,44 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
- <property>
- <name>oozie.service.JPAService.jdbc.driver</name>
- <value>org.postgresql.Driver</value>
- <description>JDBC driver class.</description>
- </property>
- <property>
- <name>oozie.service.JPAService.jdbc.url</name>
- <value>jdbc:postgresql://localhost:5432/OOZIEDB</value>
- <description>JDBC URL.</description>
- </property>
- <property>
- <name>oozie.service.JPAService.jdbc.username</name>
- <value>OOZIE</value>
- <description>DB user name.</description>
- </property>
- <property>
- <name>oozie.service.JPAService.jdbc.password</name>
- <value>oozie</value>
- <description>
- DB user password. IMPORTANT: if password is emtpy leave a 1 space string, the service trims the
- value, if empty Configuration assumes it is NULL.
- </description>
- </property>
-</configuration>
Modified: incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki Mon Nov 14 09:16:04 2011
@@ -86,12 +86,22 @@ $ mvn clean test -Doozie.test.hadoop.min
NOTE: The embedded minicluster cannot be used when testing with 'kerberos' authentication.
-*Using a custom Oozie configuration for testcases (for example, to use a MySQL DB):*
+*Using a custom Oozie configuration for testcases:*
<verbatim>
-$ mvn clean test -Dmysql -Doozie.test.config.file=/home/tucu/oozie-site-mysql.xml
+$ mvn clean test -Doozie.test.config.file=/home/tucu/custom-oozie-sitel.xml
</verbatim>
+*Running the testcases with different databases:*
+
+<verbatim>
+$ mvn clean test -Doozie.test.db=[hsqldb*|derby|mysql|postgres|oracle]
+</verbatim>
+
+Using =mysql= and =oracle= enables profiles that will include their JARs files in the build. If using
+ =oracle=, the Oracle JDBC JAR file must be manually installed in the local Maven cache (the JAR is
+not available in public Maven repos).
+
---+++ Build Options Reference
All these options can be set using *-D*.
@@ -105,19 +115,15 @@ undefined (Hadoop JARs are not included)
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
-*mysql* (*): Includes MySQL JDBC driver in the distro, default is undefined, MySQL JDBC driver is not included.
-
-*oracle* (*): Includes Oracle JDBC driver in the distro, default is undefined, Oracle JDBC driver is not included.
-
*skipTests* (*): skips the execution of all testcases, no value required, default is undefined
*test*= (*): runs a single test case, to run a test give the test class name without package and extension, no default
*hadoop20*= (*) : indicates the build/test should not include classes for Hadoop 20S, default is 'false'
-*oozie.test.db*= (*): indicates the database to use for running the testcases, supported values are 'hsqldb', 'derby' and 'other';
-default value is 'hsqldb'. IMPORTANT, when using 'derby' a Maven profile is activated to run all the testcases in
-'always' fork mode (otherwise Derby driver goes bonker after 50 testcases or so due to repeated initializations). Use 'other' when using MySQL, Oracle or PostgreSQL databases, if using 'other' a '-Doozie.test.config.file=' must be used with the correct JDBC information must be provided (the core/src/test/resources/ directory has samples for the different databases).
+*oozie.test.db*= (*): indicates the database to use for running the testcases, supported values are 'hsqldb', 'derby',
+ 'mysql', 'postgres' and 'oracle'; default value is 'hsqldb'. For each database there is
+ =core/src/test/resources/DATABASE-oozie-site.xml= file preconfigured.
*oozie.test.properties* (*): indicates the file to load the test properties from, by default is =test.properties=.
Having this option allows having different test properties sets, for example: minicluster, simple & kerberos.
Modified: incubator/oozie/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/pom.xml?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/pom.xml (original)
+++ incubator/oozie/trunk/pom.xml Mon Nov 14 09:16:04 2011
@@ -55,9 +55,15 @@
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>
- <oozie.data.dir>${project.build.directory}/test-data</oozie.data.dir>
+ <oozie.test.dir>${project.build.directory}/test-data</oozie.test.dir>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
+
+ <oozie.data.dir>${oozie.test.dir}</oozie.data.dir>
+ <oozie.test.db.host>localhost</oozie.test.db.host>
+ <oozie.test.db>hsqldb</oozie.test.db>
+ <oozie.test.default.config.file>${basedir}/src/test/resources/${oozie.test.db}-oozie-site.xml</oozie.test.default.config.file>
+ <oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file>
</properties>
<modules>
@@ -425,7 +431,7 @@
<dependency>
<groupId>com.oracle</groupId>
<artifactId>ojdbc6</artifactId>
- <version>11.2.0.1.0</version>
+ <version>11.2.0.2.0</version>
</dependency>
<dependency>
@@ -600,7 +606,9 @@
<argLine>-Xmx1024m</argLine>
<systemPropertiesVariables>
<hadoop.log.dir>/tmp</hadoop.log.dir>
- <oozie.data.dir>${oozie.data.dir}</oozie.data.dir>
+ <oozie.test.data.dir>${oozie.test.data.dir}</oozie.test.data.dir>
+ <oozie.test.db.host>${oozie.test.db.host}</oozie.test.db.host>
+ <oozie.test.config.file>${oozie.test.config.file}</oozie.test.config.file>
</systemPropertiesVariables>
<excludes>
<exclude>**/${test.exclude}.java</exclude>
@@ -646,5 +654,5 @@
</properties>
</profile>
</profiles>
-
+
</project>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1201640&r1=1201639&r2=1201640&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Mon Nov 14 09:16:04 2011
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-557 Simplify/normalize testing configuration when using different databases.
OOZIE-590 Log Retrieval from multiple .gz archive files
OOZIE-587 Out of memory issues due to current query design
OOZIE-600 Bump-up the version to 3.2.0-SNAPSHOT