You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/01/11 07:53:20 UTC
[3/5] kylin git commit: KYLIN-2380 refactor DbUnit assertion
KYLIN-2380 refactor DbUnit assertion
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e088159
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e088159
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e088159
Branch: refs/heads/yang22-hbase1.x
Commit: 2e0881596e251c7e517748c1468d459ec86210d3
Parents: 3a3ee3c
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 11 12:10:36 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/query/HackedDbUnitAssert.java | 125 ++++++++++++++++---
.../apache/kylin/query/ITMassInQueryTest.java | 9 +-
.../org/apache/kylin/query/KylinTestBase.java | 60 +++++----
3 files changed, 148 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index 338f698..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
import org.dbunit.dataset.DataSetException;
import org.dbunit.dataset.ITable;
import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ *
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql with limit
*/
public class HackedDbUnitAssert extends DbUnitAssert {
private static final Logger logger = LoggerFactory.getLogger(HackedDbUnitAssert.class);
+ private boolean hackCheckContains;
+ private boolean hackIgnoreIntBigIntMismatch;
+
+ public void hackCheckContains() {
+ hackCheckContains = true;
+ }
+
+ public void hackIgnoreIntBigIntMismatch() {
+ hackIgnoreIntBigIntMismatch = true;
+ }
+
+ // THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY hackXXX CONDITION CHECKS.
public void assertEquals(ITable expectedTable, ITable actualTable, FailureHandler failureHandler) throws DatabaseUnitException {
logger.trace("assertEquals(expectedTable, actualTable, failureHandler) - start");
logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,22 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
ITableMetaData actualMetaData = actualTable.getTableMetaData();
String expectedTableName = expectedMetaData.getTableName();
- // // Verify row count
- // int expectedRowsCount = expectedTable.getRowCount();
- // int actualRowsCount = actualTable.getRowCount();
- // if (expectedRowsCount != actualRowsCount) {
- // String msg = "row count (table=" + expectedTableName + ")";
- // Error error =
- // failureHandler.createFailure(msg, String
- // .valueOf(expectedRowsCount), String
- // .valueOf(actualRowsCount));
- // logger.error(error.toString());
- // throw error;
- // }
+ // Verify row count
+ int expectedRowsCount = expectedTable.getRowCount();
+ int actualRowsCount = actualTable.getRowCount();
+ if (!hackCheckContains) {
+ if (expectedRowsCount != actualRowsCount) {
+ String msg = "row count (table=" + expectedTableName + ")";
+ Error error = failureHandler.createFailure(msg, String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+ logger.error(error.toString());
+ throw error;
+ }
+ }
// if both tables are empty, it is not necessary to compare columns, as such comparison
// can fail if column metadata is different (which could occurs when comparing empty tables)
- if (expectedTable.getRowCount() == 0 && actualTable.getRowCount() == 0) {
+ if (expectedRowsCount == 0 && actualRowsCount == 0) {
logger.debug("Tables are empty, hence equals.");
return;
}
@@ -94,10 +111,86 @@ public class HackedDbUnitAssert extends DbUnitAssert {
ComparisonColumn[] comparisonCols = getComparisonColumns(expectedTableName, expectedColumns, actualColumns, failureHandler);
// Finally compare the data
- compareData(expectedTable, actualTable, comparisonCols, failureHandler);
+ if (hackCheckContains)
+ compareDataContains(expectedTable, actualTable, comparisonCols, failureHandler);
+ else
+ compareData(expectedTable, actualTable, comparisonCols, failureHandler);
+ }
+
+ // THIS METHOD IS COPIED FROM SUPER CLASS TO CHANGE ComparisonColumn TO OUR OWN.
+ @Override
+ protected ComparisonColumn[] getComparisonColumns(String expectedTableName, Column[] expectedColumns, Column[] actualColumns, FailureHandler failureHandler) {
+ ComparisonColumn[] result = new ComparisonColumn[expectedColumns.length];
+
+ for (int j = 0; j < expectedColumns.length; j++) {
+ Column expectedColumn = expectedColumns[j];
+ Column actualColumn = actualColumns[j];
+ result[j] = new HackedComparisonColumn(expectedTableName, expectedColumn, actualColumn, failureHandler);
+ }
+ return result;
+ }
+
+ // MOSTLY COPIED FROM SUPER CLASS
+ private class HackedComparisonColumn extends ComparisonColumn {
+ private String columnName;
+ private DataType dataType;
+
+ public HackedComparisonColumn(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) {
+
+ // super class is actually useless, all public methods are overridden below
+ super(tableName, expectedColumn, expectedColumn, failureHandler);
+
+ this.columnName = expectedColumn.getColumnName();
+ this.dataType = getComparisonDataType(tableName, expectedColumn, actualColumn, failureHandler);
+ }
+
+ @Override
+ public String getColumnName() {
+ return this.columnName;
+ }
+
+ @Override
+ public DataType getDataType() {
+ return this.dataType;
+ }
+
+ // COPIED FROM SUPER CLASS, CHANGES ARE LEAD BY hackXXX CONDITION CHECKS.
+ private DataType getComparisonDataType(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) {
+ if (logger.isDebugEnabled())
+ logger.debug("getComparisonDataType(tableName={}, expectedColumn={}, actualColumn={}, failureHandler={}) - start", new Object[] { tableName, expectedColumn, actualColumn, failureHandler });
+
+ DataType expectedDataType = expectedColumn.getDataType();
+ DataType actualDataType = actualColumn.getDataType();
+
+ // The two columns have different data type
+ if (!expectedDataType.getClass().isInstance(actualDataType)) {
+ // Expected column data type is unknown, use actual column data type
+ if (expectedDataType instanceof UnknownDataType) {
+ return actualDataType;
+ }
+
+ // Actual column data type is unknown, use expected column data type
+ if (actualDataType instanceof UnknownDataType) {
+ return expectedDataType;
+ }
+
+ if (hackIgnoreIntBigIntMismatch) {
+ if (expectedDataType instanceof IntegerDataType && actualDataType instanceof BigIntegerDataType)
+ return actualDataType;
+ }
+
+ // Impossible to determine which data type to use
+ String msg = "Incompatible data types: (table=" + tableName + ", col=" + expectedColumn.getColumnName() + ")";
+ throw failureHandler.createFailure(msg, String.valueOf(expectedDataType), String.valueOf(actualDataType));
+ }
+
+ // Both columns have same data type, return any one of them
+ return expectedDataType;
+ }
+
}
- protected void compareData(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException {
+ private void compareDataContains(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException {
logger.debug("compareData(expectedTable={}, actualTable={}, " + "comparisonCols={}, failureHandler={}) - start", new Object[] { expectedTable, actualTable, comparisonCols, failureHandler });
if (expectedTable == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index fd3abfb..28cdb67 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -30,12 +30,9 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.dbunit.Assertion;
-import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.dbunit.dataset.ITable;
-import org.dbunit.ext.h2.H2Connection;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -144,13 +141,11 @@ public class ITMassInQueryTest extends KylinTestBase {
sql = sql.replace("massin(test_kylin_fact.SELLER_ID,'vip_customers')", "test_kylin_fact.SELLER_ID in ( " + org.apache.commons.lang.StringUtils.join(vipSellers, ",") + ")");
printInfo("Query Result from H2 - " + queryName);
printInfo("Query for H2 - " + sql);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index b8a48ef..a948dac 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -53,7 +53,7 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
-import org.dbunit.Assertion;
+import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
@@ -231,11 +231,13 @@ public class KylinTestBase {
sql = changeJoinType(sql, joinType);
ITable queryTable = dbConn.createQueryTable(resultTableName + queryName, sql);
- String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length];
- for (int i = 0; i < columnNames.length; i++) {
- columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName();
- }
if (needSort) {
+ String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length];
+ for (int i = 0; i < columnNames.length; i++) {
+ columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName();
+ }
+ Arrays.sort(columnNames);
+
queryTable = new SortedTable(queryTable, columnNames);
}
if (PRINT_RESULT)
@@ -359,7 +361,7 @@ public class KylinTestBase {
// compare the result
Assert.assertEquals(queryName, expectRowCount, kylinTable.getRowCount());
- // Assertion.assertEquals(expectRowCount, kylinTable.getRowCount());
+ // assertTableEquals(expectRowCount, kylinTable.getRowCount());
}
}
@@ -382,7 +384,7 @@ public class KylinTestBase {
ITable kylinTable = executeQuery(kylinConn, queryName, sql, false);
// compare the result
- Assertion.assertEquals(expectTable, kylinTable);
+ assertTableEquals(expectTable, kylinTable);
}
}
@@ -405,9 +407,7 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
@@ -468,13 +468,10 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, false);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, false);
try {
- HackedDbUnitAssert hackedDbUnitAssert = new HackedDbUnitAssert();
- hackedDbUnitAssert.assertEquals(h2Table, kylinTable);
+ assertTableContains(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
@@ -507,13 +504,11 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
@@ -546,14 +541,33 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- IDatabaseConnection h2Conn = new DatabaseConnection(h2Connection);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeDynamicQuery(h2Conn, queryName, sql, parameters, needSort);
+ ITable h2Table = executeDynamicQuery(newH2Connection(), queryName, sql, parameters, needSort);
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
}
}
+
+ protected void assertTableEquals(ITable h2Table, ITable kylinTable) throws DatabaseUnitException {
+ HackedDbUnitAssert dbUnit = new HackedDbUnitAssert();
+ dbUnit.hackIgnoreIntBigIntMismatch();
+ dbUnit.assertEquals(h2Table, kylinTable);
+ }
+
+ protected void assertTableContains(ITable h2Table, ITable kylinTable) throws DatabaseUnitException {
+ HackedDbUnitAssert dbUnit = new HackedDbUnitAssert();
+ dbUnit.hackIgnoreIntBigIntMismatch();
+ dbUnit.hackCheckContains();
+ dbUnit.assertEquals(h2Table, kylinTable);
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
protected int runSqlFile(String file) throws Exception {
return runSQL(new File(file), true, false);