You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2019/06/28 10:00:26 UTC
[asterixdb] branch master updated: [ASTERIXDB-2593][FUN] TPC-DS
always parallelize + gen all tables
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c71173d [ASTERIXDB-2593][FUN] TPC-DS always parallelize + gen all tables
c71173d is described below
commit c71173d12335a41e6a9b8e4ca8b8d435bad22b5e
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Thu Jun 20 05:26:43 2019 +0300
[ASTERIXDB-2593][FUN] TPC-DS always parallelize + gen all tables
- user model changes: yes
- user can call the tpcds_datagen() by passing only the scaling
factor, and that will automatically generate all the tables.
- storage format changes: no
- interface changes: no
Details:
- Added support to a second version of the tpcds_datagen()
function to take only a single parameter that will result in
generating the data for all the tables in one go. The user
now can generate the data for a single table at a time, or
all the tables in one go.
- Overridden the behavior for activating the parallelism.
Before, the library wouldn't activate parallelism unless
the tables are big, and the number of data generated per
table is over 1,000,000. With this change, the parallelism
is always activated, regardless of the table size or the
data size being generated.
- Added a new test, TPCExecutionTest, to execute the long
tests for the TPC. Some of the TPC tests take longer than
5 minutes to finish and could result in high usage of
disk space.
Change-Id: Iff199b0c533d22bcae1caf5057788b257ba4e486
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3437
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../asterix/app/function/FunctionRewriter.java | 2 +-
... => TPCDSAllTablesDataGeneratorDatasource.java} | 31 ++-
...va => TPCDSAllTablesDataGeneratorRewriter.java} | 54 +++--
.../app/function/TPCDSDataGeneratorFunction.java | 19 +-
.../app/function/TPCDSDataGeneratorReader.java | 270 +++++++++++++++------
...> TPCDSSingleTableDataGeneratorDatasource.java} | 23 +-
... => TPCDSSingleTableDataGeneratorRewriter.java} | 44 ++--
.../asterix/util/MetadataBuiltinFunctions.java | 16 +-
.../asterix/test/runtime/TPCExecutionTest.java | 86 +++++++
.../src/test/resources/runtimets/only_tpc.xml | 23 ++
.../datagen_sf_1_all_tables_1.1.query.sqlpp} | 0
.../datagen_sf_1_all_tables_2.1.query.sqlpp | 25 ++
.../datagen_sf_1_all_tables_1.1.adm} | 0
.../datagen_sf_1_all_tables_2.1.adm | 24 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 12 -
.../src/test/resources/runtimets/testsuite_tpc.xml | 45 ++++
.../metadata/declared/FunctionDataSource.java | 2 +-
17 files changed, 524 insertions(+), 152 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionRewriter.java
index d1f1898..1cc0a94 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionRewriter.java
@@ -49,7 +49,7 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
public abstract class FunctionRewriter implements IFunctionToDataSourceRewriter {
- private FunctionIdentifier functionId;
+ FunctionIdentifier functionId;
public FunctionRewriter(FunctionIdentifier functionId) {
this.functionId = functionId;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
similarity index 65%
copy from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index cb7e357..352fea8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -18,27 +18,30 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
/**
* This TPC-DS function is used to generate data with accordance to the specifications of the TPC Benchmark DS.
*/
-public class TPCDSDataGeneratorDatasource extends FunctionDataSource {
- private final String tableName;
+public class TPCDSAllTablesDataGeneratorDatasource extends FunctionDataSource {
+
private final double scalingFactor;
+ private final FunctionIdentifier functionIdentifier;
- public TPCDSDataGeneratorDatasource(INodeDomain domain, String tableName, double scalingFactor)
- throws AlgebricksException {
- super(createDataSourceId(tableName, scalingFactor), domain);
- this.tableName = tableName;
+ TPCDSAllTablesDataGeneratorDatasource(INodeDomain domain, double scalingFactor,
+ FunctionIdentifier functionIdentifier) throws AlgebricksException {
+ super(createDataSourceId(scalingFactor), domain);
this.scalingFactor = scalingFactor;
+ this.functionIdentifier = functionIdentifier;
}
/**
@@ -46,20 +49,24 @@ public class TPCDSDataGeneratorDatasource extends FunctionDataSource {
* DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls
* are happening with different parameters and the optimizer understands them as a single function.
*
- * @param tableName
- * table name to be added as part of the DataSourceId
* @param scalingFactor
* scaling factor to be added as part of the DataSourceId
* @return A DataSourceId that's based on the function details and its parameters
*/
- private static DataSourceId createDataSourceId(String tableName, double scalingFactor) {
- return new DataSourceId(TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR.getNamespace(),
- TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR.getName(), tableName, Double.toString(scalingFactor));
+ private static DataSourceId createDataSourceId(double scalingFactor) {
+ return new DataSourceId(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getNamespace(),
+ TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR.getName(),
+ Double.toString(scalingFactor));
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new TPCDSDataGeneratorFunction(locations, tableName, scalingFactor);
+ return new TPCDSDataGeneratorFunction(locations, null, scalingFactor, functionIdentifier);
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return new AlgebricksAbsolutePartitionConstraint(csm.getClusterLocations().getLocations());
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorRewriter.java
similarity index 68%
copy from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorRewriter.java
index fb871b9..da4a7c5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorRewriter.java
@@ -26,7 +26,6 @@ import org.apache.asterix.om.base.AInt16;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.AInt8;
-import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.exceptions.TypeMismatchException;
@@ -43,17 +42,20 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
/**
* This TPC-DS function is used to generate data with accordance to the specifications of the TPC Benchmark DS.
*
- * The data generator takes 2 arguments:
- * - first argument: a valid table name.
- * - second argument: the desired scaling factor.
+ * This version of the function takes 1 argument:
+ * - Scaling factor that decides the data size to be generated.
+ *
+ * This function will generate the data for all the TPC-DS tables
*/
-public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
- public static final FunctionIdentifier TPCDS_DATA_GENERATOR =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "tpcds-datagen", 2);
- public static final TPCDSDataGeneratorRewriter INSTANCE = new TPCDSDataGeneratorRewriter(TPCDS_DATA_GENERATOR);
+public class TPCDSAllTablesDataGeneratorRewriter extends FunctionRewriter {
+
+ public static final FunctionIdentifier TPCDS_ALL_TABLES_DATA_GENERATOR =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "tpcds-datagen", 1);
+ public static final TPCDSAllTablesDataGeneratorRewriter INSTANCE =
+ new TPCDSAllTablesDataGeneratorRewriter(TPCDS_ALL_TABLES_DATA_GENERATOR);
- private TPCDSDataGeneratorRewriter(FunctionIdentifier functionId) {
+ private TPCDSAllTablesDataGeneratorRewriter(FunctionIdentifier functionId) {
super(functionId);
}
@@ -61,35 +63,27 @@ public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
protected FunctionDataSource toDatasource(IOptimizationContext context, AbstractFunctionCallExpression function)
throws AlgebricksException {
- // TODO(Hussain) check if it is safe to assume that we're receiving constant expressions only as arguments.
UnnestingFunctionCallExpression functionCall = (UnnestingFunctionCallExpression) function;
- ConstantExpression tableNameArgument = (ConstantExpression) functionCall.getArguments().get(0).getValue();
- ConstantExpression scalingFactorArgument = (ConstantExpression) functionCall.getArguments().get(1).getValue();
+ ConstantExpression scalingFactorArgument = (ConstantExpression) functionCall.getArguments().get(0).getValue();
// Extract the values
- IAObject tableNameArgumentValue = ((AsterixConstantValue) tableNameArgument.getValue()).getObject();
IAObject scalingFactorArgumentValue = ((AsterixConstantValue) scalingFactorArgument.getValue()).getObject();
// Get the arguments' types and validate them
- IAType tableNameType = tableNameArgumentValue.getType();
IAType scalingFactorType = scalingFactorArgumentValue.getType();
- if (tableNameType.getTypeTag() != ATypeTag.STRING) {
- throw new TypeMismatchException(functionCall.getFunctionIdentifier(), 0, tableNameType.getTypeTag(),
- ATypeTag.STRING);
- }
-
// Ensure the scaling factor can be promoted to double
if (!ATypeHierarchy.canPromote(scalingFactorType.getTypeTag(), ATypeTag.DOUBLE)) {
- throw new TypeMismatchException(TPCDS_DATA_GENERATOR, 1, scalingFactorType.getTypeTag(), ATypeTag.TINYINT,
- ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT, ATypeTag.DOUBLE);
+ throw new TypeMismatchException(getFunctionIdentifier(), 1, scalingFactorType.getTypeTag(),
+ ATypeTag.TINYINT, ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT,
+ ATypeTag.DOUBLE);
}
// Convert whichever number type we received into double
double scalingFactor = getScalingFactor(scalingFactorArgumentValue);
- String tableName = ((AString) tableNameArgumentValue).getStringValue();
- return new TPCDSDataGeneratorDatasource(context.getComputationNodeDomain(), tableName, scalingFactor);
+ return new TPCDSAllTablesDataGeneratorDatasource(context.getComputationNodeDomain(), scalingFactor,
+ getFunctionIdentifier());
}
/**
@@ -114,8 +108,18 @@ public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
case DOUBLE:
return ((ADouble) value).getDoubleValue();
default:
- throw new TypeMismatchException(TPCDS_DATA_GENERATOR, 1, value.getType().getTypeTag(), ATypeTag.TINYINT,
- ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT, ATypeTag.DOUBLE);
+ throw new TypeMismatchException(getFunctionIdentifier(), 1, value.getType().getTypeTag(),
+ ATypeTag.TINYINT, ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT,
+ ATypeTag.DOUBLE);
}
}
+
+ /**
+ * Gets the function identifier
+ *
+ * @return function identifier
+ */
+ private FunctionIdentifier getFunctionIdentifier() {
+ return functionId;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorFunction.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorFunction.java
index 4c52bca..b7bada7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorFunction.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorFunction.java
@@ -21,12 +21,14 @@ package org.apache.asterix.app.function;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.metadata.declared.AbstractDatasourceFunction;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
* This TPC-DS function is used to generate data with accordance to the specifications of the TPC Benchmark DS.
*/
+
public class TPCDSDataGeneratorFunction extends AbstractDatasourceFunction {
private static final long serialVersionUID = 1L;
@@ -34,12 +36,14 @@ public class TPCDSDataGeneratorFunction extends AbstractDatasourceFunction {
private final String tableName;
private final double scalingFactor;
private final int parallelism;
+ private final FunctionIdentifier functionIdentifier;
- public TPCDSDataGeneratorFunction(AlgebricksAbsolutePartitionConstraint locations, String tableName,
- double scalingFactor) {
+ TPCDSDataGeneratorFunction(AlgebricksAbsolutePartitionConstraint locations, String tableName, double scalingFactor,
+ FunctionIdentifier functionIdentifier) {
super(locations);
this.tableName = tableName;
this.scalingFactor = scalingFactor;
+ this.functionIdentifier = functionIdentifier;
/*
TPC-DS has the option to parallelize the data generation and produce the data as chunks. We'll match the
@@ -51,6 +55,15 @@ public class TPCDSDataGeneratorFunction extends AbstractDatasourceFunction {
@Override
public IRecordReader<char[]> createRecordReader(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
- return new TPCDSDataGeneratorReader(tableName, scalingFactor, parallelism, partition);
+ return new TPCDSDataGeneratorReader(tableName, scalingFactor, parallelism, partition, getFunctionIdentifier());
+ }
+
+ /**
+ * Gets the function identifier
+ *
+ * @return function identifier
+ */
+ private FunctionIdentifier getFunctionIdentifier() {
+ return functionIdentifier;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorReader.java
index 626dee3..1122a29 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorReader.java
@@ -19,6 +19,7 @@
package org.apache.asterix.app.function;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
@@ -35,72 +36,161 @@ import com.teradata.tpcds.Session;
import com.teradata.tpcds.Table;
/**
- * Each partition will be running a TPCDSDataGeneratorReader instance. Depending on the number of partitions, the data
- * generator will parallelize its work based on the number of partitions. The reader is passed the parallelism level
- * (depending on the number of partitions).
+ * Each partition will be running a TPCDS data generator reader instance. Depending on the number of partitions, the
+ * data generator will parallelize its work based on the number of partitions. The reader is passed the parallelism
+ * level based on the number of partition instances.
*
- * Note: The data generator does not apply the parallelism unless at least 1,000,000 rows generation is requested (this
- * depends on the scaling factor and the table for whose the rows are being generated). This means, despite the number
- * of available partitions, if the minimum number of rows is not met, a single partition will generate all the rows
- * while all the other partitions will generate 0 rows.
+ * The function automatically handles generating the data for a single specified table or for all the tables. Also,
+ * the parallelism will take place regardless of the selected data size to be generated.
*/
+
public class TPCDSDataGeneratorReader extends FunctionReader {
- private final int parallelism;
- private final int chunkNumber;
- private final String tableName;
- private final double scalingFactor;
- private Table selectedTable;
+ private final FunctionIdentifier functionIdentifier;
+
+ // Table members
+ private final List<Table> selectedTables;
private final StringBuilder builder = new StringBuilder();
- private final Iterator<List<List<String>>> dataGeneratorIterator;
-
- public TPCDSDataGeneratorReader(String tableName, double scalingFactor, int parallelism, int partitionNumber)
- throws HyracksDataException {
- this.tableName = tableName;
- this.scalingFactor = scalingFactor;
- this.parallelism = parallelism;
-
- /*
- Since we already ensured the parallelism level for the TPC-DS matches the number of partitions we have, we
- need a way to tell each partition which chunk to generate. Since each TPCDSDataGeneratorReader is receiving
- the partition number that's running it, we're gonna use that as the chunk to be produced by the data
- generator.
- Note that the indexing for the partitions starts at position 0, but the data generator chunks start at 1,
- so the chunk will always be the partitionNumber + 1
- */
- chunkNumber = partitionNumber + 1;
-
- dataGeneratorIterator = getDataGeneratorIterator();
+ private final List<Iterator<List<List<String>>>> tableIterators = new ArrayList<>();
+ private Table currentTable;
+ private final int tableCount;
+ private int currentTableIndex;
+
+ // Parent tables will generate 2 records, 1 for the main, and another for the child's row, we'll push the parent
+ // record, and in the next rotation we'll push the child row, then go to next row generation
+ private String childRow = null;
+
+ // This will guide the next() method to know where to get the next record from (Parent or child record)
+ private boolean getFromParent = true;
+ private final boolean generateAllTables;
+
+ TPCDSDataGeneratorReader(String tableName, double scalingFactor, int parallelism, int partitionNumber,
+ FunctionIdentifier functionIdentifier) throws HyracksDataException {
+ this.functionIdentifier = functionIdentifier;
+
+ // Note: chunk numbers start with 1, so each chunk will be partition number + 1
+ // Create the session with the specified properties, the sessions also specifies the chunk to be generated
+ Session session = Session.getDefaultSession().withScale(scalingFactor).withParallelism(parallelism)
+ .withChunkNumber(partitionNumber + 1);
+
+ // If the tableName is null, then we're generating all the tables
+ generateAllTables = tableName == null;
+
+ // Get the table(s)
+ selectedTables = getTableFromStringTableName(tableName);
+
+ // These variables will monitor and assist with each table's data generation
+ currentTableIndex = 0;
+ tableCount = selectedTables.size();
+ currentTable = selectedTables.get(currentTableIndex);
+
+ // Iterators for the tables to generate the data for
+ for (Table table : selectedTables) {
+ Results result = calculateParallelism(table, session);
+ tableIterators.add(result.iterator());
+ }
}
@Override
public boolean hasNext() {
- return dataGeneratorIterator.hasNext();
+
+ // Return children generated records of current table
+ if (generateAllTables && childRow != null) {
+ getFromParent = false;
+ return true;
+ }
+
+ // Current table still has more
+ if (tableIterators.get(currentTableIndex).hasNext()) {
+ return true;
+ }
+
+ // We went over all the tables
+ if (currentTableIndex == tableCount - 1) {
+ return false;
+ }
+
+ // Go to the next table
+ currentTableIndex++;
+ currentTable = selectedTables.get(currentTableIndex);
+
+ return hasNext();
}
@Override
public IRawRecord<char[]> next() throws IOException, InterruptedException {
CharArrayRecord record = new CharArrayRecord();
- record.append((formatRecord(dataGeneratorIterator.next())).toCharArray());
+
+ if (getFromParent) {
+ record.append((formatRecord(tableIterators.get(currentTableIndex).next())).toCharArray());
+ } else {
+ record.append(childRow.toCharArray());
+ childRow = null; // Always reset the child row after writing it
+ getFromParent = true;
+ }
+
record.endRecord();
return record;
}
/**
- * Create the data generator iterator with the specified properties passed to the session.
+ * Builds the string record from the generated values by the data generator. The column name for each value is
+ * extracted from the table from which the data is being generated.
*
- * @return A lazy iterator to generate the data based on the specified properties.
+ * @param values List containing all the generated column values
+ *
+ * @return The built string record from the generated values
*/
- private Iterator<List<List<String>>> getDataGeneratorIterator() throws HyracksDataException {
- selectedTable = getTableFromStringTableName(tableName);
+ private String formatRecord(List<List<String>> values) {
+ // Clear the builder (This is faster than re-creating the builder each iteration)
+ builder.setLength(0);
- // Create the session with the specified properties, the sessions also specifies the chunk to be generated
- Session session = Session.getDefaultSession().withTable(selectedTable).withScale(scalingFactor)
- .withParallelism(parallelism).withChunkNumber(chunkNumber);
+ builder.append("{\"tableName\":\"");
+ builder.append(currentTable.toString());
+ builder.append("\"");
- // Construct the Results and Results iterator
- Results results = Results.constructResults(selectedTable, session);
- return results.iterator();
+ // Build the record data
+ for (int counter = 0; counter < values.get(0).size(); counter++) {
+ builder.append(",\"");
+ builder.append(currentTable.getColumns()[counter].getName());
+ builder.append("\":\"");
+ builder.append(values.get(0).get(counter));
+ builder.append("\"");
+ }
+
+ // Close the record
+ builder.append("}");
+
+ // Reference to the parent row to be returned, before resetting the builder again
+ String parentRow = builder.toString();
+
+ // In some cases, the generate generates 2 records, one for current table, and one for child table. If a child
+ // record is generated, we're gonna add it to a list, and start pushing it once all the parent records
+ // are done
+ if (generateAllTables && values.size() > 1) {
+ builder.setLength(0);
+ builder.append("{\"tableName\":\"");
+ builder.append(currentTable.getChild().toString());
+ builder.append("\"");
+
+ // Build the record data
+ for (int counter = 0; counter < values.get(1).size(); counter++) {
+ builder.append(",\"");
+ builder.append(currentTable.getChild().getColumns()[counter].getName());
+ builder.append("\":\"");
+ builder.append(values.get(0).get(counter));
+ builder.append("\"");
+ }
+
+ // Close the record
+ builder.append("}");
+
+ // Add it to the children rows list
+ childRow = builder.toString();
+ }
+
+ // Return parent row
+ return parentRow;
}
/**
@@ -109,55 +199,91 @@ public class TPCDSDataGeneratorReader extends FunctionReader {
* @param tableName String table name to search for.
* @return Table if found, throws an exception otherwise.
*/
- private Table getTableFromStringTableName(String tableName) throws HyracksDataException {
+ private List<Table> getTableFromStringTableName(String tableName) throws HyracksDataException {
+
+ // Get all the tables
+ if (generateAllTables) {
+ // Remove the DBGEN_VERSION table and all children tables, parent tables will generate them
+ return Table.getBaseTables().stream()
+ .filter(table -> !table.equals(Table.DBGEN_VERSION) && !table.isChild())
+ .collect(Collectors.toList());
+ }
+ // Search for the table
List<Table> matchedTables = Table.getBaseTables().stream()
.filter(table -> tableName.equalsIgnoreCase(table.getName())).collect(Collectors.toList());
// Ensure the table was found
if (matchedTables.size() != 1) {
- throw new RuntimeDataException(ErrorCode.TPCDS_INVALID_TABLE_NAME, getIdentifier().getName(), tableName);
+ throw new RuntimeDataException(ErrorCode.TPCDS_INVALID_TABLE_NAME, getFunctionIdentifier().getName(),
+ tableName);
}
- return matchedTables.get(0);
+ return matchedTables;
}
/**
- * Builds the string record from the generated values by the data generator. The column name for each value is
- * extracted from the table from which the data is being generated.
+ * As the TPC-DS library has constraints on activating the parallelism (table must be generating 1,000,000 records
+ * based on a scaling factor), we're gonna override that behavior and calculate the parallelism manually. This
+ * will ensure the activation of the parallelism regardless of the data size being generated.
*
- * @param values List containing all the generated column values
+ * @param table table to generate the data for
+ * @param session session containing the parallelism and scaling information
*
- * @return The built string record from the generated values
+ * @return Results that holds a lazy-iterator to generate the data based on the calculated parameters.
*/
- private String formatRecord(List<List<String>> values) {
- // Clear the builder (This is faster than re-creating the builder each iteration)
- builder.setLength(0);
+ private Results calculateParallelism(Table table, Session session) {
- int counter;
- builder.append("{");
+ // Total and parallelism level
+ long total = session.getScaling().getRowCount(table);
+ int parallelism = session.getParallelism();
- // We loop only to the item before the last, then add the last item manually to avoid appending the ","
- // at the end, this way we avoid constantly checking if we're at the last item or substring the whole record
- for (counter = 0; counter < values.get(0).size() - 1; counter++) {
- builder.append("\"");
- builder.append(selectedTable.getColumns()[counter].getName());
- builder.append("\":\"");
- builder.append(values.get(0).get(counter));
- builder.append("\",");
+ // Row set size to be generated for each partition
+ long rowSetSize = total / parallelism;
+
+ // Special case: WEB_SITE table sometimes relies on the previous records, this could be a problem if the
+ // previous record is on a different thread. Since it's a small table, we'll generate it all on the first
+ // thread and let the other threads generate nothing
+ if (table.equals(Table.WEB_SITE)) {
+ if (session.getChunkNumber() - 1 == 0) {
+ return new Results(table, 1, total, session);
+ }
+ // Don't generate anything on other partition (start > end)
+ else {
+ return new Results(table, 2, 1, session);
+ }
}
- // This is the last item to be appended, don't append the "," after appending the field
- builder.append("\"");
- builder.append(selectedTable.getColumns()[counter].getName());
- builder.append("\":\"");
- builder.append(values.get(0).get(counter));
- builder.append("\"}");
+ // Special case: For very small tables, if the rowSetSize ends up being 1, this will cause an issue in the
+ // parallelism, so we'll just let the first thread do all the work
+ if (rowSetSize == 1) {
+ if (session.getChunkNumber() - 1 == 0) {
+ return new Results(table, 1, total, session);
+ }
+ // Don't generate anything on other partition (start > end)
+ else {
+ return new Results(table, 2, 1, session);
+ }
+ }
+
+ // Start and end calculated for each partition
+ long startRow = (session.getChunkNumber() - 1) * rowSetSize + 1;
+ long rowCount = startRow + rowSetSize - 1;
+
+ // Any extra rows (not evenly divided) will be done by the last partition
+ if (session.getChunkNumber() == parallelism) {
+ rowCount += total % parallelism;
+ }
- return builder.toString();
+ return new Results(table, startRow, rowCount, session);
}
- private FunctionIdentifier getIdentifier() {
- return TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR;
+ /**
+ * Gets the function identifier
+ *
+ * @return function identifier
+ */
+ private FunctionIdentifier getFunctionIdentifier() {
+ return functionIdentifier;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
similarity index 71%
rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index cb7e357..968e289 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -18,27 +18,32 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
/**
* This TPC-DS function is used to generate data with accordance to the specifications of the TPC Benchmark DS.
*/
-public class TPCDSDataGeneratorDatasource extends FunctionDataSource {
+
+public class TPCDSSingleTableDataGeneratorDatasource extends FunctionDataSource {
private final String tableName;
private final double scalingFactor;
+ private final FunctionIdentifier functionIdentifier;
- public TPCDSDataGeneratorDatasource(INodeDomain domain, String tableName, double scalingFactor)
- throws AlgebricksException {
+ TPCDSSingleTableDataGeneratorDatasource(INodeDomain domain, String tableName, double scalingFactor,
+ FunctionIdentifier functionIdentifier) throws AlgebricksException {
super(createDataSourceId(tableName, scalingFactor), domain);
this.tableName = tableName;
this.scalingFactor = scalingFactor;
+ this.functionIdentifier = functionIdentifier;
}
/**
@@ -53,13 +58,19 @@ public class TPCDSDataGeneratorDatasource extends FunctionDataSource {
* @return A DataSourceId that's based on the function details and its parameters
*/
private static DataSourceId createDataSourceId(String tableName, double scalingFactor) {
- return new DataSourceId(TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR.getNamespace(),
- TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR.getName(), tableName, Double.toString(scalingFactor));
+ return new DataSourceId(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getNamespace(),
+ TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR.getName(), tableName,
+ Double.toString(scalingFactor));
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new TPCDSDataGeneratorFunction(locations, tableName, scalingFactor);
+ return new TPCDSDataGeneratorFunction(locations, tableName, scalingFactor, functionIdentifier);
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return new AlgebricksAbsolutePartitionConstraint(csm.getClusterLocations().getLocations());
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorRewriter.java
similarity index 74%
rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorRewriter.java
index fb871b9..fb7142d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSDataGeneratorRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorRewriter.java
@@ -43,17 +43,21 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
/**
* This TPC-DS function is used to generate data with accordance to the specifications of the TPC Benchmark DS.
*
- * The data generator takes 2 arguments:
- * - first argument: a valid table name.
- * - second argument: the desired scaling factor.
+ * This version of the function takes 2 arguments:
+ * - Table name that decides the table to generate the data for.
+ * - Scaling factor that decides the data size to be generated.
+ *
+ * This function will generate the data only for the specified table.
*/
-public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
- public static final FunctionIdentifier TPCDS_DATA_GENERATOR =
+public class TPCDSSingleTableDataGeneratorRewriter extends FunctionRewriter {
+
+ public static final FunctionIdentifier TPCDS_SINGLE_TABLE_DATA_GENERATOR =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "tpcds-datagen", 2);
- public static final TPCDSDataGeneratorRewriter INSTANCE = new TPCDSDataGeneratorRewriter(TPCDS_DATA_GENERATOR);
+ public static final TPCDSSingleTableDataGeneratorRewriter INSTANCE =
+ new TPCDSSingleTableDataGeneratorRewriter(TPCDS_SINGLE_TABLE_DATA_GENERATOR);
- private TPCDSDataGeneratorRewriter(FunctionIdentifier functionId) {
+ private TPCDSSingleTableDataGeneratorRewriter(FunctionIdentifier functionId) {
super(functionId);
}
@@ -61,7 +65,6 @@ public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
protected FunctionDataSource toDatasource(IOptimizationContext context, AbstractFunctionCallExpression function)
throws AlgebricksException {
- // TODO(Hussain) check if it is safe to assume that we're receiving constant expressions only as arguments.
UnnestingFunctionCallExpression functionCall = (UnnestingFunctionCallExpression) function;
ConstantExpression tableNameArgument = (ConstantExpression) functionCall.getArguments().get(0).getValue();
ConstantExpression scalingFactorArgument = (ConstantExpression) functionCall.getArguments().get(1).getValue();
@@ -75,21 +78,22 @@ public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
IAType scalingFactorType = scalingFactorArgumentValue.getType();
if (tableNameType.getTypeTag() != ATypeTag.STRING) {
- throw new TypeMismatchException(functionCall.getFunctionIdentifier(), 0, tableNameType.getTypeTag(),
- ATypeTag.STRING);
+ throw new TypeMismatchException(getFunctionIdentifier(), 0, tableNameType.getTypeTag(), ATypeTag.STRING);
}
// Ensure the scaling factor can be promoted to double
if (!ATypeHierarchy.canPromote(scalingFactorType.getTypeTag(), ATypeTag.DOUBLE)) {
- throw new TypeMismatchException(TPCDS_DATA_GENERATOR, 1, scalingFactorType.getTypeTag(), ATypeTag.TINYINT,
- ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT, ATypeTag.DOUBLE);
+ throw new TypeMismatchException(getFunctionIdentifier(), 1, scalingFactorType.getTypeTag(),
+ ATypeTag.TINYINT, ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT,
+ ATypeTag.DOUBLE);
}
// Convert whichever number type we received into double
double scalingFactor = getScalingFactor(scalingFactorArgumentValue);
String tableName = ((AString) tableNameArgumentValue).getStringValue();
- return new TPCDSDataGeneratorDatasource(context.getComputationNodeDomain(), tableName, scalingFactor);
+ return new TPCDSSingleTableDataGeneratorDatasource(context.getComputationNodeDomain(), tableName, scalingFactor,
+ getFunctionIdentifier());
}
/**
@@ -114,8 +118,18 @@ public class TPCDSDataGeneratorRewriter extends FunctionRewriter {
case DOUBLE:
return ((ADouble) value).getDoubleValue();
default:
- throw new TypeMismatchException(TPCDS_DATA_GENERATOR, 1, value.getType().getTypeTag(), ATypeTag.TINYINT,
- ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT, ATypeTag.DOUBLE);
+ throw new TypeMismatchException(getFunctionIdentifier(), 1, value.getType().getTypeTag(),
+ ATypeTag.TINYINT, ATypeTag.SMALLINT, ATypeTag.INTEGER, ATypeTag.BIGINT, ATypeTag.FLOAT,
+ ATypeTag.DOUBLE);
}
}
+
+ /**
+ * Gets the function identifier
+ *
+ * @return function identifier
+ */
+ private FunctionIdentifier getFunctionIdentifier() {
+ return functionId;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
index c708cd1..26726f6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java
@@ -27,7 +27,8 @@ import org.apache.asterix.app.function.FeedRewriter;
import org.apache.asterix.app.function.JobSummariesRewriter;
import org.apache.asterix.app.function.PingRewriter;
import org.apache.asterix.app.function.StorageComponentsRewriter;
-import org.apache.asterix.app.function.TPCDSDataGeneratorRewriter;
+import org.apache.asterix.app.function.TPCDSAllTablesDataGeneratorRewriter;
+import org.apache.asterix.app.function.TPCDSSingleTableDataGeneratorRewriter;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.utils.RecordUtil;
@@ -60,11 +61,16 @@ public class MetadataBuiltinFunctions {
BuiltinFunctions.addUnnestFun(PingRewriter.PING, true);
BuiltinFunctions.addDatasourceFunction(PingRewriter.PING, PingRewriter.INSTANCE);
// TPC-DS data generation function
- BuiltinFunctions.addPrivateFunction(TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR,
+ BuiltinFunctions.addPrivateFunction(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
- BuiltinFunctions.addUnnestFun(TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR, true);
- BuiltinFunctions.addDatasourceFunction(TPCDSDataGeneratorRewriter.TPCDS_DATA_GENERATOR,
- TPCDSDataGeneratorRewriter.INSTANCE);
+ BuiltinFunctions.addUnnestFun(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR, true);
+ BuiltinFunctions.addDatasourceFunction(TPCDSSingleTableDataGeneratorRewriter.TPCDS_SINGLE_TABLE_DATA_GENERATOR,
+ TPCDSSingleTableDataGeneratorRewriter.INSTANCE);
+ BuiltinFunctions.addPrivateFunction(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR,
+ (expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
+ BuiltinFunctions.addUnnestFun(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR, true);
+ BuiltinFunctions.addDatasourceFunction(TPCDSAllTablesDataGeneratorRewriter.TPCDS_ALL_TABLES_DATA_GENERATOR,
+ TPCDSAllTablesDataGeneratorRewriter.INSTANCE);
// Active requests function
BuiltinFunctions.addFunction(ActiveRequestsRewriter.ACTIVE_REQUESTS,
(expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/TPCExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/TPCExecutionTest.java
new file mode 100644
index 0000000..f777bd5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/TPCExecutionTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the Transaction Processing Council (TPC) runtime tests with the storage parallelism.
+ */
+
+@RunWith(Parameterized.class)
+public class TPCExecutionTest {
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new TestExecutor();
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "TPCExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_tpc.xml", "testsuite_tpc.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public TPCExecutionTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static void setNcEndpoints(TestExecutor testExecutor) {
+ final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
+ final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ final String ip = InetAddress.getLoopbackAddress().getHostAddress();
+ for (NodeControllerService nc : ncs) {
+ final String nodeId = nc.getId();
+ final INcApplicationContext appCtx = (INcApplicationContext) nc.getApplicationContext();
+ int apiPort = appCtx.getExternalProperties().getNcApiPort();
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort));
+ }
+ testExecutor.setNcEndPoints(ncEndPoints);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/only_tpc.xml b/asterixdb/asterix-app/src/test/resources/runtimets/only_tpc.xml
new file mode 100644
index 0000000..334dd52
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/only_tpc.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="failed">
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables/datagen_sf_1_all_tables.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables_1/datagen_sf_1_all_tables_1.1.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables/datagen_sf_1_all_tables.1.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables_1/datagen_sf_1_all_tables_1.1.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.query.sqlpp
new file mode 100644
index 0000000..303c1c7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+set `import-private-functions` `true`;
+
+select d.tableName, count(*) as count
+from tpcds_datagen(1) as d
+group by d.tableName
+order by d.tableName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables/datagen_sf_1_all_tables.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables_1/datagen_sf_1_all_tables_1.1.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables/datagen_sf_1_all_tables.1.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables_1/datagen_sf_1_all_tables_1.1.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.adm
new file mode 100644
index 0000000..4040b6b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/tpcds/datagen_sf_1_all_tables_2/datagen_sf_1_all_tables_2.1.adm
@@ -0,0 +1,24 @@
+{ "count": 6, "tableName": "CALL_CENTER" }
+{ "count": 11718, "tableName": "CATALOG_PAGE" }
+{ "count": 144067, "tableName": "CATALOG_RETURNS" }
+{ "count": 1441548, "tableName": "CATALOG_SALES" }
+{ "count": 100000, "tableName": "CUSTOMER" }
+{ "count": 50000, "tableName": "CUSTOMER_ADDRESS" }
+{ "count": 1920800, "tableName": "CUSTOMER_DEMOGRAPHICS" }
+{ "count": 73049, "tableName": "DATE_DIM" }
+{ "count": 7200, "tableName": "HOUSEHOLD_DEMOGRAPHICS" }
+{ "count": 20, "tableName": "INCOME_BAND" }
+{ "count": 11745000, "tableName": "INVENTORY" }
+{ "count": 18000, "tableName": "ITEM" }
+{ "count": 300, "tableName": "PROMOTION" }
+{ "count": 35, "tableName": "REASON" }
+{ "count": 20, "tableName": "SHIP_MODE" }
+{ "count": 12, "tableName": "STORE" }
+{ "count": 287514, "tableName": "STORE_RETURNS" }
+{ "count": 2880404, "tableName": "STORE_SALES" }
+{ "count": 86400, "tableName": "TIME_DIM" }
+{ "count": 5, "tableName": "WAREHOUSE" }
+{ "count": 60, "tableName": "WEB_PAGE" }
+{ "count": 71763, "tableName": "WEB_RETURNS" }
+{ "count": 719384, "tableName": "WEB_SALES" }
+{ "count": 30, "tableName": "WEB_SITE" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 282c03a..5b2cacf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -9084,18 +9084,6 @@
</test-group>
<test-group name="tpcds">
<test-case FilePath="tpcds">
- <compilation-unit name="datagen_sf_1_invalid_table">
- <output-dir compare="Text">datagen_sf_1_invalid_table</output-dir>
- <expected-error>is not a TPC-DS table name</expected-error>
- <source-location>false</source-location>
- </compilation-unit>
- </test-case>
- <test-case FilePath="tpcds">
- <compilation-unit name="datagen_sf_1_small_tables">
- <output-dir compare="Text">datagen_sf_1_small_tables</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="tpcds">
<compilation-unit name="query-ASTERIXDB-1580">
<output-dir compare="Text">query-ASTERIXDB-1580</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_tpc.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_tpc.xml
new file mode 100644
index 0000000..d21fb33
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_tpc.xml
@@ -0,0 +1,45 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp" SourceLocation="true">
+ <test-group name="tcpds">
+ <test-case FilePath="tpcds">
+ <compilation-unit name="datagen_sf_1_all_tables_1">
+ <output-dir compare="Text">datagen_sf_1_all_tables_1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="datagen_sf_1_all_tables_2">
+ <output-dir compare="Text">datagen_sf_1_all_tables_2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="datagen_sf_1_small_tables">
+ <output-dir compare="Text">datagen_sf_1_small_tables</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="tpcds">
+ <compilation-unit name="datagen_sf_1_invalid_table">
+ <output-dir compare="Text">datagen_sf_1_invalid_table</output-dir>
+ <expected-error>is not a TPC-DS table name</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 3b5cf2e..1635507 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -91,7 +91,7 @@ public abstract class FunctionDataSource extends DataSource {
protected abstract IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations);
- protected static AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
String[] allPartitions = csm.getClusterLocations().getLocations();
Set<String> ncs = new HashSet<>(Arrays.asList(allPartitions));
return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new String[ncs.size()]));