You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2017/02/02 16:54:01 UTC
[7/8] asterixdb git commit: RAT Cleanup, Resulting LICENSE fixes
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
deleted file mode 100644
index d8b79df..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
+++ /dev/null
@@ -1 +0,0 @@
-(function(B){var A={series:{threshold:null}};function C(D){function E(L,S,M){if(!S.threshold){return }var F=M.pointsize,I,O,N,G,K,H=B.extend({},S);H.datapoints={points:[],pointsize:F};H.label=null;H.color=S.threshold.color;H.threshold=null;H.originSeries=S;H.data=[];var P=S.threshold.below,Q=M.points,R=S.lines.show;threspoints=[];newpoints=[];for(I=0;I<Q.length;I+=F){O=Q[I];N=Q[I+1];K=G;if(N<P){G=threspoints}else{G=newpoints}if(R&&K!=G&&O!=null&&I>0&&Q[I-F]!=null){var J=(O-Q[I-F])/(N-Q[I-F+1])*(P-N)+O;K.push(J);K.push(P);for(m=2;m<F;++m){K.push(Q[I+m])}G.push(null);G.push(null);for(m=2;m<F;++m){G.push(Q[I+m])}G.push(J);G.push(P);for(m=2;m<F;++m){G.push(Q[I+m])}}G.push(O);G.push(N)}M.points=newpoints;H.datapoints.points=threspoints;if(H.datapoints.points.length>0){L.getData().push(H)}}D.hooks.processDatapoints.push(E)}B.plot.plugins.push({init:C,options:A,name:"threshold",version:"1.0"})})(jQuery);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
index 8caa9ec..6cfaaa5 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
@@ -41,6 +41,20 @@
<root.dir>${basedir}/../..</root.dir>
</properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/data/beer.txt</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
index 73086cd..b88d292 100644
--- a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
@@ -50,7 +50,7 @@
<executions>
<execution>
<id>copy-scripts</id>
- <phase>package</phase>
+ <phase>generate-test-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
@@ -61,7 +61,6 @@
<directory>src/main/resources</directory>
</resource>
</resources>
- <directoryMode>0755</directoryMode>
</configuration>
</execution>
</executions>
@@ -73,7 +72,7 @@
<executions>
<execution>
<id>process-test-classes</id>
- <phase>package</phase>
+ <phase>process-test-resources</phase>
<configuration>
<target>
<chmod file="target/appassembler/bin/*" perm="755" />
@@ -85,6 +84,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/main/resources/conf/master</exclude>
+ <exclude>src/main/resources/conf/slaves</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
index d1f1c70..21e22ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
@@ -37,6 +37,15 @@
<skip>true</skip>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>data/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
<properties>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
index 1327912..394fa11 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
@@ -99,6 +99,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>data/file1.txt</exclude>
+ <exclude>data/file2.txt</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
<profiles>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
index 9c25d13..b4abc4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
@@ -33,25 +33,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>2.5</version>
- <configuration>
- <filesets>
- <fileset>
- <directory>.</directory>
- <includes>
- <include>edu*</include>
- <include>actual*</include>
- <include>build*</include>
- <include>expect*</include>
- <include>ClusterController*</include>
- <include>edu.uci.*</include>
- </includes>
- </fileset>
- </filesets>
- </configuration>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.2</version>
@@ -81,6 +62,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/data/customer.tbl</exclude>
+ <exclude>src/test/resources/expected/part-0</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
@@ -409,5 +400,11 @@
<artifactId>commons-io</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
index c5ca25e..2307a43 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
@@ -19,6 +19,8 @@
package org.apache.hyracks.hdfs.dataflow;
+import static org.apache.hyracks.test.support.TestUtils.joinPath;
+
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@@ -67,15 +69,19 @@ import org.apache.hyracks.hdfs.utils.TestUtils;
@SuppressWarnings({ "deprecation" })
public class DataflowTest extends TestCase {
- private static final String ACTUAL_RESULT_DIR = "actual";
- private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected";
- private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+ protected static final String ACTUAL_RESULT_DIR = joinPath("target", "actual");
+ private static final String TEST_RESOURCES = joinPath("src", "test", "resources");
+ protected static final String EXPECTED_RESULT_PATH = joinPath(TEST_RESOURCES, "expected");
+ private static final String PATH_TO_HADOOP_CONF = joinPath(TEST_RESOURCES, "hadoop", "conf");
+ protected static final String BUILD_DIR = joinPath("target", "build");
+
- private static final String DATA_PATH = "src/test/resources/data/customer.tbl";
- private static final String HDFS_INPUT_PATH = "/customer/";
- private static final String HDFS_OUTPUT_PATH = "/customer_result/";
+ private static final String DATA_PATH = joinPath(TEST_RESOURCES, "data", "customer.tbl");
+ protected static final String HDFS_INPUT_PATH = "/customer/";
+ protected static final String HDFS_OUTPUT_PATH = "/customer_result/";
private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
+ private static final String MINIDFS_BASEDIR = joinPath("target", "hdfs");
private MiniDFSCluster dfsCluster;
private JobConf conf = new JobConf();
@@ -91,10 +97,16 @@ public class DataflowTest extends TestCase {
}
private void cleanupStores() throws IOException {
- FileUtils.forceMkdir(new File("teststore"));
- FileUtils.forceMkdir(new File("build"));
- FileUtils.cleanDirectory(new File("teststore"));
- FileUtils.cleanDirectory(new File("build"));
+ FileUtils.forceMkdir(new File(MINIDFS_BASEDIR));
+ FileUtils.cleanDirectory(new File(MINIDFS_BASEDIR));
+ }
+
+ protected Configuration getConfiguration() {
+ return conf;
+ }
+
+ protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws IOException {
+ return new MiniDFSCluster(conf, numberOfNC, true, null);
}
/**
@@ -102,16 +114,17 @@ public class DataflowTest extends TestCase {
*
* @throws IOException
*/
- private void startHDFS() throws IOException {
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
+ protected void startHDFS() throws IOException {
+ getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
+ getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
+ getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
FileSystem lfs = FileSystem.getLocal(new Configuration());
- lfs.delete(new Path("build"), true);
- System.setProperty("hadoop.log.dir", "logs");
- dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
- FileSystem dfs = FileSystem.get(conf);
+ lfs.delete(new Path(BUILD_DIR), true);
+ System.setProperty("hadoop.log.dir", joinPath("target", "logs"));
+ getConfiguration().set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR);
+ dfsCluster = getMiniDFSCluster(getConfiguration(), numberOfNC);
+ FileSystem dfs = FileSystem.get(getConfiguration());
Path src = new Path(DATA_PATH);
Path dest = new Path(HDFS_INPUT_PATH);
Path result = new Path(HDFS_OUTPUT_PATH);
@@ -120,7 +133,7 @@ public class DataflowTest extends TestCase {
dfs.copyFromLocalFile(src, dest);
DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
- conf.writeXml(confOutput);
+ getConfiguration().writeXml(confOutput);
confOutput.flush();
confOutput.close();
}
@@ -178,14 +191,14 @@ public class DataflowTest extends TestCase {
* @return true if correct
* @throws Exception
*/
- private boolean checkResults() throws Exception {
- FileSystem dfs = FileSystem.get(conf);
+ protected boolean checkResults() throws Exception {
+ FileSystem dfs = FileSystem.get(getConfiguration());
Path result = new Path(HDFS_OUTPUT_PATH);
Path actual = new Path(ACTUAL_RESULT_DIR);
dfs.copyToLocalFile(result, actual);
- TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File(
- ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0"));
+ TestUtils.compareWithResult(new File(joinPath(EXPECTED_RESULT_PATH, "part-0")), new File(
+ joinPath(ACTUAL_RESULT_DIR, "customer_result", "part-0")));
return true;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
index 09e3a35..d522b2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
@@ -29,7 +29,7 @@ import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.client.NodeStatus;
import org.apache.hyracks.api.comm.NetworkAddress;
-public class TestUtils {
+public class TestUtils extends org.apache.hyracks.test.support.TestUtils {
public static void compareWithResult(File expectedFile, File actualFile) throws Exception {
BufferedReader readerExpected = new BufferedReader(new FileReader(expectedFile));
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
index fb6f32e..3c9b1c0 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
@@ -19,18 +19,10 @@
package org.apache.hyracks.hdfs2.dataflow;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.util.List;
import junit.framework.Assert;
-import junit.framework.TestCase;
-
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -40,7 +32,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;
-
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -48,6 +39,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
@@ -61,73 +53,32 @@ import org.apache.hyracks.hdfs.lib.RawBinaryHashFunctionFactory;
import org.apache.hyracks.hdfs.lib.TextKeyValueParserFactory;
import org.apache.hyracks.hdfs.lib.TextTupleWriterFactory;
import org.apache.hyracks.hdfs.utils.HyracksUtils;
-import org.apache.hyracks.hdfs.utils.TestUtils;
import org.apache.hyracks.hdfs2.scheduler.Scheduler;
/**
* Test the org.apache.hyracks.hdfs2.dataflow package,
* the operators for the Hadoop new API.
*/
-public class DataflowTest extends TestCase {
-
- private static final String ACTUAL_RESULT_DIR = "actual";
- private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected";
- private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+public class DataflowTest extends org.apache.hyracks.hdfs.dataflow.DataflowTest {
- private static final String DATA_PATH = "src/test/resources/data/customer.tbl";
- private static final String HDFS_INPUT_PATH = "/customer/";
- private static final String HDFS_OUTPUT_PATH = "/customer_result/";
-
- private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
- private MiniDFSCluster dfsCluster;
private MiniDFSClusterFactory dfsClusterFactory = new MiniDFSClusterFactory();
private Job conf;
- private int numberOfNC = 2;
@Override
public void setUp() throws Exception {
conf = new Job();
- cleanupStores();
- HyracksUtils.init();
- FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
- FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
- startHDFS();
+ super.setUp();
}
- private void cleanupStores() throws IOException {
- FileUtils.forceMkdir(new File("teststore"));
- FileUtils.forceMkdir(new File("build"));
- FileUtils.cleanDirectory(new File("teststore"));
- FileUtils.cleanDirectory(new File("build"));
+ @Override
+ protected Configuration getConfiguration() {
+ return conf.getConfiguration();
}
- /**
- * Start the HDFS cluster and setup the data files
- *
- * @throws IOException
- */
- private void startHDFS() throws IOException {
- conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
- conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
- conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
-
- FileSystem lfs = FileSystem.getLocal(new Configuration());
- lfs.delete(new Path("build"), true);
- System.setProperty("hadoop.log.dir", "logs");
- dfsCluster = dfsClusterFactory.getMiniDFSCluster(conf.getConfiguration(), numberOfNC);
- FileSystem dfs = FileSystem.get(conf.getConfiguration());
- Path src = new Path(DATA_PATH);
- Path dest = new Path(HDFS_INPUT_PATH);
- Path result = new Path(HDFS_OUTPUT_PATH);
- dfs.mkdirs(dest);
- dfs.mkdirs(result);
- dfs.copyFromLocalFile(src, dest);
-
- DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
- conf.getConfiguration().writeXml(confOutput);
- confOutput.flush();
- confOutput.close();
+ @Override
+ protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws HyracksDataException {
+ return dfsClusterFactory.getMiniDFSCluster(conf, numberOfNC);
}
/**
@@ -142,7 +93,7 @@ public class DataflowTest extends TestCase {
conf.setInputFormatClass(TextInputFormat.class);
Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
- InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), conf.getConfiguration());
+ InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), getConfiguration());
List<InputSplit> splits = inputFormat.getSplits(conf);
String[] readSchedule = scheduler.getLocationConstraints(splits);
@@ -178,35 +129,4 @@ public class DataflowTest extends TestCase {
Assert.assertEquals(true, checkResults());
}
-
- /**
- * Check if the results are correct
- *
- * @return true if correct
- * @throws Exception
- */
- private boolean checkResults() throws Exception {
- FileSystem dfs = FileSystem.get(conf.getConfiguration());
- Path result = new Path(HDFS_OUTPUT_PATH);
- Path actual = new Path(ACTUAL_RESULT_DIR);
- dfs.copyToLocalFile(result, actual);
-
- TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File(
- ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0"));
- return true;
- }
-
- /**
- * cleanup hdfs cluster
- */
- private void cleanupHDFS() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Override
- public void tearDown() throws Exception {
- HyracksUtils.deinit();
- cleanupHDFS();
- }
-
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
index 1628544..be43c5d 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
@@ -1,4 +1,5 @@
<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
! Licensed to the Apache Software Foundation (ASF) under one
! or more contributor license agreements. See the NOTICE file
@@ -18,8 +19,6 @@
! under the License.
!-->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
<!-- Put site-specific property overrides in this file. -->
<configuration>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
index 8784730..9dca49a 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
@@ -1,4 +1,5 @@
<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
! Licensed to the Apache Software Foundation (ASF) under one
! or more contributor license agreements. See the NOTICE file
@@ -18,8 +19,6 @@
! under the License.
!-->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
<!-- Put site-specific property overrides in this file. -->
<configuration>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
index 3e8bfe4..e765c65 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
@@ -1,4 +1,5 @@
<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
! Licensed to the Apache Software Foundation (ASF) under one
! or more contributor license agreements. See the NOTICE file
@@ -18,8 +19,6 @@
! under the License.
!-->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
<!-- Put site-specific property overrides in this file. -->
<configuration>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
deleted file mode 100644
index e031a51..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.IOException;
-import java.util.Random;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-
-/**
- * Quick & dirty data generator for multi-thread testing.
- */
-@SuppressWarnings("rawtypes")
-public class DataGenThread extends Thread {
- public final BlockingQueue<TupleBatch> tupleBatchQueue;
- private final int maxNumBatches;
- private final int maxOutstandingBatches;
- private int numBatches = 0;
- private final Random rnd;
-
- // maxOutstandingBatches pre-created tuple-batches for populating the queue.
- private TupleBatch[] tupleBatches;
- private int ringPos;
-
- public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes,
- int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) {
- this.maxNumBatches = maxNumBatches;
- this.maxOutstandingBatches = maxOutstandingBatches;
- rnd = new Random(rndSeed);
- tupleBatches = new TupleBatch[maxOutstandingBatches];
- IFieldValueGenerator[] fieldGens = DataGenUtils.getFieldGensFromSerdes(fieldSerdes, rnd, sorted);
- for (int i = 0; i < maxOutstandingBatches; i++) {
- tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize);
- }
- tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches);
- ringPos = 0;
- }
-
- public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes,
- IFieldValueGenerator[] fieldGens, int rndSeed, int maxOutstandingBatches) {
- this.maxNumBatches = maxNumBatches;
- this.maxOutstandingBatches = maxOutstandingBatches;
- rnd = new Random(rndSeed);
- tupleBatches = new TupleBatch[maxOutstandingBatches];
- for (int i = 0; i < maxOutstandingBatches; i++) {
- tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, 0);
- }
- tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches);
- ringPos = 0;
- }
-
- @Override
- public void run() {
- while (numBatches < maxNumBatches) {
- boolean added = false;
- try {
- if (tupleBatches[ringPos].inUse.compareAndSet(false, true)) {
- tupleBatches[ringPos].generate();
- tupleBatchQueue.put(tupleBatches[ringPos]);
- added = true;
- }
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- if (added) {
- numBatches++;
- ringPos++;
- if (ringPos >= maxOutstandingBatches) {
- ringPos = 0;
- }
- }
- }
- }
-
- public TupleBatch getBatch() throws InterruptedException {
- return tupleBatchQueue.take();
- }
-
- public void releaseBatch(TupleBatch batch) {
- batch.inUse.set(false);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
deleted file mode 100644
index 130f7e2..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-
-@SuppressWarnings("rawtypes")
-public class DataGenUtils {
- public static IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde, Random rnd, boolean sorted) {
- if (serde instanceof IntegerSerializerDeserializer) {
- if (sorted) {
- return new SortedIntegerFieldValueGenerator();
- } else {
- return new IntegerFieldValueGenerator(rnd);
- }
- } else if (serde instanceof FloatSerializerDeserializer) {
- if (sorted) {
- return new SortedFloatFieldValueGenerator();
- } else {
- return new FloatFieldValueGenerator(rnd);
- }
- } else if (serde instanceof DoubleSerializerDeserializer) {
- if (sorted) {
- return new SortedDoubleFieldValueGenerator();
- } else {
- return new DoubleFieldValueGenerator(rnd);
- }
- } else if (serde instanceof UTF8StringSerializerDeserializer) {
- return new StringFieldValueGenerator(20, rnd);
- }
- return null;
- }
-
- public static IFieldValueGenerator[] getFieldGensFromSerdes(ISerializerDeserializer[] serdes, Random rnd,
- boolean sorted) {
- IFieldValueGenerator[] fieldValueGens = new IFieldValueGenerator[serdes.length];
- for (int i = 0; i < serdes.length; i++) {
- fieldValueGens[i] = getFieldGenFromSerde(serdes[i], rnd, sorted);
- }
- return fieldValueGens;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
deleted file mode 100644
index 1e8077c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class DocumentStringFieldValueGenerator implements IFieldValueGenerator<String> {
- private final String FIRST_NAMES_FILE = "dist.all.first.cleaned";
- private final String LAST_NAMES_FILE = "dist.all.last.cleaned";
-
- private final int docMinWords;
- private final int docMaxWords;
- private final int maxDictionarySize;
- private final Random rnd;
- private int[] cumulIntRanges;
-
- private List<String> tokenDict = new ArrayList<String>();
-
- public DocumentStringFieldValueGenerator(int docMinWords, int docMaxWords, int maxDictionarySize, Random rnd)
- throws IOException {
- this.docMinWords = docMinWords;
- this.docMaxWords = docMaxWords;
- this.maxDictionarySize = maxDictionarySize;
- this.rnd = rnd;
- initDictionary();
- double[] zipfProbDist = ProbabilityHelper.getZipfProbDist(tokenDict.size(), 1);
- cumulIntRanges = ProbabilityHelper.getCumulIntRanges(zipfProbDist);
- }
-
- private void initDictionary() throws IOException {
- String line;
- int count = 0;
-
- // Read first names from data file.
- InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE);
- BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn));
- try {
- while (count < maxDictionarySize && (line = firstNamesReader.readLine()) != null) {
- tokenDict.add(line.trim());
- count++;
- }
- } finally {
- firstNamesReader.close();
- }
-
- // Read last names from data file.
- InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE);
- BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn));
- try {
- while (count < maxDictionarySize && (line = lastNamesReader.readLine()) != null) {
- tokenDict.add(line.trim());
- count++;
- }
- } finally {
- lastNamesReader.close();
- }
- }
-
- @Override
- public String next() {
- StringBuilder strBuilder = new StringBuilder();
- int numWords = MathUtil.stripSignBit(rnd.nextInt()) % (docMaxWords - docMinWords + 1) + docMinWords;
- for (int i = 0; i < numWords; i++) {
- int ix = ProbabilityHelper.choose(cumulIntRanges, rnd.nextInt());
- strBuilder.append(tokenDict.get(ix));
- if (i != numWords - 1) {
- strBuilder.append(" ");
- }
- }
- return strBuilder.toString();
- }
-
- public List<String> getTokenDictionary() {
- return tokenDict;
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
deleted file mode 100644
index 8e36335..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class DoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
- protected final Random rnd;
-
- public DoubleFieldValueGenerator(Random rnd) {
- this.rnd = rnd;
- }
-
- @Override
- public Double next() {
- return rnd.nextDouble();
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
deleted file mode 100644
index 851a0f4..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class FloatFieldValueGenerator implements IFieldValueGenerator<Float> {
- protected final Random rnd;
-
- public FloatFieldValueGenerator(Random rnd) {
- this.rnd = rnd;
- }
-
- @Override
- public Float next() {
- return rnd.nextFloat();
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
deleted file mode 100644
index dbd4bfc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public interface IFieldValueGenerator<T> {
- public T next();
- public void reset();
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
deleted file mode 100644
index 6c5f649..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
- protected final Random rnd;
-
- public IntegerFieldValueGenerator(Random rnd) {
- this.rnd = rnd;
- }
-
- @Override
- public Integer next() {
- return rnd.nextInt();
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
deleted file mode 100644
index 5af0b0f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class PersonNameFieldValueGenerator implements IFieldValueGenerator<String> {
- private final String FIRST_NAMES_FILE = "dist.all.first.cleaned";
- private final String LAST_NAMES_FILE = "dist.all.last.cleaned";
-
- private final Random rnd;
- private final double middleInitialProb;
- private final String letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
- private List<String> firstNames = new ArrayList<String>();
- private List<String> lastNames = new ArrayList<String>();
-
- public PersonNameFieldValueGenerator(Random rnd, double middleInitialProb)
- throws IOException {
- this.rnd = rnd;
- this.middleInitialProb = middleInitialProb;
- initNames();
- }
-
- private void initNames() throws IOException {
- String line;
-
- // Read first names from data file.
- InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE);
- BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn));
- try {
- while ((line = firstNamesReader.readLine()) != null) {
- firstNames.add(line.trim());
- }
- } finally {
- firstNamesReader.close();
- }
-
- // Read last names from data file.
- InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE);
- BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn));
- try {
- while ((line = lastNamesReader.readLine()) != null) {
- lastNames.add(line.trim());
- }
- } finally {
- lastNamesReader.close();
- }
- }
-
- @Override
- public String next() {
- StringBuilder strBuilder = new StringBuilder();
-
- // First name.
- int fix = MathUtil.stripSignBit(rnd.nextInt()) % firstNames.size();
- strBuilder.append(firstNames.get(fix));
- strBuilder.append(" ");
-
- // Optional middle initial.
- double d = Math.abs(rnd.nextDouble());
- if (d <= middleInitialProb) {
- int mix = MathUtil.stripSignBit(rnd.nextInt()) % letters.length();
- strBuilder.append(letters.charAt(mix));
- strBuilder.append(". ");
- }
-
- // Last name.
- int lix = MathUtil.stripSignBit(rnd.nextInt()) % lastNames.size();
- strBuilder.append(lastNames.get(lix));
-
- return strBuilder.toString();
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
deleted file mode 100644
index c7fbd3f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Arrays;
-
-public class ProbabilityHelper {
- public static double[] getUniformProbDist(int numChoices) {
- double[] probDist = new double[numChoices];
- for (int i = 0; i < numChoices; i++) {
- probDist[i] = 1.0 / (double) numChoices;
- }
- return probDist;
- }
-
- public static double[] getZipfProbDist(int numChoices, int zipfSkew) {
- double[] probDist = new double[numChoices];
- double divisor = 0;
- for (int i = 1; i <= numChoices; i++) {
- divisor += 1.0 / (double) Math.pow((double) i, (double) zipfSkew);
- }
- for (int i = 1; i <= numChoices; i++) {
- probDist[i - 1] = (1.0 / (double) Math.pow((double) i, (double) zipfSkew)) / divisor;
- }
- return probDist;
- }
-
- public static int[] getCumulIntRanges(double[] probDist) {
- int[] opRanges = new int[probDist.length];
- if (opRanges.length > 1) {
- opRanges[0] = (int) Math.floor(Integer.MAX_VALUE * probDist[0]);
- for (int i = 1; i < opRanges.length - 1; i++) {
- opRanges[i] = opRanges[i - 1] + (int) Math.floor(Integer.MAX_VALUE * probDist[i]);
- }
- opRanges[opRanges.length - 1] = Integer.MAX_VALUE;
- } else {
- opRanges[0] = Integer.MAX_VALUE;
- }
- return opRanges;
- }
-
- public static int choose(int[] cumulIntRanges, int randomInt) {
- int rndVal = Math.abs(randomInt);
- int ix = Arrays.binarySearch(cumulIntRanges, rndVal);
- if (ix < 0) {
- ix = -ix - 1;
- }
- return ix;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
deleted file mode 100644
index ddca6f3..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedDoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
- private double val;
- private final double startVal;
-
- public SortedDoubleFieldValueGenerator() {
- startVal = 0.0d;
- reset();
- }
-
- public SortedDoubleFieldValueGenerator(double startVal) {
- this.startVal = startVal;
- reset();
- }
-
- @Override
- public Double next() {
- return val++;
- }
-
- @Override
- public void reset() {
- val = startVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
deleted file mode 100644
index 1e5dd19..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedFloatFieldValueGenerator implements IFieldValueGenerator<Float> {
- private float val = 0.0f;
- private final float startVal;
-
- public SortedFloatFieldValueGenerator() {
- startVal = 0.0f;
- reset();
- }
-
- public SortedFloatFieldValueGenerator(float startVal) {
- this.startVal = startVal;
- reset();
- }
-
- @Override
- public Float next() {
- return val++;
- }
-
- @Override
- public void reset() {
- val = startVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
deleted file mode 100644
index 3ab17ca..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
- private int val = 0;
- private final int startVal;
-
- public SortedIntegerFieldValueGenerator() {
- startVal = 0;
- reset();
- }
-
- public SortedIntegerFieldValueGenerator(int startVal) {
- this.startVal = startVal;
- reset();
- }
-
- @Override
- public Integer next() {
- return val++;
- }
-
- @Override
- public void reset() {
- val = startVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
deleted file mode 100644
index 9418447..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class StringFieldValueGenerator implements IFieldValueGenerator<String> {
- private int maxLen;
- private final Random rnd;
-
- public StringFieldValueGenerator(int maxLen, Random rnd) {
- this.maxLen = maxLen;
- this.rnd = rnd;
- }
-
- public void setMaxLength(int maxLen) {
- this.maxLen = maxLen;
- }
-
- @Override
- public String next() {
- String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < maxLen; i++) {
- strBuilder.append(s.charAt(MathUtil.stripSignBit(rnd.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-
- @Override
- public void reset() {
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
deleted file mode 100644
index c34c7bc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-@SuppressWarnings("rawtypes")
-public class TupleBatch {
- private final int size;
- private final TupleGenerator[] tupleGens;
- public final AtomicBoolean inUse = new AtomicBoolean(false);
-
- public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
- this.size = size;
- tupleGens = new TupleGenerator[size];
- for (int i = 0; i < size; i++) {
- tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize);
- }
- }
-
- public void generate() throws IOException {
- for(TupleGenerator tupleGen : tupleGens) {
- tupleGen.next();
- }
- }
-
- public int size() {
- return size;
- }
-
- public ITupleReference get(int ix) {
- return tupleGens[ix].get();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
deleted file mode 100644
index 4f26065..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-@SuppressWarnings({"rawtypes", "unchecked" })
-public class TupleGenerator {
- protected final ISerializerDeserializer[] fieldSerdes;
- protected final IFieldValueGenerator[] fieldGens;
- protected final ArrayTupleBuilder tb;
- protected final ArrayTupleReference tuple;
- protected final byte[] payload;
- protected final DataOutput tbDos;
-
- public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
- this.fieldSerdes = fieldSerdes;
- this.fieldGens = fieldGens;
- tuple = new ArrayTupleReference();
- if (payloadSize > 0) {
- tb = new ArrayTupleBuilder(fieldSerdes.length + 1);
- payload = new byte[payloadSize];
- } else {
- tb = new ArrayTupleBuilder(fieldSerdes.length);
- payload = null;
- }
- tbDos = tb.getDataOutput();
- }
-
- public ITupleReference next() throws IOException {
- tb.reset();
- for (int i = 0; i < fieldSerdes.length; i++) {
- fieldSerdes[i].serialize(fieldGens[i].next(), tbDos);
- tb.addFieldEndOffset();
- }
- if (payload != null) {
- tbDos.write(payload);
- tb.addFieldEndOffset();
- }
- tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
- return tuple;
- }
-
- public ITupleReference get() {
- return tuple;
- }
-
- public void reset() {
- for (IFieldValueGenerator fieldGen : fieldGens) {
- fieldGen.reset();
- }
- }
-
- public ISerializerDeserializer[] getFieldSerdes() {
- return fieldSerdes;
- }
-
- public IFieldValueGenerator[] getFieldGens() {
- return fieldGens;
- }
-}