You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/08/25 05:00:41 UTC
Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!
From Shiva Jahangiri <sh...@uci.edu>:
Shiva Jahangiri has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967 )
Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................
PLEASE EDIT to provide a meaningful commit message!
[NO ISSUE] Added Wisconsin Data Generator for tests
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Wisconsin Data Generator has been added as
a separate package to generate data for related
test queries.
Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
22 files changed, 2,762 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/67/12967/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
new file mode 100644
index 0000000..a590995
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+MIT License
+
+Copyright (c) 2020 Shiva Jahangiri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
new file mode 100644
index 0000000..b40d9a0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+This project is under **MIT license** (for this software to be used as part of other project as modified or not modified) and can be referenced in scientific papers, books, journals, etc. upon the usage using **DOI: 10.5281/zenodo.4315680 (http://doi.org/10.5281/zenodo.4315937)**
+
+**Cite as:**
+Shiva Jahangiri. (2020, December 11). shivajah/JSON-Wisconsin-Data-Generator: First Release (Version v1.0.2). Zenodo. http://doi.org/10.5281/zenodo.4316003
+# JSON Wisconsin Data Generator: To JSON and Beyond
+
+[JSON Wisconsin Data Generator](https://github.com/shivajah/JSON-Wisconsin-Data-Generator) is a [Wisconsin](http://jimgray.azurewebsites.net/benchmarkhandbook/chapter4.pdf)-inspired data generator tool implemented in Java. Wisconsin Benchmark was designed at the University of Wisconsin by DeWitt et al to test the performance of the major components of a relational database system while the semantics and statistics of its relations would be easy to understand. This Benchmark was initially a single-user micro-benchmark for measuring individual query performance. A multi-user version of this benchmark which was developed a few years later did not attract as much as attraction as past, due to existence of other well-established competitors such as DebitCredit Benchmark, a multi-user OLTP benchmark led by Jim Gray.
+While the Wisconsin Benchmark does not have as large an audience today as in the past, we believe that its simple but powerful design easy to understand and highly controllable relations and attributes are unique and can be beneficial in evaluating database systems from many angles. As such, we have built a Wisconsin-inspired JSON data generator in java with more add-ons and advances features to support more modern data features, including: Variable length records based on size distributions, skewed attributes, and other features to support semi-structured data behavior as well.
+This data generator has been used in several studies and publications for benchmarking [AsterixDB](https://asterixdb.incubator.apache.org).
+
+## Wisconsin Benchmark:
+Figure below shows the schema for the Wisconsin Benchmark which is the default schema for our JSON Wisconsin Data Generator.
+<p align="center">
+ <img src="https://github.com/shivajah/JSON-Wisconsin-Data-Generator/blob/main/Wisconsin%20Benchmark-%20Original%20Schema.png">
+</p>
+
+## JSON Wisconsin Data Generator:
+
+### Inputs:
+The following JSON shows how this schema looks like in JSON to be used as an input to the program. A JSON array is used to show the schema for a relation while the JSON objects included in this array make the relation attributes. User can add more attributes by adding a new JSON object in the JSON array or drop them by simply removing them from the array.
+
+```JSON
+[
+ {
+ "name":"unique1",
+ "type":"integer",
+ "order":"random"
+},
+ {
+ "name":"unique2",
+ "type":"integer",
+ "order":"sequential"
+ },
+ {
+ "name":"two",
+ "type":"integer",
+ "order":"random",
+ "range":2
+ },
+ {
+ "name":"four",
+ "type":"integer",
+ "order":"random",
+ "range":4
+ },
+ {
+ "name":"ten",
+ "type":"integer",
+ "order":"random",
+ "range":10
+ },
+ {
+ "name":"twenty",
+ "type":"integer",
+ "order":"random",
+ "range":20
+ },
+ {
+ "name":"onePercent",
+ "type":"integer",
+ "order":"random",
+ "range":100
+ },
+ {
+ "name":"tenPercent",
+ "type":"integer",
+ "order":"random",
+ "range":10
+ },
+ {
+ "name":"twentyPercent",
+ "type":"integer",
+ "order":"random",
+ "range":5
+ },
+ {
+ "name":"fiftyPercent",
+ "type":"integer",
+ "order":"random",
+ "range":2
+ },
+ {
+ "name":"unique3",
+ "type":"integer",
+ "order":"random"
+ },
+ {
+ "name":"evenOnePercent",
+ "type":"integer",
+ "order":"random",
+ "even":true,
+ "range":100
+ },
+ {
+ "name":"oddOnePercent",
+ "type":"integer",
+ "order":"random",
+ "odd":true,
+ "range":100
+ },
+ {
+ "name":"stringu1",
+ "type":"string",
+ "length":"100",
+ "order":"sequential"
+ },
+ {
+ "name":"stringu2",
+ "type":"string",
+ "length":"100",
+ "order":"random"
+ },
+ {
+ "name":"string4",
+ "type":"string",
+ "length":"400",
+ "order":"random"
+ }
+]
+
+```
+
+### Outputs:
+
+The output of the program is a set of JSON objects where each object represents a record in the relation. Figure 3 shows an example of 5 output records.
+
+```JSON
+{"unique1":2, "unique2":1, "two":0, "four":2, "ten":2, "twenty":2, "onePercent":2, "tenPercent":2, "twentyPercent":2, "fiftyPercent":0, "unique3":2, "evenOnePercent":4, "oddOnePercent":5, "stringu1":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":4, "unique2":2, "two":0, "four":0, "ten":4, "twenty":4, "onePercent":4, "tenPercent":4, "twentyPercent":4, "fiftyPercent":0, "unique3":4, "evenOnePercent":8, "oddOnePercent":9, "stringu1":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":1, "unique2":3, "two":1, "four":1, "ten":1, "twenty":1, "onePercent":1, "tenPercent":1, "twentyPercent":1, "fiftyPercent":1, "unique3":1, "evenOnePercent":2, "oddOnePercent":3, "stringu1":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":5, "unique2":4, "two":1, "four":1, "ten":5, "twenty":5, "onePercent":5, "tenPercent":5, "twentyPercent":0, "fiftyPercent":1, "unique3":5, "evenOnePercent":10, "oddOnePercent":11, "stringu1":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":3, "unique2":5, "two":1, "four":3, "ten":3, "twenty":3, "onePercent":3, "tenPercent":3, "twentyPercent":3, "fiftyPercent":1, "unique3":3, "evenOnePercent":6, "oddOnePercent":7, "stringu1":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+
+```
+
+### Usage:
+1. Checkout the [JSON Wisconsin Data Generator](https://github.com/shivajah/JSON-Wisconsin-Data-Generator) in a directory using git. In this document we assume that the path of the mentioned directory is $HOME/JsonWiscDataGen and we will refer to it as JSONWISCDATAGEN_HOME. You can choose any other directory as the home directory for JSON Wisconsin Data Generator.
+2. Set the $HOME/JsonWiscDataGen as an environment variable on your machine using the following command:
+```
+export JSONWISCDATAGEN_HOME=$HOME/JsonWiscDataGen
+```
+3. Go to the home directory of JSON Wisconsin Data Generator
+```
+cd $JSONWISCDATAGEN_HOME
+```
+4. Build the data generator using maven as follow:
+```
+mvn clean package
+```
+5. If the build is successful, a target folder will be created which contains the required jar file and dependencies. The workload file providing the schema and other relation information for Json Wisconsin Data Generator should be created as a json file in '$JSONWISCDATAGEN_HOME/Workloads' directory. There are some example workloads including the default workload in this folder. You can start creating your own workload file using these examples and information provided here.
+6. You can start generating the dataset by running the following command:
+```
+java -jar $JSONWISCDATAGEN_HOME/target/wisconsin-datagen.jar writer=file workload=default.json cardinality=99999999
+```
+7. Default configurations are provided in $JSONWISCDATAGEN_HOMEB/src/main/resources/wisconsin_datagen.properties which can be overwritten similar to the above example (step 6).
+- filesize: In MB, which would be another terminator for the program. Program will stop generating records if the asked cardinality of filesize is reached. Whichever that happen first, would be the terminator.
+- writer: asterixdb or file. "file" writes the output to a file in target folder with the name provided in "fileoutput". "asterixdb" writer loads the records directly to AsterixDB using AsterixDBLoadPort. For using the asterixdb writer, data generator should run on one of the NC nodes for it in order to work properly.
+
+### Workloads and Advanced Features:
+Sample workloads are provided in Workloads folder. The Default.json shows the schema for the original wisconsin benchmark relation. The Advanced.json has more fields with other features such as: varibale length strings, different length distribution, different value distributions( value skewness), real-word strings, nullable, and optional(missing) fields.
+In this section we explain these advanced features with some examples.
+
+## Strings:
+In the original Wisconsin Data Generator, strings are generated in a random or cyclic format. In the case of random, the string representation of the unique1 attribute is used as the prefix (which is unique as well) and enough ‘x’ characters will be added to the end of the string to reach to the desired length for that attribute. In the case of cyclic, string values are generated from the domain of four prefix in a cyclic format.In JSON Wisconsin Data Generator we only support the random format for generating the strings.
+### 1. Real-Word and HEX Strings:
+In addition to supporting the mentioned algorithm for generating strings, we support strings that are generated by concatenating words from a list made of 10,000 real words. This approach helps with reducing the impact of data compression due to less repetitive characters. In case of specifiying a distribution for the length of the strings, HEX strings will be generated.
+In order to generate strings made of real words, user can specify ```"word":true``` in the field definition. If word is set to true and string is set to be variable length```("variableLength":true)```, an average number of words to be included in the string is provided as an input, and algorithm concatenates as many as asked from the word list.
+### 2. Variable Length Strings:
+A string attribute can have values with different lengths by setting ```("variableLength":true)```. There are multiple options for the distribution based on which these lengths are generated.
+
+#### 2.1 Word:
+If for a variable length string, "```word=true``` then a Normal Distributin is used for calculating the actual number of words in the string. The ```length``` property of the string attribute is used as the average number of words, and the ```standardDeviation``` is used as the standard deviation used in normal distribution.
+#### 2.2 Hex String & Different Distributions on Length of the String:
+- If a string attribute is defined as variable length and ```"normalDistribution": true``` then a Hex String will be generated with the string length (```length``` property) as the mean and the standard deviation(```standardDeviation```) provided as a property for the string attribute.
+
+- If a string attribute is defined as variable length and ```"bernouliDistribution": true``` then user should also provide values for four other properties: ```"probLargeRecord", "minSizeSmall",maxSizeSmall","minSizeLarge","maxSizeLarge"```. The first property specifies what percentage of the strings should be large. The rest of the properties shows the maximum and minimum length for the small and large strings. First a bernouli distribution will be used for each record to determine if it is a large string or small. Then, a uniform distribution will choose the actual string length from the specified range for large and small strings. The generated string is a HEX string.
+
+- If ```"zipfDistribution": true```, the user should specify three other properties: ```"zipfMinSize","zipfMaxSize","zipfSkew"```. The first two properties specify the range of the length of the string, and the last property is the skewness of Zipf distribution. The rank is calculated based on the number of elements between the ```"zipfMinSize","zipfMaxSize"``` values. The generated string is a HEX string.
+
+- If none of the above distributions were selected, a Gamma distribution will be used to deterime the length of the string. The generated string is a HEX string.
+
+### 3. Fixed Length Strings:
+We offer different ways of generating fixed length strings. A fixed length string can either be made of words (```"word":true```) or based on the original Wisconsin Benchmark string generation technique. If ```"word":true```, then two distributions of Zipf and Uniform on selecting which word should be used (distribution on the value not the length of the string) can be performed. Extra 'X' characters can be appended to the string if the requested (but fixed) length of the string is larger than the length of the selected word.
+
+## Skewness in Integer Attribute Values:
+The integer attributes can be generated in three ways. The first way is to generate is as the original Wisconsin Benchmark would choose the integer value based on being sequential or random. The other ways are based on Gamma and Normal Distributions that we have defined in order to be able to introduce skew on the values of an integer attribute.
+
+## Nullable and Optional (missing) fields:
+While the original Wisconsin Benchmark does not provide a way to introduce null values, in our JSON Wisconsin Data Generator we have added the capability for having nullable and optional fields. Optional fields are mostly used in semi-structured data where the attribute itself maynot appear in some of the records. For specifying a nullable attribute, the user should set ```"nullable":true``` and also set the ```nulls``` to a value between 0 and 1. The latter attribute shows the probability of the attribute to have a null value for which a binomial distribution is used.
+For specifying the optional values, the user should set ```"optional":true``` and also set the ```missings``` to a value between 0 and 1. The latter attribute shows the probability of the attribute to be missing for which a binomial distribution is used.
+
+## Binary Data type:
+In addition to integer and string data types, we have also introduced Binary data type which generates HEX values.
+
+## Acknowledgement:
+Special thanks to Professor Michael Carey and Professor Johann-Christoph Freytag for their advice and support and to Taewoo Kim for his contribution to the development of variable length strings and binary data types.
+
+
+
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
new file mode 100644
index 0000000..9a6aff9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
@@ -0,0 +1,149 @@
+<!--
+#/*
+ #* Licensed to the Apache Software Foundation (ASF) under one
+ #* or more contributor license agreements. See the NOTICE file
+ #* distributed with this work for additional information
+ #* regarding copyright ownership. The ASF licenses this file
+ #* to you under the Apache License, Version 2.0 (the
+ #* "License"); you may not use this file except in compliance
+ #* with the License. You may obtain a copy of the License at
+ #*
+ #* http://www.apache.org/licenses/LICENSE-2.0
+ #*
+ #* Unless required by applicable law or agreed to in writing,
+ #* software distributed under the License is distributed on an
+ #* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ #* KIND, either express or implied. See the License for the
+ #* specific language governing permissions and limitations
+ #* under the License.
+ #*/
+ #/*Copyright (c) 2020 Shiva Jahangiri
+
+ #Permission is hereby granted, free of charge, to any person obtaining a copy
+ #of this software and associated documentation files (the "Software"), to deal
+ #in the Software without restriction, including without limitation the rights
+ #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ #copies of the Software, and to permit persons to whom the Software is
+ #furnished to do so, subject to the following conditions:
+
+ #The above copyright notice and this permission notice shall be included in all
+ #copies or substantial portions of the Software.
+
+ #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ #SOFTWARE.
+ #*/-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics</groupId>
+ <artifactId>datagenClient</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>Datagen Client</name>
+
+ <build>
+ <plugins>
+ <!-- JDK compiler level -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+
+ <!-- Assemble whole project into JAR -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>wisconsin-datagen</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20070829</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.6.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.couchbase.client</groupId>
+ <artifactId>java-client</artifactId>
+ <version>2.7.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.couchbase.client</groupId>
+ <artifactId>core-io</artifactId>
+ <version>1.7.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.teradata.tpcds</groupId>
+ <artifactId>tpcds</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.11.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
new file mode 100644
index 0000000..743b20c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+public class DataTypes {
+
+ public enum DataType {
+ STRING,
+ INTEGER,
+ BINARY
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
new file mode 100644
index 0000000..0380aaf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+public class Order {
+ public enum order {
+ RANDOM,
+ SEQUENTIAL
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
new file mode 100644
index 0000000..cc4f364
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.io.FileUtils;
+
+public class StringWordList {
+
+ private static List<String> wordList;
+ private static StringWordList instance;
+
+ StringWordList() throws IOException {
+ wordList = FileUtils.readLines(new File(Server.workloadsFolder + "wordlist"), "utf-8");
+ }
+
+ public static StringWordList getInstance() throws IOException {
+ if (instance == null) {
+ synchronized (StringWordList.class) {
+ if (instance == null) {
+ instance = new StringWordList();
+ }
+ }
+ }
+ return instance;
+ }
+
+ public List<String> getWordList() {
+ return wordList;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
new file mode 100644
index 0000000..19f34ad
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+
+public abstract class AWisconsinGenerator {
+ Schema schema;
+ int fieldId;
+ Random rand;
+ DataTypes.DataType generatorType;
+ double nullStartRang;
+ double nullEndRange;
+ double missingStartRange;
+ double missingEndRange;
+
+ public enum ValueType {
+ NULL,
+ MISSING,
+ VALUE
+ }
+
+ public AWisconsinGenerator(Schema schema, int fieldId) {
+ this.schema = schema;
+ this.fieldId = fieldId;
+ this.rand = new Random();
+ setUpNullAndMissingRanges();
+ }
+
+ private void setUpNullAndMissingRanges() {
+ this.nullStartRang = -1;
+ this.nullEndRange = -1;
+ this.missingStartRange = -1;
+ this.missingEndRange = -1;
+ if (schema.getFields().get(fieldId).isNullable()) {//NULL
+ this.nullStartRang = 0;
+ this.nullEndRange = schema.getFields().get(fieldId).getNulls();
+ if (schema.getFields().get(fieldId).isOptional()) { //NULL & MISSING
+ this.missingStartRange = nullEndRange;
+ this.missingEndRange = nullEndRange + schema.getFields().get(fieldId).getMissings();
+ }
+ } else if (schema.getFields().get(fieldId).isOptional()) {// MISSING
+ this.missingStartRange = 0;
+ this.missingEndRange = schema.getFields().get(fieldId).getMissings();
+ }
+
+ }
+
+ public abstract Object next(long i);
+
+ public long rand(long seed, long cardinality) {
+ do {
+ seed = (schema.getGenerator() * seed) % schema.getPrime();
+ } while (seed > cardinality);
+ return (seed);
+ }
+
+ public ValueType getValueType() {
+ float nextEvent = rand.nextFloat();
+ if (nextEvent >= nullStartRang && nextEvent < nullEndRange) {
+ return ValueType.NULL;
+ } else if (nextEvent >= missingStartRange && nextEvent < missingEndRange) {
+ return ValueType.MISSING;
+ }
+ return ValueType.VALUE;
+ }
+
+ public DataTypes.DataType getDataType() {
+ return generatorType;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
new file mode 100644
index 0000000..ced67b3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+import java.util.Stack;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinBinaryGenerator extends AWisconsinGenerator {
+ private static final char[] VALUES =
+ { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+ private static final int HEXRANGE = 15;
+ Random r;
+
+ public WisconsinBinaryGenerator(Schema schema, int fieldId) {
+ super(schema, fieldId);
+ r = new Random(fieldId);
+ generatorType = DataType.BINARY;
+ }
+
+ public String next(long seed) {
+ Field f = schema.getFields().get(fieldId);
+ if (getValueType() == ValueType.NULL) {
+ return "\"null\"";
+ } else if (getValueType() == ValueType.NULL) {
+ return "#MISSING";
+ }
+ if (f.getOrder() != null && f.getOrder() == Order.order.RANDOM) {
+ seed = rand(seed, Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME)));
+ }
+ if (f.getRange() > 0) {
+ seed = seed % f.getRange();
+ }
+ Stack<Character> temp = new Stack<>();
+
+ r.setSeed(seed);
+ char nextHex;
+ int length = schema.getFields().get(fieldId).getStringLength();
+ for (int i = 0; i < length; i++) {
+ nextHex = VALUES[r.nextInt(HEXRANGE)];
+ temp.push(nextHex);
+ }
+
+ char[] returnCharValues = new char[length];
+ int index = 0;
+ while (!temp.isEmpty()) {
+ returnCharValues[index] = temp.pop();
+ index++;
+ }
+
+ return String.valueOf(returnCharValues);
+
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
new file mode 100644
index 0000000..0110f55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.math3.distribution.GammaDistribution;
+
+public class WisconsinNumberGenerator extends AWisconsinGenerator {
+ GammaDistribution gd;
+ Random r;
+ Field field;
+
+ public WisconsinNumberGenerator(Schema schema, int fieldId) {
+ super(schema, fieldId);
+ generatorType = DataType.INTEGER;
+ field = schema.getFields().get(fieldId);
+ gd = new GammaDistribution(field.getShape(), field.getScale());
+ r = new Random(fieldId);
+ }
+
+ @Override
+ public Object next(long seed) {
+
+ long result;
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME));
+ ValueType type = getValueType();
+ if (type == ValueType.NULL) {
+ return "\"null\"";
+ } else if (type == ValueType.MISSING) {
+ return "#MISSING";
+ }
+ // Use cardinality and Jim Gray's algo
+ if (field.getOrder() != null && field.getOrder() == Order.order.RANDOM) {
+ result = rand(seed, cardinality);
+ } else
+ result = seed;
+ if (field.getRange() > 0) {
+ result = result % field.getRange();
+ }
+ if (field.isEven()) {
+ result *= 2;
+ }
+ if (field.isOdd()) {
+ result = result * 2 + 1;
+ }
+
+ if (field.isGammaDistribution()) {
+
+ long range = field.getRange() > 0 ? field.getRange() : cardinality;
+ double sample = gd.sample();
+ result = (int) ((sample * result) % range);
+
+ } else if (field.isNormalDistribution()) {
+ double mean = field.getMean() > 0 ? field.getMean() : cardinality / 2;
+ double stdev = field.getStandardDeviation();
+ result = (int) (r.nextGaussian() * stdev + mean);
+ return result;
+
+ }
+ return result;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
new file mode 100644
index 0000000..8274344
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.LinkedHashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.Stack;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.StringWordList;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.math3.distribution.GammaDistribution;
+
+public class WisconsinStringGenerator extends AWisconsinGenerator {
+ private static final char[] VALUES =
+ { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+ private static final DecimalFormat decFormat = new DecimalFormat("#.######");
+
+ Random r;
+ StringWordList wordList;
+ StringBuilder builder;
+ Set<Integer> wordsSet;
+ int wordListSize;
+ long hexStringLength;
+ int maxHexStringLength;
+ int minHexStringLength;
+ int hexStringCount;
+ char[] returnHexCharValues;
+ GammaDistribution gd;
+ Field f;
+ ZipfGenerator zipfGenerator;
+
+ // To write the rawdata of gamma distribution.
+ BufferedWriter writer = null;
+ String fileName = "gamma_distribution_rawdata.txt";
+
+ public WisconsinStringGenerator(Schema schema, int fieldId) throws IOException {
+ super(schema, fieldId);
+ generatorType = DataType.STRING;
+ r = new Random(fieldId);
+ wordList = StringWordList.getInstance();
+ builder = new StringBuilder();
+ wordsSet = new LinkedHashSet<>();
+ wordListSize = wordList.getWordList().size();
+ hexStringLength = 0;
+ // Just default size
+ returnHexCharValues = new char[10];
+ maxHexStringLength = 0;
+ minHexStringLength = Integer.MAX_VALUE;
+ hexStringCount = 0;
+ f = schema.getFields().get(fieldId);
+ if (f.isVariableLength() && !f.isWord() && !f.isNormalDistribution()) {
+ gd = new GammaDistribution(f.getShape(), f.getScale());
+ writer = new BufferedWriter(new FileWriter(fileName));
+ }
+ if (f.isZipfDistribution()) {
+ if (f.isWord()) {
+ int bound = Math.min(wordListSize, f.getRange());
+ zipfGenerator = new ZipfGenerator(bound, f.getZipfSkew());
+ } else {
+ int rangeSize = f.getZipfMaxSize() - f.getZipfMinSize();
+ assert (rangeSize > 0);
+ zipfGenerator = new ZipfGenerator(rangeSize + 1, f.getZipfSkew());
+ }
+ }
+ }
+
+ public String next(long seed) {
+ ValueType type = getValueType();
+ if (type == ValueType.NULL) {
+ return "\"null\"";
+ } else if (type == ValueType.MISSING) {
+ return "#MISSING";
+ }
+ if (f.getOrder() != null && f.getOrder() == Order.order.RANDOM) {
+ seed = rand(seed, Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME)));
+ }
+ if (f.getRange() > 0) {
+ seed = seed % f.getRange();
+ }
+
+ if (f.isVariableLength()) {
+ if (f.isWord()) {
+ // Variable length word generation
+ // For this case, the length is regarded as the average word count.
+ return generateRandomWords(f.getStringLength(), f.getStandardDeviation());
+ } else if (f.isNormalDistribution()) {
+ // Variable length hex char string generation
+ // For this case, the length is regarded as the average length.
+ // return generateRandomHexChars();
+ return generateRandomHexCharsGaussian(f.getStringLength(), f.getStandardDeviation());
+ } else if (f.isBernoulliDistribution()) {
+ // Bernoulli Distribution: with some probability the record is large
+ if (r.nextFloat() < f.getProbLargeRecord()) {
+ return generateRandomHexCharsUniformDist(f.getMinSizeLarge(), f.getMaxSizeLarge());
+ }
+ return generateRandomHexCharsUniformDist(f.getMinSizeSmall(), f.getMaxSizeSmall());
+ } else if (f.isZipfDistribution()) {
+ return generateRandomHexCharsZipfianDist(f.getZipfMinSize());
+ } else {
+ // Gamma distribution case...
+ // Variable length hex char string generation
+ // For this case, the length is regarded as the average length.
+ // return generateRandomHexChars();
+ String result = generateRandomHexCharsGamma(f.getStringLength());
+ return result;
+ }
+ } else {//Fixed Length Records
+ if (f.isWord()) {
+ if (f.isZipfDistribution()) {
+ //Zipf distribution is for join skew not field length.
+ int rank = zipfGenerator.next();
+ String word = wordList.getWordList().get(rank);
+ return addPaddingX(word, f.isVariableLength());
+ } else {//uniform
+ int rank = r.nextInt(wordListSize);
+ if (f.getRange() > 0) {
+ rank = rank % f.getRange();
+ }
+ return addPaddingX(wordList.getWordList().get(rank), f.isVariableLength());
+
+ }
+ } else {
+ Stack<Character> temp = new Stack<>();
+ char[] field = { 'A', 'A', 'A', 'A', 'A', 'A', 'A' };
+ if (seed == 0) {
+ long rem = seed % 26;
+ temp.push((char) ('A' + rem));
+ }
+ while (seed > 0) {
+ long rem = seed % 26;
+ temp.push((char) ('A' + rem));
+ seed = seed / 26;
+ }
+ int i = 0;
+ while (!temp.empty() && i < field.length) {
+ field[i] = temp.pop();
+ i++;
+ }
+ return addPaddingX(String.valueOf(field), f.isVariableLength());
+ }
+ }
+ }
+
+ private String addPaddingX(String value, boolean variableLength) {
+ int stLen = f.getStringLength();
+ int diff = Math.max(0, stLen - value.length());
+ if (variableLength) {
+ diff = r.nextInt(diff);
+ }
+ for (int i = 0; i < diff; i++) {
+ value = value + 'X';
+ }
+ return value;
+ }
+
+ private String generateRandomWords(int wordCount, double standardDeviation) {
+ builder.delete(0, builder.length());
+ wordsSet.clear();
+
+ // Based on the Gaussian distribution, we get the actual word count.
+ int actualWordCount = (int) (r.nextGaussian() * standardDeviation + wordCount);
+ // At least it should be one.
+ actualWordCount = Math.max(actualWordCount, 1);
+ // At maximum, it cannot be greater than (stringLength + (stringLength - 1)) to make the distribution symmetric.
+ //
+ // x = (stringLength - 1) x = (stringLength - 1)
+ // 1 ------------------------ stringLength --------------------------- (stringLength + (stringLength - 1))
+ // since the left range is between 1 and the given stringLength.
+ actualWordCount = Math.min(actualWordCount, 2 * actualWordCount - 1);
+
+ int idx;
+ for (int i = 0; i < actualWordCount; i++) {
+ // Find a random number that is not in the set yet.
+ while (true) {
+ idx = r.nextInt(wordListSize);
+ if (!wordsSet.contains(idx)) {
+ wordsSet.add(idx);
+ builder.append(wordList.getWordList().get(idx));
+ if (i != (actualWordCount - 1)) {
+ builder.append(" ");
+ }
+ break;
+ }
+ }
+ }
+ return builder.toString();
+ }
+
+ private String generateRandomHexString(int length) {
+ if (length >= maxHexStringLength) {
+ maxHexStringLength = length;
+ returnHexCharValues = new char[maxHexStringLength];
+ }
+ if (length < minHexStringLength) {
+ minHexStringLength = length;
+ }
+ char nextHex;
+ for (int i = 0; i < length; i++) {
+ nextHex = VALUES[r.nextInt(VALUES.length)];
+ returnHexCharValues[i] = nextHex;
+ }
+ return String.valueOf(returnHexCharValues, 0, length);
+ }
+
+ private String generateRandomHexChars(int length) {
+ hexStringLength += length;
+ hexStringCount++;
+ return generateRandomHexString(length);
+ }
+
+ private String generateRandomHexCharsGaussian(int stringAvgLength, double standardDeviation) {
+ // Based on the Gaussian distribution, we get the actual word count.
+ int stringLength = (int) (r.nextGaussian() * standardDeviation + stringAvgLength);
+ // At least it should be one.
+ stringLength = Math.max(stringLength, 1);
+ // At maximum, it cannot be greater than (stringLength + (stringLength - 1)) to make the distribution symmetric.
+ //
+ // x = (stringLength - 1) x = (stringLength - 1)
+ // 1 ------------------------ stringLength --------------------------- (stringLength + (stringLength - 1))
+ // since the left range is between 1 and the given stringLength.
+ stringLength = Math.min(stringLength, 2 * stringLength - 1);
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsGamma(int multiplier) {
+ // Based on the gamma distribution...
+ // At least the actual string length should be one.
+ int stringLength = (int) (gd.sample() * multiplier) + 1;
+ //System.out.print(stringLength+",");
+ try {
+ writer.write(Integer.toString(stringLength) + "\n");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsUniformDist(int minSize, int maxSize) {
+ int rangeSize = maxSize - minSize;
+ assert (rangeSize > 0);
+ int stringLength = r.nextInt(rangeSize + 1) + minSize;
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsZipfianDist(int minSize) {
+ int rank = zipfGenerator.next(); // a number in the range of max-min
+ int stringLength = rank + minSize;
+ return generateRandomHexChars(stringLength);
+ }
+
+ public long getHexStringLength() {
+ return hexStringLength;
+ }
+
+ public String getAvgHexStringLength() {
+ if (hexStringLength > 0 && hexStringCount > 0) {
+ return ("Hex-string-count: " + hexStringCount + " Total length(KB): " + (double) hexStringLength / 1024
+ + " Max: " + maxHexStringLength + " Min: " + minHexStringLength + " Avg: "
+ + decFormat.format((double) hexStringLength / hexStringCount) + " (KB): "
+ + decFormat.format((double) hexStringLength / hexStringCount / 1024));
+ }
+ return "";
+ }
+
+ public void closeGammaDistributionFile() {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
new file mode 100644
index 0000000..8caa34e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+public class ZipfGenerator {
+ private Random rnd = new Random(System.currentTimeMillis());
+ private int size;
+ private double skew;
+ private double bottom = 0;
+
+ public ZipfGenerator(int size, double skew) {
+ this.size = size;
+ this.skew = skew;
+
+ for (int i = 1; i < size; i++) {
+ this.bottom += (1 / Math.pow(i, this.skew));
+ }
+ }
+
+ // the next() method returns an random rank id.
+ // The frequency of returned rank ids are follows Zipf distribution.
+ public int next() {
+ int rank;
+ double frequency = 0;
+ double dice;
+
+ rank = rnd.nextInt(size);
+ frequency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ dice = rnd.nextDouble();
+
+ while (!(dice < frequency)) {
+ rank = rnd.nextInt(size);
+ frequency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ dice = rnd.nextDouble();
+ }
+
+ return rank;
+ }
+
+ // This method returns a probability that the given rank occurs.
+ public double getProbability(int rank) {
+ return (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ }
+
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
new file mode 100644
index 0000000..52378d9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.BufferedOutputStream;
+import java.text.DecimalFormat;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinStringGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Utils.Utils;
+import org.apache.commons.math3.util.Pair;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public abstract class AWisconsinOutputGenerator {
+
+ private static final Logger LOGGER = LogManager.getRootLogger();
+ protected Schema schema;
+ private List<AWisconsinGenerator> generators;
+ final String directory = "./";
+ protected Map<Integer, BufferedOutputStream> threadToFileOutputStream;
+ private static long maxRecordLength;
+ protected ExecutorService executorService;
+ //A map from executor sequence id to a map for start and end row index
+ private Map<Integer, Pair<Long, Long>> executorsToStartAndEnd;
+ private int partitions;
+ private int partition;
+ private static final DecimalFormat decFormat = new DecimalFormat("#.######");
+ //TODO: totalFileSize should be calculated from the record size to know the exact start and end row index(if the file size is deciding where we finish instead of number of records
+ public double totalFileSize;
+ public Map<Long, Integer> sizeToCount;
+
+ public AWisconsinOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ this.partitions = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITIONS_NAME));
+ this.partition = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME));
+ this.schema = schema;
+ this.generators = generators;
+ this.threadToFileOutputStream = new HashMap<>();
+ this.maxRecordLength = 0;
+ this.totalFileSize = 0;
+ this.executorsToStartAndEnd = new HashMap<>();
+ this.sizeToCount = new HashMap<>();
+ }
+
+ private void initExecutors() {
+ // if partition == -1, then all partitions should get created (in parallel)
+ if (partition < 0) {
+ executorService = Executors.newFixedThreadPool(partitions);
+ } else { // Otherwise, only the mentioned partition will be generated using one thread
+ executorService = Executors.newFixedThreadPool(1);
+ }
+ }
+
+ private void initReaderToStartAndEnd() {
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME));
+ long length = cardinality / partitions;
+ long start;
+ for (int i = 0; i < partitions; i++) {
+ start = Math.max(1, length * i);
+ Pair<Long, Long> p =
+ i == partitions - 1 ? new Pair<>(start, cardinality) : new Pair<>(start, start + length);
+ executorsToStartAndEnd.put(i, p);
+ }
+ }
+
+ public void execute() {
+ if (partition < 0) { // All partitions
+ IntStream.range(0, partitions).forEach(readerId -> {
+ executorService.submit(() -> {
+ executorTask(readerId);
+ });
+ });
+ } else { //only one partition
+ executorService.submit(() -> executorTask(partition));
+ }
+ shutDownExecutors();
+
+ }
+
+ private void executorTask(int readerId) {
+ int batchIndex = 0; // batch1, batch2,...
+ int batchSize = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.BATCHSIZE_NAME)); // Maximum number of records in a batch
+ long fileSize = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.FILESIZE_NAME));
+ int recordCount = 0;
+ long totalRecordLength = 0;
+ long minRecordLength = Long.MAX_VALUE;
+ long startTime = System.currentTimeMillis();
+ List<String> batchOfRecords = new LinkedList<>();
+
+ LOGGER.info("start index: " + executorsToStartAndEnd.get(readerId).getKey() + " end index: "
+ + executorsToStartAndEnd.get(readerId).getValue());
+
+ for (long id = executorsToStartAndEnd.get(readerId).getKey(); id <= executorsToStartAndEnd.get(readerId)
+ .getValue(); id++) {
+ if (fileSize > 0 && totalFileSize >= fileSize * 1024 * 1024) {
+ break;
+ }
+ if (id % batchSize == 0) {
+ LOGGER.info("Thread " + readerId + " Created " + id + " records in "
+ + Utils.getPrintableTimeDifference(startTime, System.currentTimeMillis()));
+ }
+ long currentRecordsize = 0;
+ String record = "{";
+
+ boolean allFieldsNull = true;
+ for (int i = 0; i < schema.getFields().size(); i++) {
+
+ Object val = generators.get(i).next(id);
+ if (val instanceof String && val == "#MISSING") {
+ continue;
+ } else {
+ try {
+ boolean comma = record.equalsIgnoreCase("{") ? false : true;
+ if (comma) {
+ record = record + ", ";
+ }
+ allFieldsNull = false;
+ record = record + "\"" + schema.getFields().get(i).getName() + "\":";
+ if (generators.get(i).getDataType() == DataType.BINARY) {
+ record = record + "hex(\"" + val + "\")";
+ currentRecordsize += ((String) val).length();
+ } else if (generators.get(i).getDataType() == DataType.STRING) {
+ record = record + "\"" + val + "\"";
+ currentRecordsize += ((String) val).length();
+ } else if (generators.get(i).getDataType() == DataType.INTEGER) {
+ record = record + val;
+ currentRecordsize += 8;
+ }
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+ }
+ }
+ record = record + "}\n";
+ if (allFieldsNull)
+ continue;
+ if (batchOfRecords.size() >= batchSize) {
+ // Empty the batch
+ write(readerId, batchOfRecords, batchIndex);
+ batchIndex++;
+ batchOfRecords.clear();
+ }
+ batchOfRecords.add(record);
+
+ if (!sizeToCount.containsKey(currentRecordsize)) {
+ sizeToCount.put(currentRecordsize, 0);
+ }
+ int count = sizeToCount.get(currentRecordsize);
+ sizeToCount.put(currentRecordsize, count + 1);
+
+ totalFileSize += currentRecordsize;
+ if (currentRecordsize > maxRecordLength) {
+ maxRecordLength = currentRecordsize;
+ } else if (currentRecordsize < minRecordLength) {
+ minRecordLength = currentRecordsize;
+ }
+ totalRecordLength += currentRecordsize;
+ recordCount++;
+ }
+ if (batchOfRecords.size() > 0) {
+ write(readerId, batchOfRecords, batchIndex);
+ }
+ closeWriter(readerId);
+
+ // Stats info logging
+ LOGGER.info("Thread " + readerId + " Processed " + recordCount + " in "
+ + Utils.getPrintableTimeDifference(startTime, System.currentTimeMillis()));
+ LOGGER.info("Record Generation Done. recordCount: " + recordCount + " minRecordLength: " + minRecordLength
+ + " maxRecordLength: " + maxRecordLength + " totalRecordLength: " + totalRecordLength
+ + " totalFileSize: " + (totalFileSize / (double) 1024 / (double) 1024) + " (MB)"
+ + " avg record length: " + decFormat.format((double) totalRecordLength / recordCount));
+
+ // Calculate and print string length on average
+ LOGGER.info(printStringLengthInfo());
+ LOGGER.info(Arrays.asList(sizeToCount));
+
+ }
+
+ private void shutDownExecutors() {
+ try {
+ executorService.shutdown();
+ executorService.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); // TODO: Is this necessary?
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } finally {
+ if (!executorService.isTerminated()) {
+ System.out.println("canceling all pending tasks");
+ }
+ executorService.shutdownNow();
+ System.out.println("Shutdown complete!");
+ }
+ }
+
+ //Stats Printing
+ private String printStringLengthInfo() {
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < schema.getFields().size(); i++) {
+ if (generators.get(i).getDataType() == DataType.STRING) {
+ long length = ((WisconsinStringGenerator) generators.get(i)).getHexStringLength();
+ // Variable Length?
+ if (length > 0) {
+ strBuilder.append(((WisconsinStringGenerator) generators.get(i)).getAvgHexStringLength() + "\n");
+ ((WisconsinStringGenerator) generators.get(i)).closeGammaDistributionFile();
+ }
+ }
+ }
+ return strBuilder.toString();
+ }
+
+ // Abstract Methods
+ public abstract void write(int threadId, List<String> batchOfRecords, int batchIndex);
+
+ public abstract void closeWriter(int threadID);
+
+ public void initiate() {
+ initExecutors();
+ initReaderToStartAndEnd();
+ };
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
new file mode 100644
index 0000000..29a6253
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinAsterixDBLoadOutputGenerator extends AWisconsinOutputGenerator {
+
+ Socket sock;
+ OutputStream output = null;
+ PrintWriter writer;
+
+ public WisconsinAsterixDBLoadOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ super(schema, generators);
+ initiate();
+ }
+
+ @Override
+ public void write(int threadId, List<String> batchOfRecords, int batchIndex) {
+ for (String record : batchOfRecords) {
+ writer.write(record);
+ }
+ }
+
+ @Override
+ public void closeWriter(int threadID) {
+ try {
+ writer.flush();
+ writer.close();
+ sock.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ @Override
+ public void initiate() {
+ super.initiate();
+ try {
+ sock = new Socket(Server.JSONDataGenConfiguration.get(Server.HOST_NAME_FIELD_NAME),
+ Integer.parseInt(Server.JSONDataGenConfiguration.get(Server.ASTERIXDB_LOAD_PORT)));
+ output = sock.getOutputStream();
+ writer = new PrintWriter(output, true);
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
new file mode 100644
index 0000000..8f70ead
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.IntStream;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinFileOutputGenerator extends AWisconsinOutputGenerator {
+
+ public WisconsinFileOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ super(schema, generators);
+ initiate();
+ }
+
+ @Override
+ public void write(int threadId, List<String> batchOfRecords, int batchIndex) {
+ for (String record : batchOfRecords) {
+ try {
+ threadToFileOutputStream.get(threadId).write(record.getBytes());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public void closeWriter(int threadId) {
+ try {
+ threadToFileOutputStream.get(threadId).close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public void initiate() {
+ super.initiate();
+ String filePrefix = Server.JSONDataGenConfiguration.get(Server.FILEOUTPUT_NAME).split(".adm")[0];
+ final String fileName = (filePrefix != null) ? directory + filePrefix : directory + "tempFile";
+ File f = new File(fileName);
+ if (Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME)) < 0) {
+ IntStream.range(0, Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITIONS_NAME)))
+ .forEach(readerId -> {
+ createFile(fileName, readerId);
+
+ });
+ } else { // Make File for the only partition that is mentioned
+ createFile(fileName, Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME)));
+ }
+ }
+
+ private void createFile(String fileName, int readerId) {
+ String execFilename = fileName + "p_" + readerId + ".adm";
+ File execFile = new File(execFilename);
+ try {
+ execFile.createNewFile();
+ threadToFileOutputStream.put(readerId, new BufferedOutputStream(new FileOutputStream(execFilename)));
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
new file mode 100644
index 0000000..9af3606
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser;
+
+public class PField {
+ public String name;
+ public String type;
+ public String order;
+ public int length;
+ public int range;
+ public boolean optional;
+ public double missings;
+ public boolean nullable;
+ public double nulls;
+ public boolean even;
+ public boolean odd;
+ public boolean variableLength = false;
+ // Generate word in case of a string?
+ public boolean word = false;
+ public double standardDeviation = 0.0;
+ // normal distribution? Or gamma distribution?
+ public boolean normalDistribution = false;
+ public boolean gammaDistribution = false;
+ // normal distribution mean
+ public double mean = 1;
+ // shape and scale in the gamma distribution
+ public double shape = 1.5;
+ public double scale = 1.5;
+
+ // Bernoulli distribution with prob = largeRecordProb.
+ public boolean bernoulliDistribution;
+ public double probLargeRecord = 0.05;
+ public int minSizeSmall = 187;
+ public int maxSizeSmall = 987;
+ public int minSizeLarge = 17787;
+ public int maxSizeLarge = 19787;
+
+ public boolean zipfDistribution = false;
+ public int zipfMinSize = 0;
+ public int zipfMaxSize = 30720;
+ public double zipfSkew = 2;
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
new file mode 100644
index 0000000..5c6ab7c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class Parser {
+ public Schema parseWisconsinConfigFile(String configFile) throws Exception {
+ List<PField> fields = new LinkedList<>();
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ File file = new File(configFile);
+ try {
+ fields = objectMapper.readValue(file, new TypeReference<List<PField>>() {
+ });
+ } catch (Exception e) {
+ System.err.println(e.toString());
+ e.printStackTrace();
+ }
+
+ return generateSchema(fields);
+ }
+
+ private Schema generateSchema(List<PField> pfields) throws Exception {
+ List<Field> fields = new LinkedList<>();
+ Field field;
+ for (PField f : pfields) {
+ field = new Field();
+ field.setName(f.name);
+ field.setNullable(f.nullable);
+ field.setNulls(f.nulls);
+ field.setOptional(f.optional);
+ field.setMissings(f.missings);
+ field.setRange(f.range);
+ field.setOdd(f.odd);
+ field.setEven(f.even);
+ field.setStringLength(f.length);
+ field.setVariableLength(f.variableLength);
+ field.setWord(f.word);
+ field.setNormalDistribution(f.normalDistribution);
+ field.setGammaDistribution(f.gammaDistribution);
+ field.setStandardDeviation(f.standardDeviation);
+ field.setShape(f.shape);
+ field.setScale(f.scale);
+ field.setBernoulliDistribution(f.bernoulliDistribution);
+ field.setProbLargeRecord(f.probLargeRecord);
+ field.setMinSizeSmall(f.minSizeSmall);
+ field.setMaxSizeSmall(f.maxSizeSmall);
+ field.setMinSizeLarge(f.minSizeLarge);
+ field.setMaxSizeLarge(f.maxSizeLarge);
+ field.setZipfDistribution(f.zipfDistribution);
+ field.setZipfMinSize(f.zipfMinSize);
+ field.setZipfMaxSize(f.zipfMaxSize);
+ field.setZipfSkew(f.zipfSkew);
+ Order.order order = f.order == null ? null
+ : (f.order.equalsIgnoreCase("random") ? Order.order.RANDOM : Order.order.SEQUENTIAL);
+ field.setOrder(order);
+ field.setMean(f.mean);
+ DataTypes.DataType dataType;
+
+ if (f.type.equalsIgnoreCase("string")) {
+ dataType = DataTypes.DataType.STRING;
+ } else if (f.type.equalsIgnoreCase("integer")) {
+ dataType = DataTypes.DataType.INTEGER;
+ } else if (f.type.equalsIgnoreCase("binary")) {
+ dataType = DataTypes.DataType.BINARY;
+ } else {
+ throw new Exception("Unknown Data Type");
+ }
+
+ field.setType(dataType);
+ fields.add(field);
+ }
+ Schema schema = new Schema();
+ schema.setFields(fields);
+ setPrimeAndGenerator(schema);
+ return schema;
+ }
+
+ private void setPrimeAndGenerator(Schema schema) {
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get("cardinality"));
+ long generator;
+ long prime;
+ if (cardinality <= Math.pow(10, 1)) {
+ prime = 11;
+ generator = 2;
+ } else if (cardinality <= Math.pow(10, 2)) {
+ prime = 101;
+ generator = 7;
+ } else if (cardinality <= Math.pow(10, 3)) {
+ prime = 1009;
+ generator = 26;
+
+ } else if (cardinality <= Math.pow(10, 4)) {
+ prime = 10007;
+ generator = 59;
+
+ } else if (cardinality <= Math.pow(10, 5)) {
+ prime = 100003;
+ generator = 242;
+
+ } else if (cardinality <= Math.pow(10, 6)) {
+ prime = 1000003;
+ generator = 568;
+ } else if (cardinality <= Math.pow(10, 7)) {
+ prime = 10000019;
+ generator = 1792;
+ } else if (cardinality <= Math.pow(10, 8)) {
+ prime = 100000007;
+ generator = 5649;
+ } else if (cardinality <= Math.pow(10, 9)) {
+ prime = 2147483647;
+ generator = 16807;
+ } else {
+ throw new IllegalArgumentException("Cannot generate rows more than 10^9");
+ }
+ schema.setGenerator(generator);
+ schema.setPrime(prime);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
new file mode 100644
index 0000000..7e8f4d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+
+public class Field {
+ private DataTypes.DataType type;
+ private String name;
+ private boolean optional;
+ private boolean nullable;
+ private double nulls;
+ private double missings;
+ private Order.order order;
+ private int stringLength;
+ private int range;
+ private boolean even;
+ private boolean odd;
+ private boolean variableLength;
+ // true: normal distribution, false: gamma distribution
+ private boolean normalDistribution;
+ private boolean gammaDistribution;
+ //For normal distribution
+ private double mean = 1;
+ // For the gamma distribution
+ private double shape = 1.5;
+ private double scale = 1.5;
+ // generate word?
+ private boolean word;
+ // standard deviation for the variable-length field
+ private double standardDeviation;
+ // Bernoulli distribution with prob = largeRecordProb.
+ private boolean bernoulliDistribution;
+ private double probLargeRecord = 0.5;
+ private int minSizeSmall = 100;
+ private int maxSizeSmall = 32 * 1024 - 1;
+ private int minSizeLarge = 32 * 1024;
+ private int maxSizeLarge = 5 * 32 * 1024;
+ private boolean zipfDistribution = false;
+ private int zipfMinSize = 0;
+ private int zipfMaxSize = 30720;
+ private double zipfSkew = 2;
+
+ public boolean isZipfDistribution() {
+ return zipfDistribution;
+ }
+
+ public int getZipfMinSize() {
+ return zipfMinSize;
+ }
+
+ public int getZipfMaxSize() {
+ return zipfMaxSize;
+ }
+
+ public double getZipfSkew() {
+ return zipfSkew;
+ }
+
+ public int getRange() {
+ return range;
+ }
+
+ public double getMean() {
+ return mean;
+ }
+
+ public void setMean(double mean) {
+ this.mean = mean;
+ }
+
+ public void setRange(int range) {
+ this.range = range;
+ }
+
+ public DataTypes.DataType getType() {
+ return type;
+ }
+
+ public void setType(DataTypes.DataType type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Order.order getOrder() {
+ return order;
+ }
+
+ public void setOrder(Order.order order) {
+ this.order = order;
+ }
+
+ public int getStringLength() {
+ return stringLength;
+ }
+
+ public void setStringLength(int stringLength) {
+ if (stringLength != 0)
+ this.stringLength = stringLength;
+ }
+
+ public boolean isOptional() {
+ return optional;
+ }
+
+ public void setOptional(boolean optional) {
+ this.optional = optional;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ public void setNullable(boolean nullable) {
+ this.nullable = nullable;
+ }
+
+ public double getNulls() {
+ return nulls;
+ }
+
+ public void setNulls(double nulls) {
+ this.nulls = nulls;
+ }
+
+ public double getMissings() {
+ return missings;
+ }
+
+ public void setMissings(double missings) {
+ this.missings = missings;
+ }
+
+ public boolean isEven() {
+ return even;
+ }
+
+ public void setEven(boolean even) {
+ this.even = even;
+ }
+
+ public boolean isOdd() {
+ return odd;
+ }
+
+ public void setOdd(boolean odd) {
+ this.odd = odd;
+ }
+
+ public boolean isVariableLength() {
+ return variableLength;
+ }
+
+ public void setVariableLength(boolean variableLength) {
+ this.variableLength = variableLength;
+ }
+
+ public double getStandardDeviation() {
+ return standardDeviation;
+ }
+
+ public void setStandardDeviation(double standardDeviation) {
+ this.standardDeviation = standardDeviation;
+ }
+
+ public boolean isWord() {
+ return word;
+ }
+
+ public void setWord(boolean word) {
+ this.word = word;
+ }
+
+ public boolean isNormalDistribution() {
+ return normalDistribution;
+ }
+
+ public boolean isGammaDistribution() {
+ return gammaDistribution;
+ }
+
+ public void setNormalDistribution(boolean normalDistribution) {
+ this.normalDistribution = normalDistribution;
+ }
+
+ public void setGammaDistribution(boolean gammaDistribution) {
+ this.gammaDistribution = gammaDistribution;
+ }
+
+ public double getShape() {
+ return shape;
+ }
+
+ public void setShape(double shape) {
+ this.shape = shape;
+ }
+
+ public double getScale() {
+ return scale;
+ }
+
+ public void setScale(double scale) {
+ this.scale = scale;
+ }
+
+ public boolean isBernoulliDistribution() {
+ return bernoulliDistribution;
+ }
+
+ public void setBernoulliDistribution(boolean bernoulliDistribution) {
+ this.bernoulliDistribution = bernoulliDistribution;
+ }
+
+ public double getProbLargeRecord() {
+ return probLargeRecord;
+ }
+
+ public void setProbLargeRecord(double probLargeRecord) {
+ this.probLargeRecord = probLargeRecord;
+ }
+
+ public int getMinSizeSmall() {
+ return minSizeSmall;
+ }
+
+ public void setMinSizeSmall(int minSizeSmall) {
+ this.minSizeSmall = minSizeSmall;
+ }
+
+ public int getMinSizeLarge() {
+ return minSizeLarge;
+ }
+
+ public void setMinSizeLarge(int minSizeLarge) {
+ this.minSizeLarge = minSizeLarge;
+ }
+
+ public int getMaxSizeSmall() {
+ return maxSizeSmall;
+ }
+
+ public void setMaxSizeSmall(int maxSizeSmall) {
+ this.maxSizeSmall = maxSizeSmall;
+ }
+
+ public int getMaxSizeLarge() {
+ return maxSizeLarge;
+ }
+
+ public void setMaxSizeLarge(int maxSizeLarge) {
+ this.maxSizeLarge = maxSizeLarge;
+ }
+
+ public void setZipfDistribution(boolean zipfDistribution) {
+ this.zipfDistribution = zipfDistribution;
+ }
+
+ public void setZipfMinSize(int zipfMinSize) {
+ this.zipfMinSize = zipfMinSize;
+ }
+
+ public void setZipfMaxSize(int zipfMaxSize) {
+ this.zipfMaxSize = zipfMaxSize;
+ }
+
+ public void setZipfSkew(double zipfSkew) {
+ this.zipfSkew = zipfSkew;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
new file mode 100644
index 0000000..4a484b4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class Schema {
+
+ private List<Field> fields;
+ private long prime;
+ private long generator;
+
+ public Schema() {
+ this.fields = new LinkedList<Field>();
+ }
+
+ public List<Field> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<Field> fields) {
+ this.fields = fields;
+ }
+
+ public long getPrime() {
+ return prime;
+ }
+
+ public void setPrime(long prime) {
+ this.prime = prime;
+ }
+
+ public long getGenerator() {
+ return generator;
+ }
+
+ public void setGenerator(long generator) {
+ this.generator = generator;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
new file mode 100644
index 0000000..b85fded
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinBinaryGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinNumberGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinStringGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.AWisconsinOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.WisconsinAsterixDBLoadOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.WisconsinFileOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser.Parser;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class Server {
+
+ private static Map<String, String> env = System.getenv();
+ private static String DATAGEN_HOME = env.get("DATAGEN_HOME");
+ public static String workloadsFolder = DATAGEN_HOME != null ? DATAGEN_HOME + "/Workloads/" : "Workloads/";
+ private static final Logger LOGGER = LogManager.getRootLogger();
+
+ // Configuration file name
+ private static final String PROPERTIES_FILE_NAME = "wisconsin_datagen.properties";
+ private static final String PROPERTIES_FILE_PATH_FIELD_NAME = "propertiesfilepath";
+
+ // // Properties field names - Dataset
+ public static final String WORKLOAD_NAME = "workload";
+ public static final String FILESIZE_NAME = "filesize";
+ public static final String CARDINALITY_NAME = "cardinality";
+ public static final String BATCHSIZE_NAME = "batchsize";
+ public static final String PARTITIONS_NAME = "partitions";
+ public static final String PARTITION_NAME = "partition";
+ public static final String FILEOUTPUT_NAME = "fileoutput";
+ public static final String WRITER_NAME = "writer";
+ public static final String ASTERIXDB_LOAD_PORT = "AsterixDBLoadPort";
+ public static final String HOST_NAME_FIELD_NAME = "hostname";
+ public static final Map<String, String> JSONDataGenConfiguration = new HashMap<>();
+
+ static {
+ JSONDataGenConfiguration.put(PROPERTIES_FILE_PATH_FIELD_NAME, PROPERTIES_FILE_NAME);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Map<String, String> commandLineCfg = processCommandLineConfig(args);
+ processAndSetConfiguration(commandLineCfg);
+
+ String workload = workloadsFolder + (JSONDataGenConfiguration.containsKey(WORKLOAD_NAME)
+ ? JSONDataGenConfiguration.get("workload") : "Default.json");
+
+ Parser parser = new Parser();
+ Schema schema = parser.parseWisconsinConfigFile(workload);
+ List<AWisconsinGenerator> generators = new LinkedList<>();
+ int fieldId = 0;
+ AWisconsinGenerator wg;
+ for (Field field : schema.getFields()) {
+ //use field Id
+ if (field.getType() == DataTypes.DataType.INTEGER) {
+ wg = new WisconsinNumberGenerator(schema, fieldId);
+ } else if (field.getType() == DataTypes.DataType.STRING) {
+ wg = new WisconsinStringGenerator(schema, fieldId);
+ } else if (field.getType() == DataTypes.DataType.BINARY) {
+ wg = new WisconsinBinaryGenerator(schema, fieldId);
+ } else {
+ throw new IllegalArgumentException(
+ "Invalid data type has been entered.Currently we only support String and Integer.");
+ }
+ generators.add(wg);
+ fieldId++;
+ }
+
+ // Default outputwriter is filewriter.
+ AWisconsinOutputGenerator outputGenerators = null;
+ String writer = JSONDataGenConfiguration.get(WRITER_NAME);
+ if (writer != null && writer.equalsIgnoreCase("asterixdb")) {
+ outputGenerators = new WisconsinAsterixDBLoadOutputGenerator(schema, generators);
+ } else {
+ outputGenerators = new WisconsinFileOutputGenerator(schema, generators);
+ }
+ if (outputGenerators != null) {
+ outputGenerators.execute();
+ }
+ }
+
+ private static Map<String, String> processCommandLineConfig(String[] args) {
+ Map<String, String> commandLineConfig = new HashMap<>();
+ if (args != null) {
+ for (String arg : args) {
+ if (arg.contains("=")) {
+ commandLineConfig.put(arg.substring(0, arg.indexOf("=")).toLowerCase(),
+ arg.substring(arg.indexOf("=") + 1));
+ }
+ }
+ }
+ return commandLineConfig;
+ }
+
+ private static void processAndSetConfiguration(Map<String, String> cmdlineConfig) {
+ // Command line configurations
+ String propertiesFilePath = null;
+
+ // Get the properties file if it was provided in command line arguments
+ for (Map.Entry<String, String> entry : cmdlineConfig.entrySet()) {
+ if (entry.getKey().equalsIgnoreCase(PROPERTIES_FILE_PATH_FIELD_NAME)) {
+ propertiesFilePath = entry.getValue();
+ break;
+ }
+ }
+
+ // Properties file configurations
+ Map<String, String> propertiesFileConfig = readPropertiesFileConfiguration(propertiesFilePath);
+
+ // Make sure the command line configs override the properties file configs
+ JSONDataGenConfiguration.putAll(propertiesFileConfig);
+ JSONDataGenConfiguration.putAll(cmdlineConfig);
+ }
+
+ /**
+ * Reads the configuration from the properties file. Uses the default properties file if path for properties file
+ * is not provided.
+ *
+ * @param propertiesFilePath properties file path that is provided by the user
+ * @return a map containing the read configuration
+ */
+ private static Map<String, String> readPropertiesFileConfiguration(String propertiesFilePath) {
+ Map<String, String> configs = new HashMap<>();
+ boolean isPropertiesFilePathProvided = false;
+
+ // User provided file path
+ if (propertiesFilePath != null) {
+ isPropertiesFilePathProvided = true;
+ LOGGER.info("Loaded properties file: " + propertiesFilePath);
+ } else {
+ // No file provided, use default
+ LOGGER.info("No properties file provided, using default properties file");
+ }
+
+ try (InputStream inputStream = isPropertiesFilePathProvided
+ ? new FileInputStream(Paths.get(propertiesFilePath).toFile())
+ : WisconsinFileOutputGenerator.class.getClassLoader().getResourceAsStream(PROPERTIES_FILE_NAME)) {
+
+ Properties properties = new Properties();
+
+ // Load the properties from the configuration file
+ if (inputStream != null) {
+ properties.load(inputStream);
+ } else {
+ // This should never happen, default configuration file exists in resources
+ LOGGER.error("Properties configuration file not found.");
+ }
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ configs.put(entry.getKey().toString().toLowerCase(), entry.getValue().toString());
+ System.out.println(entry.getKey().toString().toLowerCase() + ":" + entry.getValue().toString() + "\n");
+ }
+ } catch (FileNotFoundException ex) {
+ LOGGER.error("Configuration file not found. " + ex.getMessage());
+ } catch (Exception ex) {
+ LOGGER.error("Failed to read configuration file. " + ex.getMessage());
+ }
+
+ return configs;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
new file mode 100644
index 0000000..71ed8d8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Utils;
+
+public class Utils {
+
+ public static String getPrintableTimeDifference(long startTime, long endTime) {
+ long duration = endTime - startTime;
+ long seconds = (duration / 1000) % 60;
+ long minutes = ((duration / (1000 * 60)) % 60);
+ long hours = ((duration / (1000 * 60 * 60)) % 24);
+ return hours + ":" + minutes + ":" + seconds;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..d5302bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
@@ -0,0 +1,57 @@
+<!--
+#/*
+ #* Licensed to the Apache Software Foundation (ASF) under one
+ #* or more contributor license agreements. See the NOTICE file
+ #* distributed with this work for additional information
+ #* regarding copyright ownership. The ASF licenses this file
+ #* to you under the Apache License, Version 2.0 (the
+ #* "License"); you may not use this file except in compliance
+ #* with the License. You may obtain a copy of the License at
+ #*
+ #* http://www.apache.org/licenses/LICENSE-2.0
+ #*
+ #* Unless required by applicable law or agreed to in writing,
+ #* software distributed under the License is distributed on an
+ #* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ #* KIND, either express or implied. See the License for the
+ #* specific language governing permissions and limitations
+ #* under the License.
+ #*/
+ #/*Copyright (c) 2020 Shiva Jahangiri
+
+ #Permission is hereby granted, free of charge, to any person obtaining a copy
+ #of this software and associated documentation files (the "Software"), to deal
+ #in the Software without restriction, including without limitation the rights
+ #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ #copies of the Software, and to permit persons to whom the Software is
+ #furnished to do so, subject to the following conditions:
+
+ #The above copyright notice and this permission notice shall be included in all
+ #copies or substantial portions of the Software.
+
+ #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ #SOFTWARE.
+ #*/-->
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level - %msg%n"/>
+ </Console>
+ <File name="File" fileName="logs/logs.log">
+ <PatternLayout>
+ <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
+ </PatternLayout>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="debug" additivity="false">
+ <AppenderRef ref="Console"/>
+ <AppenderRef ref="File"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
new file mode 100644
index 0000000..fe6357f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
@@ -0,0 +1,51 @@
+#/*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*/
+#/*Copyright (c) 2020 Shiva Jahangiri
+
+#Permission is hereby granted, free of charge, to any person obtaining a copy
+#of this software and associated documentation files (the "Software"), to deal
+#in the Software without restriction, including without limitation the rights
+#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#copies of the Software, and to permit persons to whom the Software is
+#furnished to do so, subject to the following conditions:
+
+#The above copyright notice and this permission notice shall be included in all
+#copies or substantial portions of the Software.
+
+#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#SOFTWARE.
+#*/
+#Workload
+workload=Default.json
+cardinality=1000000
+filesize=-1
+fileoutput=output.adm
+#options for writer:file,asterixdb
+writer=file
+# Connection details
+hostname=localhost
+AsterixDBLoadPort=10001
+partition=0
+partitions=1
+batchsize=1000
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
Gerrit-Change-Number: 12967
Gerrit-PatchSet: 1
Gerrit-Owner: Shiva Jahangiri <sh...@uci.edu>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [NO ISSUE] Added Wisconsin Data Generator for tests
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Glenn Galvizo <gg...@uci.edu>:
Glenn Galvizo has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967 )
Change subject: [NO ISSUE] Added Wisconsin Data Generator for tests
......................................................................
Patch Set 2: Code-Review+1
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
Gerrit-Change-Number: 12967
Gerrit-PatchSet: 2
Gerrit-Owner: Shiva Jahangiri <sh...@uci.edu>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Glenn Galvizo <gg...@uci.edu>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Wed, 25 Aug 2021 19:39:54 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment
Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Shiva Jahangiri <sh...@uci.edu>:
Shiva Jahangiri has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967 )
Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................
PLEASE EDIT to provide a meaningful commit message!
[NO ISSUE] Added Wisconsin Data Generator for tests
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Wisconsin Data Generator has been added as
a separate package to generate data for related
test queries.
Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
22 files changed, 2,762 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/67/12967/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
new file mode 100644
index 0000000..a590995
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+MIT License
+
+Copyright (c) 2020 Shiva Jahangiri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
new file mode 100644
index 0000000..b40d9a0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+This project is under **MIT license** (for this software to be used as part of other project as modified or not modified) and can be referenced in scientific papers, books, journals, etc. upon the usage using **DOI: 10.5281/zenodo.4315680 (http://doi.org/10.5281/zenodo.4315937)**
+
+**Cite as:**
+Shiva Jahangiri. (2020, December 11). shivajah/JSON-Wisconsin-Data-Generator: First Release (Version v1.0.2). Zenodo. http://doi.org/10.5281/zenodo.4316003
+# JSON Wisconsin Data Generator: To JSON and Beyond
+
+[JSON Wisconsin Data Generator](https://github.com/shivajah/JSON-Wisconsin-Data-Generator) is a [Wisconsin](http://jimgray.azurewebsites.net/benchmarkhandbook/chapter4.pdf)-inspired data generator tool implemented in Java. Wisconsin Benchmark was designed at the University of Wisconsin by DeWitt et al to test the performance of the major components of a relational database system while the semantics and statistics of its relations would be easy to understand. This Benchmark was initially a single-user micro-benchmark for measuring individual query performance. A multi-user version of this benchmark which was developed a few years later did not attract as much as attraction as past, due to existence of other well-established competitors such as DebitCredit Benchmark, a multi-user OLTP benchmark led by Jim Gray.
+While the Wisconsin Benchmark does not have as large an audience today as in the past, we believe that its simple but powerful design easy to understand and highly controllable relations and attributes are unique and can be beneficial in evaluating database systems from many angles. As such, we have built a Wisconsin-inspired JSON data generator in java with more add-ons and advances features to support more modern data features, including: Variable length records based on size distributions, skewed attributes, and other features to support semi-structured data behavior as well.
+This data generator has been used in several studies and publications for benchmarking [AsterixDB](https://asterixdb.incubator.apache.org).
+
+## Wisconsin Benchmark:
+Figure below shows the schema for the Wisconsin Benchmark which is the default schema for our JSON Wisconsin Data Generator.
+<p align="center">
+ <img src="https://github.com/shivajah/JSON-Wisconsin-Data-Generator/blob/main/Wisconsin%20Benchmark-%20Original%20Schema.png">
+</p>
+
+## JSON Wisconsin Data Generator:
+
+### Inputs:
+The following JSON shows how this schema looks like in JSON to be used as an input to the program. A JSON array is used to show the schema for a relation while the JSON objects included in this array make the relation attributes. User can add more attributes by adding a new JSON object in the JSON array or drop them by simply removing them from the array.
+
+```JSON
+[
+ {
+ "name":"unique1",
+ "type":"integer",
+ "order":"random"
+},
+ {
+ "name":"unique2",
+ "type":"integer",
+ "order":"sequential"
+ },
+ {
+ "name":"two",
+ "type":"integer",
+ "order":"random",
+ "range":2
+ },
+ {
+ "name":"four",
+ "type":"integer",
+ "order":"random",
+ "range":4
+ },
+ {
+ "name":"ten",
+ "type":"integer",
+ "order":"random",
+ "range":10
+ },
+ {
+ "name":"twenty",
+ "type":"integer",
+ "order":"random",
+ "range":20
+ },
+ {
+ "name":"onePercent",
+ "type":"integer",
+ "order":"random",
+ "range":100
+ },
+ {
+ "name":"tenPercent",
+ "type":"integer",
+ "order":"random",
+ "range":10
+ },
+ {
+ "name":"twentyPercent",
+ "type":"integer",
+ "order":"random",
+ "range":5
+ },
+ {
+ "name":"fiftyPercent",
+ "type":"integer",
+ "order":"random",
+ "range":2
+ },
+ {
+ "name":"unique3",
+ "type":"integer",
+ "order":"random"
+ },
+ {
+ "name":"evenOnePercent",
+ "type":"integer",
+ "order":"random",
+ "even":true,
+ "range":100
+ },
+ {
+ "name":"oddOnePercent",
+ "type":"integer",
+ "order":"random",
+ "odd":true,
+ "range":100
+ },
+ {
+ "name":"stringu1",
+ "type":"string",
+ "length":"100",
+ "order":"sequential"
+ },
+ {
+ "name":"stringu2",
+ "type":"string",
+ "length":"100",
+ "order":"random"
+ },
+ {
+ "name":"string4",
+ "type":"string",
+ "length":"400",
+ "order":"random"
+ }
+]
+
+```
+
+### Outputs:
+
+The output of the program is a set of JSON objects where each object represents a record in the relation. Figure 3 shows an example of 5 output records.
+
+```JSON
+{"unique1":2, "unique2":1, "two":0, "four":2, "ten":2, "twenty":2, "onePercent":2, "tenPercent":2, "twentyPercent":2, "fiftyPercent":0, "unique3":2, "evenOnePercent":4, "oddOnePercent":5, "stringu1":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":4, "unique2":2, "two":0, "four":0, "ten":4, "twenty":4, "onePercent":4, "tenPercent":4, "twentyPercent":4, "fiftyPercent":0, "unique3":4, "evenOnePercent":8, "oddOnePercent":9, "stringu1":"CAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":1, "unique2":3, "two":1, "four":1, "ten":1, "twenty":1, "onePercent":1, "tenPercent":1, "twentyPercent":1, "fiftyPercent":1, "unique3":1, "evenOnePercent":2, "oddOnePercent":3, "stringu1":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"BAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":5, "unique2":4, "two":1, "four":1, "ten":5, "twenty":5, "onePercent":5, "tenPercent":5, "twentyPercent":0, "fiftyPercent":1, "unique3":5, "evenOnePercent":10, "oddOnePercent":11, "stringu1":"EAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+{"unique1":3, "unique2":5, "two":1, "four":3, "ten":3, "twenty":3, "onePercent":3, "tenPercent":3, "twentyPercent":3, "fiftyPercent":1, "unique3":3, "evenOnePercent":6, "oddOnePercent":7, "stringu1":"FAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "stringu2":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "string4":"DAAAAAAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}
+
+```
+
+### Usage:
+1. Checkout the [JSON Wisconsin Data Generator](https://github.com/shivajah/JSON-Wisconsin-Data-Generator) in a directory using git. In this document we assume that the path of the mentioned directory is $HOME/JsonWiscDataGen and we will refer to it as JSONWISCDATAGEN_HOME. You can choose any other directory as the home directory for JSON Wisconsin Data Generator.
+2. Set the $HOME/JsonWiscDataGen as an environment variable on your machine using the following command:
+```
+export JSONWISCDATAGEN_HOME=$HOME/JsonWiscDataGen
+```
+3. Go to the home directory of JSON Wisconsin Data Generator
+```
+cd $JSONWISCDATAGEN_HOME
+```
+4. Build the data generator using maven as follow:
+```
+mvn clean package
+```
+5. If the build is successful, a target folder will be created which contains the required jar file and dependencies. The workload file providing the schema and other relation information for Json Wisconsin Data Generator should be created as a json file in '$JSONWISCDATAGEN_HOME/Workloads' directory. There are some example workloads including the default workload in this folder. You can start creating your own workload file using these examples and information provided here.
+6. You can start generating the dataset by running the following command:
+```
+java -jar $JSONWISCDATAGEN_HOME/target/wisconsin-datagen.jar writer=file workload=default.json cardinality=99999999
+```
+7. Default configurations are provided in $JSONWISCDATAGEN_HOMEB/src/main/resources/wisconsin_datagen.properties which can be overwritten similar to the above example (step 6).
+- filesize: In MB, which would be another terminator for the program. Program will stop generating records if the asked cardinality of filesize is reached. Whichever that happen first, would be the terminator.
+- writer: asterixdb or file. "file" writes the output to a file in target folder with the name provided in "fileoutput". "asterixdb" writer loads the records directly to AsterixDB using AsterixDBLoadPort. For using the asterixdb writer, data generator should run on one of the NC nodes for it in order to work properly.
+
+### Workloads and Advanced Features:
+Sample workloads are provided in Workloads folder. The Default.json shows the schema for the original wisconsin benchmark relation. The Advanced.json has more fields with other features such as: varibale length strings, different length distribution, different value distributions( value skewness), real-word strings, nullable, and optional(missing) fields.
+In this section we explain these advanced features with some examples.
+
+## Strings:
+In the original Wisconsin Data Generator, strings are generated in a random or cyclic format. In the case of random, the string representation of the unique1 attribute is used as the prefix (which is unique as well) and enough ‘x’ characters will be added to the end of the string to reach to the desired length for that attribute. In the case of cyclic, string values are generated from the domain of four prefix in a cyclic format.In JSON Wisconsin Data Generator we only support the random format for generating the strings.
+### 1. Real-Word and HEX Strings:
+In addition to supporting the mentioned algorithm for generating strings, we support strings that are generated by concatenating words from a list made of 10,000 real words. This approach helps with reducing the impact of data compression due to less repetitive characters. In case of specifiying a distribution for the length of the strings, HEX strings will be generated.
+In order to generate strings made of real words, user can specify ```"word":true``` in the field definition. If word is set to true and string is set to be variable length```("variableLength":true)```, an average number of words to be included in the string is provided as an input, and algorithm concatenates as many as asked from the word list.
+### 2. Variable Length Strings:
+A string attribute can have values with different lengths by setting ```("variableLength":true)```. There are multiple options for the distribution based on which these lengths are generated.
+
+#### 2.1 Word:
+If for a variable length string, "```word=true``` then a Normal Distributin is used for calculating the actual number of words in the string. The ```length``` property of the string attribute is used as the average number of words, and the ```standardDeviation``` is used as the standard deviation used in normal distribution.
+#### 2.2 Hex String & Different Distributions on Length of the String:
+- If a string attribute is defined as variable length and ```"normalDistribution": true``` then a Hex String will be generated with the string length (```length``` property) as the mean and the standard deviation(```standardDeviation```) provided as a property for the string attribute.
+
+- If a string attribute is defined as variable length and ```"bernouliDistribution": true``` then user should also provide values for four other properties: ```"probLargeRecord", "minSizeSmall",maxSizeSmall","minSizeLarge","maxSizeLarge"```. The first property specifies what percentage of the strings should be large. The rest of the properties shows the maximum and minimum length for the small and large strings. First a bernouli distribution will be used for each record to determine if it is a large string or small. Then, a uniform distribution will choose the actual string length from the specified range for large and small strings. The generated string is a HEX string.
+
+- If ```"zipfDistribution": true```, the user should specify three other properties: ```"zipfMinSize","zipfMaxSize","zipfSkew"```. The first two properties specify the range of the length of the string, and the last property is the skewness of Zipf distribution. The rank is calculated based on the number of elements between the ```"zipfMinSize","zipfMaxSize"``` values. The generated string is a HEX string.
+
+- If none of the above distributions were selected, a Gamma distribution will be used to deterime the length of the string. The generated string is a HEX string.
+
+### 3. Fixed Length Strings:
+We offer different ways of generating fixed length strings. A fixed length string can either be made of words (```"word":true```) or based on the original Wisconsin Benchmark string generation technique. If ```"word":true```, then two distributions of Zipf and Uniform on selecting which word should be used (distribution on the value not the length of the string) can be performed. Extra 'X' characters can be appended to the string if the requested (but fixed) length of the string is larger than the length of the selected word.
+
+## Skewness in Integer Attribute Values:
+The integer attributes can be generated in three ways. The first way is to generate is as the original Wisconsin Benchmark would choose the integer value based on being sequential or random. The other ways are based on Gamma and Normal Distributions that we have defined in order to be able to introduce skew on the values of an integer attribute.
+
+## Nullable and Optional (missing) fields:
+While the original Wisconsin Benchmark does not provide a way to introduce null values, in our JSON Wisconsin Data Generator we have added the capability for having nullable and optional fields. Optional fields are mostly used in semi-structured data where the attribute itself maynot appear in some of the records. For specifying a nullable attribute, the user should set ```"nullable":true``` and also set the ```nulls``` to a value between 0 and 1. The latter attribute shows the probability of the attribute to have a null value for which a binomial distribution is used.
+For specifying the optional values, the user should set ```"optional":true``` and also set the ```missings``` to a value between 0 and 1. The latter attribute shows the probability of the attribute to be missing for which a binomial distribution is used.
+
+## Binary Data type:
+In addition to integer and string data types, we have also introduced Binary data type which generates HEX values.
+
+## Acknowledgement:
+Special thanks to Professor Michael Carey and Professor Johann-Christoph Freytag for their advice and support and to Taewoo Kim for his contribution to the development of variable length strings and binary data types.
+
+
+
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
new file mode 100644
index 0000000..9a6aff9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
@@ -0,0 +1,149 @@
+<!--
+#/*
+ #* Licensed to the Apache Software Foundation (ASF) under one
+ #* or more contributor license agreements. See the NOTICE file
+ #* distributed with this work for additional information
+ #* regarding copyright ownership. The ASF licenses this file
+ #* to you under the Apache License, Version 2.0 (the
+ #* "License"); you may not use this file except in compliance
+ #* with the License. You may obtain a copy of the License at
+ #*
+ #* http://www.apache.org/licenses/LICENSE-2.0
+ #*
+ #* Unless required by applicable law or agreed to in writing,
+ #* software distributed under the License is distributed on an
+ #* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ #* KIND, either express or implied. See the License for the
+ #* specific language governing permissions and limitations
+ #* under the License.
+ #*/
+ #/*Copyright (c) 2020 Shiva Jahangiri
+
+ #Permission is hereby granted, free of charge, to any person obtaining a copy
+ #of this software and associated documentation files (the "Software"), to deal
+ #in the Software without restriction, including without limitation the rights
+ #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ #copies of the Software, and to permit persons to whom the Software is
+ #furnished to do so, subject to the following conditions:
+
+ #The above copyright notice and this permission notice shall be included in all
+ #copies or substantial portions of the Software.
+
+ #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ #SOFTWARE.
+ #*/-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>edu.uci.ics</groupId>
+ <artifactId>datagenClient</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>Datagen Client</name>
+
+ <build>
+ <plugins>
+ <!-- JDK compiler level -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+
+ <!-- Assemble whole project into JAR -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>wisconsin-datagen</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20070829</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.6.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.couchbase.client</groupId>
+ <artifactId>java-client</artifactId>
+ <version>2.7.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.couchbase.client</groupId>
+ <artifactId>core-io</artifactId>
+ <version>1.7.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.teradata.tpcds</groupId>
+ <artifactId>tpcds</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.11.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
new file mode 100644
index 0000000..743b20c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+public class DataTypes {
+
+ public enum DataType {
+ STRING,
+ INTEGER,
+ BINARY
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
new file mode 100644
index 0000000..0380aaf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+public class Order {
+ public enum order {
+ RANDOM,
+ SEQUENTIAL
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
new file mode 100644
index 0000000..cc4f364
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.io.FileUtils;
+
+public class StringWordList {
+
+ private static List<String> wordList;
+ private static StringWordList instance;
+
+ StringWordList() throws IOException {
+ wordList = FileUtils.readLines(new File(Server.workloadsFolder + "wordlist"), "utf-8");
+ }
+
+ public static StringWordList getInstance() throws IOException {
+ if (instance == null) {
+ synchronized (StringWordList.class) {
+ if (instance == null) {
+ instance = new StringWordList();
+ }
+ }
+ }
+ return instance;
+ }
+
+ public List<String> getWordList() {
+ return wordList;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
new file mode 100644
index 0000000..19f34ad
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+
+public abstract class AWisconsinGenerator {
+ Schema schema;
+ int fieldId;
+ Random rand;
+ DataTypes.DataType generatorType;
+ double nullStartRang;
+ double nullEndRange;
+ double missingStartRange;
+ double missingEndRange;
+
+ public enum ValueType {
+ NULL,
+ MISSING,
+ VALUE
+ }
+
+ public AWisconsinGenerator(Schema schema, int fieldId) {
+ this.schema = schema;
+ this.fieldId = fieldId;
+ this.rand = new Random();
+ setUpNullAndMissingRanges();
+ }
+
+ private void setUpNullAndMissingRanges() {
+ this.nullStartRang = -1;
+ this.nullEndRange = -1;
+ this.missingStartRange = -1;
+ this.missingEndRange = -1;
+ if (schema.getFields().get(fieldId).isNullable()) {//NULL
+ this.nullStartRang = 0;
+ this.nullEndRange = schema.getFields().get(fieldId).getNulls();
+ if (schema.getFields().get(fieldId).isOptional()) { //NULL & MISSING
+ this.missingStartRange = nullEndRange;
+ this.missingEndRange = nullEndRange + schema.getFields().get(fieldId).getMissings();
+ }
+ } else if (schema.getFields().get(fieldId).isOptional()) {// MISSING
+ this.missingStartRange = 0;
+ this.missingEndRange = schema.getFields().get(fieldId).getMissings();
+ }
+
+ }
+
+ public abstract Object next(long i);
+
+ public long rand(long seed, long cardinality) {
+ do {
+ seed = (schema.getGenerator() * seed) % schema.getPrime();
+ } while (seed > cardinality);
+ return (seed);
+ }
+
+ public ValueType getValueType() {
+ float nextEvent = rand.nextFloat();
+ if (nextEvent >= nullStartRang && nextEvent < nullEndRange) {
+ return ValueType.NULL;
+ } else if (nextEvent >= missingStartRange && nextEvent < missingEndRange) {
+ return ValueType.MISSING;
+ }
+ return ValueType.VALUE;
+ }
+
+ public DataTypes.DataType getDataType() {
+ return generatorType;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
new file mode 100644
index 0000000..ced67b3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+import java.util.Stack;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinBinaryGenerator extends AWisconsinGenerator {
+ private static final char[] VALUES =
+ { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+ private static final int HEXRANGE = 15;
+ Random r;
+
+ public WisconsinBinaryGenerator(Schema schema, int fieldId) {
+ super(schema, fieldId);
+ r = new Random(fieldId);
+ generatorType = DataType.BINARY;
+ }
+
+ public String next(long seed) {
+ Field f = schema.getFields().get(fieldId);
+ if (getValueType() == ValueType.NULL) {
+ return "\"null\"";
+ } else if (getValueType() == ValueType.NULL) {
+ return "#MISSING";
+ }
+ if (f.getOrder() != null && f.getOrder() == Order.order.RANDOM) {
+ seed = rand(seed, Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME)));
+ }
+ if (f.getRange() > 0) {
+ seed = seed % f.getRange();
+ }
+ Stack<Character> temp = new Stack<>();
+
+ r.setSeed(seed);
+ char nextHex;
+ int length = schema.getFields().get(fieldId).getStringLength();
+ for (int i = 0; i < length; i++) {
+ nextHex = VALUES[r.nextInt(HEXRANGE)];
+ temp.push(nextHex);
+ }
+
+ char[] returnCharValues = new char[length];
+ int index = 0;
+ while (!temp.isEmpty()) {
+ returnCharValues[index] = temp.pop();
+ index++;
+ }
+
+ return String.valueOf(returnCharValues);
+
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
new file mode 100644
index 0000000..0110f55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.math3.distribution.GammaDistribution;
+
+public class WisconsinNumberGenerator extends AWisconsinGenerator {
+ GammaDistribution gd;
+ Random r;
+ Field field;
+
+ public WisconsinNumberGenerator(Schema schema, int fieldId) {
+ super(schema, fieldId);
+ generatorType = DataType.INTEGER;
+ field = schema.getFields().get(fieldId);
+ gd = new GammaDistribution(field.getShape(), field.getScale());
+ r = new Random(fieldId);
+ }
+
+ @Override
+ public Object next(long seed) {
+
+ long result;
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME));
+ ValueType type = getValueType();
+ if (type == ValueType.NULL) {
+ return "\"null\"";
+ } else if (type == ValueType.MISSING) {
+ return "#MISSING";
+ }
+ // Use cardinality and Jim Gray's algo
+ if (field.getOrder() != null && field.getOrder() == Order.order.RANDOM) {
+ result = rand(seed, cardinality);
+ } else
+ result = seed;
+ if (field.getRange() > 0) {
+ result = result % field.getRange();
+ }
+ if (field.isEven()) {
+ result *= 2;
+ }
+ if (field.isOdd()) {
+ result = result * 2 + 1;
+ }
+
+ if (field.isGammaDistribution()) {
+
+ long range = field.getRange() > 0 ? field.getRange() : cardinality;
+ double sample = gd.sample();
+ result = (int) ((sample * result) % range);
+
+ } else if (field.isNormalDistribution()) {
+ double mean = field.getMean() > 0 ? field.getMean() : cardinality / 2;
+ double stdev = field.getStandardDeviation();
+ result = (int) (r.nextGaussian() * stdev + mean);
+ return result;
+
+ }
+ return result;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
new file mode 100644
index 0000000..8274344
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.LinkedHashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.Stack;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.StringWordList;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.commons.math3.distribution.GammaDistribution;
+
+public class WisconsinStringGenerator extends AWisconsinGenerator {
+ private static final char[] VALUES =
+ { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+ private static final DecimalFormat decFormat = new DecimalFormat("#.######");
+
+ Random r;
+ StringWordList wordList;
+ StringBuilder builder;
+ Set<Integer> wordsSet;
+ int wordListSize;
+ long hexStringLength;
+ int maxHexStringLength;
+ int minHexStringLength;
+ int hexStringCount;
+ char[] returnHexCharValues;
+ GammaDistribution gd;
+ Field f;
+ ZipfGenerator zipfGenerator;
+
+ // To write the rawdata of gamma distribution.
+ BufferedWriter writer = null;
+ String fileName = "gamma_distribution_rawdata.txt";
+
+ public WisconsinStringGenerator(Schema schema, int fieldId) throws IOException {
+ super(schema, fieldId);
+ generatorType = DataType.STRING;
+ r = new Random(fieldId);
+ wordList = StringWordList.getInstance();
+ builder = new StringBuilder();
+ wordsSet = new LinkedHashSet<>();
+ wordListSize = wordList.getWordList().size();
+ hexStringLength = 0;
+ // Just default size
+ returnHexCharValues = new char[10];
+ maxHexStringLength = 0;
+ minHexStringLength = Integer.MAX_VALUE;
+ hexStringCount = 0;
+ f = schema.getFields().get(fieldId);
+ if (f.isVariableLength() && !f.isWord() && !f.isNormalDistribution()) {
+ gd = new GammaDistribution(f.getShape(), f.getScale());
+ writer = new BufferedWriter(new FileWriter(fileName));
+ }
+ if (f.isZipfDistribution()) {
+ if (f.isWord()) {
+ int bound = Math.min(wordListSize, f.getRange());
+ zipfGenerator = new ZipfGenerator(bound, f.getZipfSkew());
+ } else {
+ int rangeSize = f.getZipfMaxSize() - f.getZipfMinSize();
+ assert (rangeSize > 0);
+ zipfGenerator = new ZipfGenerator(rangeSize + 1, f.getZipfSkew());
+ }
+ }
+ }
+
+ public String next(long seed) {
+ ValueType type = getValueType();
+ if (type == ValueType.NULL) {
+ return "\"null\"";
+ } else if (type == ValueType.MISSING) {
+ return "#MISSING";
+ }
+ if (f.getOrder() != null && f.getOrder() == Order.order.RANDOM) {
+ seed = rand(seed, Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME)));
+ }
+ if (f.getRange() > 0) {
+ seed = seed % f.getRange();
+ }
+
+ if (f.isVariableLength()) {
+ if (f.isWord()) {
+ // Variable length word generation
+ // For this case, the length is regarded as the average word count.
+ return generateRandomWords(f.getStringLength(), f.getStandardDeviation());
+ } else if (f.isNormalDistribution()) {
+ // Variable length hex char string generation
+ // For this case, the length is regarded as the average length.
+ // return generateRandomHexChars();
+ return generateRandomHexCharsGaussian(f.getStringLength(), f.getStandardDeviation());
+ } else if (f.isBernoulliDistribution()) {
+ // Bernoulli Distribution: with some probability the record is large
+ if (r.nextFloat() < f.getProbLargeRecord()) {
+ return generateRandomHexCharsUniformDist(f.getMinSizeLarge(), f.getMaxSizeLarge());
+ }
+ return generateRandomHexCharsUniformDist(f.getMinSizeSmall(), f.getMaxSizeSmall());
+ } else if (f.isZipfDistribution()) {
+ return generateRandomHexCharsZipfianDist(f.getZipfMinSize());
+ } else {
+ // Gamma distribution case...
+ // Variable length hex char string generation
+ // For this case, the length is regarded as the average length.
+ // return generateRandomHexChars();
+ String result = generateRandomHexCharsGamma(f.getStringLength());
+ return result;
+ }
+ } else {//Fixed Length Records
+ if (f.isWord()) {
+ if (f.isZipfDistribution()) {
+ //Zipf distribution is for join skew not field length.
+ int rank = zipfGenerator.next();
+ String word = wordList.getWordList().get(rank);
+ return addPaddingX(word, f.isVariableLength());
+ } else {//uniform
+ int rank = r.nextInt(wordListSize);
+ if (f.getRange() > 0) {
+ rank = rank % f.getRange();
+ }
+ return addPaddingX(wordList.getWordList().get(rank), f.isVariableLength());
+
+ }
+ } else {
+ Stack<Character> temp = new Stack<>();
+ char[] field = { 'A', 'A', 'A', 'A', 'A', 'A', 'A' };
+ if (seed == 0) {
+ long rem = seed % 26;
+ temp.push((char) ('A' + rem));
+ }
+ while (seed > 0) {
+ long rem = seed % 26;
+ temp.push((char) ('A' + rem));
+ seed = seed / 26;
+ }
+ int i = 0;
+ while (!temp.empty() && i < field.length) {
+ field[i] = temp.pop();
+ i++;
+ }
+ return addPaddingX(String.valueOf(field), f.isVariableLength());
+ }
+ }
+ }
+
+ private String addPaddingX(String value, boolean variableLength) {
+ int stLen = f.getStringLength();
+ int diff = Math.max(0, stLen - value.length());
+ if (variableLength) {
+ diff = r.nextInt(diff);
+ }
+ for (int i = 0; i < diff; i++) {
+ value = value + 'X';
+ }
+ return value;
+ }
+
+ private String generateRandomWords(int wordCount, double standardDeviation) {
+ builder.delete(0, builder.length());
+ wordsSet.clear();
+
+ // Based on the Gaussian distribution, we get the actual word count.
+ int actualWordCount = (int) (r.nextGaussian() * standardDeviation + wordCount);
+ // At least it should be one.
+ actualWordCount = Math.max(actualWordCount, 1);
+ // At maximum, it cannot be greater than (stringLength + (stringLength - 1)) to make the distribution symmetric.
+ //
+ // x = (stringLength - 1) x = (stringLength - 1)
+ // 1 ------------------------ stringLength --------------------------- (stringLength + (stringLength - 1))
+ // since the left range is between 1 and the given stringLength.
+ actualWordCount = Math.min(actualWordCount, 2 * actualWordCount - 1);
+
+ int idx;
+ for (int i = 0; i < actualWordCount; i++) {
+ // Find a random number that is not in the set yet.
+ while (true) {
+ idx = r.nextInt(wordListSize);
+ if (!wordsSet.contains(idx)) {
+ wordsSet.add(idx);
+ builder.append(wordList.getWordList().get(idx));
+ if (i != (actualWordCount - 1)) {
+ builder.append(" ");
+ }
+ break;
+ }
+ }
+ }
+ return builder.toString();
+ }
+
+ private String generateRandomHexString(int length) {
+ if (length >= maxHexStringLength) {
+ maxHexStringLength = length;
+ returnHexCharValues = new char[maxHexStringLength];
+ }
+ if (length < minHexStringLength) {
+ minHexStringLength = length;
+ }
+ char nextHex;
+ for (int i = 0; i < length; i++) {
+ nextHex = VALUES[r.nextInt(VALUES.length)];
+ returnHexCharValues[i] = nextHex;
+ }
+ return String.valueOf(returnHexCharValues, 0, length);
+ }
+
+ private String generateRandomHexChars(int length) {
+ hexStringLength += length;
+ hexStringCount++;
+ return generateRandomHexString(length);
+ }
+
+ private String generateRandomHexCharsGaussian(int stringAvgLength, double standardDeviation) {
+ // Based on the Gaussian distribution, we get the actual word count.
+ int stringLength = (int) (r.nextGaussian() * standardDeviation + stringAvgLength);
+ // At least it should be one.
+ stringLength = Math.max(stringLength, 1);
+ // At maximum, it cannot be greater than (stringLength + (stringLength - 1)) to make the distribution symmetric.
+ //
+ // x = (stringLength - 1) x = (stringLength - 1)
+ // 1 ------------------------ stringLength --------------------------- (stringLength + (stringLength - 1))
+ // since the left range is between 1 and the given stringLength.
+ stringLength = Math.min(stringLength, 2 * stringLength - 1);
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsGamma(int multiplier) {
+ // Based on the gamma distribution...
+ // At least the actual string length should be one.
+ int stringLength = (int) (gd.sample() * multiplier) + 1;
+ //System.out.print(stringLength+",");
+ try {
+ writer.write(Integer.toString(stringLength) + "\n");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsUniformDist(int minSize, int maxSize) {
+ int rangeSize = maxSize - minSize;
+ assert (rangeSize > 0);
+ int stringLength = r.nextInt(rangeSize + 1) + minSize;
+ return generateRandomHexChars(stringLength);
+ }
+
+ private String generateRandomHexCharsZipfianDist(int minSize) {
+ int rank = zipfGenerator.next(); // a number in the range of max-min
+ int stringLength = rank + minSize;
+ return generateRandomHexChars(stringLength);
+ }
+
+ public long getHexStringLength() {
+ return hexStringLength;
+ }
+
+ public String getAvgHexStringLength() {
+ if (hexStringLength > 0 && hexStringCount > 0) {
+ return ("Hex-string-count: " + hexStringCount + " Total length(KB): " + (double) hexStringLength / 1024
+ + " Max: " + maxHexStringLength + " Min: " + minHexStringLength + " Avg: "
+ + decFormat.format((double) hexStringLength / hexStringCount) + " (KB): "
+ + decFormat.format((double) hexStringLength / hexStringCount / 1024));
+ }
+ return "";
+ }
+
+ public void closeGammaDistributionFile() {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
new file mode 100644
index 0000000..8caa34e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators;
+
+import java.util.Random;
+
+public class ZipfGenerator {
+ private Random rnd = new Random(System.currentTimeMillis());
+ private int size;
+ private double skew;
+ private double bottom = 0;
+
+ public ZipfGenerator(int size, double skew) {
+ this.size = size;
+ this.skew = skew;
+
+ for (int i = 1; i < size; i++) {
+ this.bottom += (1 / Math.pow(i, this.skew));
+ }
+ }
+
+ // the next() method returns an random rank id.
+ // The frequency of returned rank ids are follows Zipf distribution.
+ public int next() {
+ int rank;
+ double frequency = 0;
+ double dice;
+
+ rank = rnd.nextInt(size);
+ frequency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ dice = rnd.nextDouble();
+
+ while (!(dice < frequency)) {
+ rank = rnd.nextInt(size);
+ frequency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ dice = rnd.nextDouble();
+ }
+
+ return rank;
+ }
+
+ // This method returns a probability that the given rank occurs.
+ public double getProbability(int rank) {
+ return (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+ }
+
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
new file mode 100644
index 0000000..52378d9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.BufferedOutputStream;
+import java.text.DecimalFormat;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes.DataType;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinStringGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Utils.Utils;
+import org.apache.commons.math3.util.Pair;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public abstract class AWisconsinOutputGenerator {
+
+ private static final Logger LOGGER = LogManager.getRootLogger();
+ protected Schema schema;
+ private List<AWisconsinGenerator> generators;
+ final String directory = "./";
+ protected Map<Integer, BufferedOutputStream> threadToFileOutputStream;
+ private static long maxRecordLength;
+ protected ExecutorService executorService;
+ //A map from executor sequence id to a map for start and end row index
+ private Map<Integer, Pair<Long, Long>> executorsToStartAndEnd;
+ private int partitions;
+ private int partition;
+ private static final DecimalFormat decFormat = new DecimalFormat("#.######");
+ //TODO: totalFileSize should be calculated from the record size to know the exact start and end row index(if the file size is deciding where we finish instead of number of records
+ public double totalFileSize;
+ public Map<Long, Integer> sizeToCount;
+
+ public AWisconsinOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ this.partitions = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITIONS_NAME));
+ this.partition = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME));
+ this.schema = schema;
+ this.generators = generators;
+ this.threadToFileOutputStream = new HashMap<>();
+ this.maxRecordLength = 0;
+ this.totalFileSize = 0;
+ this.executorsToStartAndEnd = new HashMap<>();
+ this.sizeToCount = new HashMap<>();
+ }
+
+ private void initExecutors() {
+ // if partition == -1, then all partitions should get created (in parallel)
+ if (partition < 0) {
+ executorService = Executors.newFixedThreadPool(partitions);
+ } else { // Otherwise, only the mentioned partition will be generated using one thread
+ executorService = Executors.newFixedThreadPool(1);
+ }
+ }
+
+ private void initReaderToStartAndEnd() {
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.CARDINALITY_NAME));
+ long length = cardinality / partitions;
+ long start;
+ for (int i = 0; i < partitions; i++) {
+ start = Math.max(1, length * i);
+ Pair<Long, Long> p =
+ i == partitions - 1 ? new Pair<>(start, cardinality) : new Pair<>(start, start + length);
+ executorsToStartAndEnd.put(i, p);
+ }
+ }
+
+ public void execute() {
+ if (partition < 0) { // All partitions
+ IntStream.range(0, partitions).forEach(readerId -> {
+ executorService.submit(() -> {
+ executorTask(readerId);
+ });
+ });
+ } else { //only one partition
+ executorService.submit(() -> executorTask(partition));
+ }
+ shutDownExecutors();
+
+ }
+
+ private void executorTask(int readerId) {
+ int batchIndex = 0; // batch1, batch2,...
+ int batchSize = Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.BATCHSIZE_NAME)); // Maximum number of records in a batch
+ long fileSize = Long.valueOf(Server.JSONDataGenConfiguration.get(Server.FILESIZE_NAME));
+ int recordCount = 0;
+ long totalRecordLength = 0;
+ long minRecordLength = Long.MAX_VALUE;
+ long startTime = System.currentTimeMillis();
+ List<String> batchOfRecords = new LinkedList<>();
+
+ LOGGER.info("start index: " + executorsToStartAndEnd.get(readerId).getKey() + " end index: "
+ + executorsToStartAndEnd.get(readerId).getValue());
+
+ for (long id = executorsToStartAndEnd.get(readerId).getKey(); id <= executorsToStartAndEnd.get(readerId)
+ .getValue(); id++) {
+ if (fileSize > 0 && totalFileSize >= fileSize * 1024 * 1024) {
+ break;
+ }
+ if (id % batchSize == 0) {
+ LOGGER.info("Thread " + readerId + " Created " + id + " records in "
+ + Utils.getPrintableTimeDifference(startTime, System.currentTimeMillis()));
+ }
+ long currentRecordsize = 0;
+ String record = "{";
+
+ boolean allFieldsNull = true;
+ for (int i = 0; i < schema.getFields().size(); i++) {
+
+ Object val = generators.get(i).next(id);
+ if (val instanceof String && val == "#MISSING") {
+ continue;
+ } else {
+ try {
+ boolean comma = record.equalsIgnoreCase("{") ? false : true;
+ if (comma) {
+ record = record + ", ";
+ }
+ allFieldsNull = false;
+ record = record + "\"" + schema.getFields().get(i).getName() + "\":";
+ if (generators.get(i).getDataType() == DataType.BINARY) {
+ record = record + "hex(\"" + val + "\")";
+ currentRecordsize += ((String) val).length();
+ } else if (generators.get(i).getDataType() == DataType.STRING) {
+ record = record + "\"" + val + "\"";
+ currentRecordsize += ((String) val).length();
+ } else if (generators.get(i).getDataType() == DataType.INTEGER) {
+ record = record + val;
+ currentRecordsize += 8;
+ }
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+ }
+ }
+ record = record + "}\n";
+ if (allFieldsNull)
+ continue;
+ if (batchOfRecords.size() >= batchSize) {
+ // Empty the batch
+ write(readerId, batchOfRecords, batchIndex);
+ batchIndex++;
+ batchOfRecords.clear();
+ }
+ batchOfRecords.add(record);
+
+ if (!sizeToCount.containsKey(currentRecordsize)) {
+ sizeToCount.put(currentRecordsize, 0);
+ }
+ int count = sizeToCount.get(currentRecordsize);
+ sizeToCount.put(currentRecordsize, count + 1);
+
+ totalFileSize += currentRecordsize;
+ if (currentRecordsize > maxRecordLength) {
+ maxRecordLength = currentRecordsize;
+ } else if (currentRecordsize < minRecordLength) {
+ minRecordLength = currentRecordsize;
+ }
+ totalRecordLength += currentRecordsize;
+ recordCount++;
+ }
+ if (batchOfRecords.size() > 0) {
+ write(readerId, batchOfRecords, batchIndex);
+ }
+ closeWriter(readerId);
+
+ // Stats info logging
+ LOGGER.info("Thread " + readerId + " Processed " + recordCount + " in "
+ + Utils.getPrintableTimeDifference(startTime, System.currentTimeMillis()));
+ LOGGER.info("Record Generation Done. recordCount: " + recordCount + " minRecordLength: " + minRecordLength
+ + " maxRecordLength: " + maxRecordLength + " totalRecordLength: " + totalRecordLength
+ + " totalFileSize: " + (totalFileSize / (double) 1024 / (double) 1024) + " (MB)"
+ + " avg record length: " + decFormat.format((double) totalRecordLength / recordCount));
+
+ // Calculate and print string length on average
+ LOGGER.info(printStringLengthInfo());
+ LOGGER.info(Arrays.asList(sizeToCount));
+
+ }
+
+ private void shutDownExecutors() {
+ try {
+ executorService.shutdown();
+ executorService.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); // TODO: Is this necessary?
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } finally {
+ if (!executorService.isTerminated()) {
+ System.out.println("canceling all pending tasks");
+ }
+ executorService.shutdownNow();
+ System.out.println("Shutdown complete!");
+ }
+ }
+
+ //Stats Printing
+ private String printStringLengthInfo() {
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < schema.getFields().size(); i++) {
+ if (generators.get(i).getDataType() == DataType.STRING) {
+ long length = ((WisconsinStringGenerator) generators.get(i)).getHexStringLength();
+ // Variable Length?
+ if (length > 0) {
+ strBuilder.append(((WisconsinStringGenerator) generators.get(i)).getAvgHexStringLength() + "\n");
+ ((WisconsinStringGenerator) generators.get(i)).closeGammaDistributionFile();
+ }
+ }
+ }
+ return strBuilder.toString();
+ }
+
+ // Abstract Methods
+ public abstract void write(int threadId, List<String> batchOfRecords, int batchIndex);
+
+ public abstract void closeWriter(int threadID);
+
+ public void initiate() {
+ initExecutors();
+ initReaderToStartAndEnd();
+ };
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
new file mode 100644
index 0000000..29a6253
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinAsterixDBLoadOutputGenerator extends AWisconsinOutputGenerator {
+
+ Socket sock;
+ OutputStream output = null;
+ PrintWriter writer;
+
+ public WisconsinAsterixDBLoadOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ super(schema, generators);
+ initiate();
+ }
+
+ @Override
+ public void write(int threadId, List<String> batchOfRecords, int batchIndex) {
+ for (String record : batchOfRecords) {
+ writer.write(record);
+ }
+ }
+
+ @Override
+ public void closeWriter(int threadID) {
+ try {
+ writer.flush();
+ writer.close();
+ sock.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ @Override
+ public void initiate() {
+ super.initiate();
+ try {
+ sock = new Socket(Server.JSONDataGenConfiguration.get(Server.HOST_NAME_FIELD_NAME),
+ Integer.parseInt(Server.JSONDataGenConfiguration.get(Server.ASTERIXDB_LOAD_PORT)));
+ output = sock.getOutputStream();
+ writer = new PrintWriter(output, true);
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
new file mode 100644
index 0000000..8f70ead
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.IntStream;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+public class WisconsinFileOutputGenerator extends AWisconsinOutputGenerator {
+
+ public WisconsinFileOutputGenerator(Schema schema, List<AWisconsinGenerator> generators) {
+ super(schema, generators);
+ initiate();
+ }
+
+ @Override
+ public void write(int threadId, List<String> batchOfRecords, int batchIndex) {
+ for (String record : batchOfRecords) {
+ try {
+ threadToFileOutputStream.get(threadId).write(record.getBytes());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public void closeWriter(int threadId) {
+ try {
+ threadToFileOutputStream.get(threadId).close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public void initiate() {
+ super.initiate();
+ String filePrefix = Server.JSONDataGenConfiguration.get(Server.FILEOUTPUT_NAME).split(".adm")[0];
+ final String fileName = (filePrefix != null) ? directory + filePrefix : directory + "tempFile";
+ File f = new File(fileName);
+ if (Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME)) < 0) {
+ IntStream.range(0, Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITIONS_NAME)))
+ .forEach(readerId -> {
+ createFile(fileName, readerId);
+
+ });
+ } else { // Make File for the only partition that is mentioned
+ createFile(fileName, Integer.valueOf(Server.JSONDataGenConfiguration.get(Server.PARTITION_NAME)));
+ }
+ }
+
+ private void createFile(String fileName, int readerId) {
+ String execFilename = fileName + "p_" + readerId + ".adm";
+ File execFile = new File(execFilename);
+ try {
+ execFile.createNewFile();
+ threadToFileOutputStream.put(readerId, new BufferedOutputStream(new FileOutputStream(execFilename)));
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
new file mode 100644
index 0000000..9af3606
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser;
+
+public class PField {
+ public String name;
+ public String type;
+ public String order;
+ public int length;
+ public int range;
+ public boolean optional;
+ public double missings;
+ public boolean nullable;
+ public double nulls;
+ public boolean even;
+ public boolean odd;
+ public boolean variableLength = false;
+ // Generate word in case of a string?
+ public boolean word = false;
+ public double standardDeviation = 0.0;
+ // normal distribution? Or gamma distribution?
+ public boolean normalDistribution = false;
+ public boolean gammaDistribution = false;
+ // normal distribution mean
+ public double mean = 1;
+ // shape and scale in the gamma distribution
+ public double shape = 1.5;
+ public double scale = 1.5;
+
+ // Bernoulli distribution with prob = largeRecordProb.
+ public boolean bernoulliDistribution;
+ public double probLargeRecord = 0.05;
+ public int minSizeSmall = 187;
+ public int maxSizeSmall = 987;
+ public int minSizeLarge = 17787;
+ public int maxSizeLarge = 19787;
+
+ public boolean zipfDistribution = false;
+ public int zipfMinSize = 0;
+ public int zipfMaxSize = 30720;
+ public double zipfSkew = 2;
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
new file mode 100644
index 0000000..5c6ab7c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Server;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class Parser {
+ public Schema parseWisconsinConfigFile(String configFile) throws Exception {
+ List<PField> fields = new LinkedList<>();
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ File file = new File(configFile);
+ try {
+ fields = objectMapper.readValue(file, new TypeReference<List<PField>>() {
+ });
+ } catch (Exception e) {
+ System.err.println(e.toString());
+ e.printStackTrace();
+ }
+
+ return generateSchema(fields);
+ }
+
+ private Schema generateSchema(List<PField> pfields) throws Exception {
+ List<Field> fields = new LinkedList<>();
+ Field field;
+ for (PField f : pfields) {
+ field = new Field();
+ field.setName(f.name);
+ field.setNullable(f.nullable);
+ field.setNulls(f.nulls);
+ field.setOptional(f.optional);
+ field.setMissings(f.missings);
+ field.setRange(f.range);
+ field.setOdd(f.odd);
+ field.setEven(f.even);
+ field.setStringLength(f.length);
+ field.setVariableLength(f.variableLength);
+ field.setWord(f.word);
+ field.setNormalDistribution(f.normalDistribution);
+ field.setGammaDistribution(f.gammaDistribution);
+ field.setStandardDeviation(f.standardDeviation);
+ field.setShape(f.shape);
+ field.setScale(f.scale);
+ field.setBernoulliDistribution(f.bernoulliDistribution);
+ field.setProbLargeRecord(f.probLargeRecord);
+ field.setMinSizeSmall(f.minSizeSmall);
+ field.setMaxSizeSmall(f.maxSizeSmall);
+ field.setMinSizeLarge(f.minSizeLarge);
+ field.setMaxSizeLarge(f.maxSizeLarge);
+ field.setZipfDistribution(f.zipfDistribution);
+ field.setZipfMinSize(f.zipfMinSize);
+ field.setZipfMaxSize(f.zipfMaxSize);
+ field.setZipfSkew(f.zipfSkew);
+ Order.order order = f.order == null ? null
+ : (f.order.equalsIgnoreCase("random") ? Order.order.RANDOM : Order.order.SEQUENTIAL);
+ field.setOrder(order);
+ field.setMean(f.mean);
+ DataTypes.DataType dataType;
+
+ if (f.type.equalsIgnoreCase("string")) {
+ dataType = DataTypes.DataType.STRING;
+ } else if (f.type.equalsIgnoreCase("integer")) {
+ dataType = DataTypes.DataType.INTEGER;
+ } else if (f.type.equalsIgnoreCase("binary")) {
+ dataType = DataTypes.DataType.BINARY;
+ } else {
+ throw new Exception("Unknown Data Type");
+ }
+
+ field.setType(dataType);
+ fields.add(field);
+ }
+ Schema schema = new Schema();
+ schema.setFields(fields);
+ setPrimeAndGenerator(schema);
+ return schema;
+ }
+
+ private void setPrimeAndGenerator(Schema schema) {
+ long cardinality = Long.valueOf(Server.JSONDataGenConfiguration.get("cardinality"));
+ long generator;
+ long prime;
+ if (cardinality <= Math.pow(10, 1)) {
+ prime = 11;
+ generator = 2;
+ } else if (cardinality <= Math.pow(10, 2)) {
+ prime = 101;
+ generator = 7;
+ } else if (cardinality <= Math.pow(10, 3)) {
+ prime = 1009;
+ generator = 26;
+
+ } else if (cardinality <= Math.pow(10, 4)) {
+ prime = 10007;
+ generator = 59;
+
+ } else if (cardinality <= Math.pow(10, 5)) {
+ prime = 100003;
+ generator = 242;
+
+ } else if (cardinality <= Math.pow(10, 6)) {
+ prime = 1000003;
+ generator = 568;
+ } else if (cardinality <= Math.pow(10, 7)) {
+ prime = 10000019;
+ generator = 1792;
+ } else if (cardinality <= Math.pow(10, 8)) {
+ prime = 100000007;
+ generator = 5649;
+ } else if (cardinality <= Math.pow(10, 9)) {
+ prime = 2147483647;
+ generator = 16807;
+ } else {
+ throw new IllegalArgumentException("Cannot generate rows more than 10^9");
+ }
+ schema.setGenerator(generator);
+ schema.setPrime(prime);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
new file mode 100644
index 0000000..7e8f4d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.Order;
+
+public class Field {
+ private DataTypes.DataType type;
+ private String name;
+ private boolean optional;
+ private boolean nullable;
+ private double nulls;
+ private double missings;
+ private Order.order order;
+ private int stringLength;
+ private int range;
+ private boolean even;
+ private boolean odd;
+ private boolean variableLength;
+ // true: normal distribution, false: gamma distribution
+ private boolean normalDistribution;
+ private boolean gammaDistribution;
+ //For normal distribution
+ private double mean = 1;
+ // For the gamma distribution
+ private double shape = 1.5;
+ private double scale = 1.5;
+ // generate word?
+ private boolean word;
+ // standard deviation for the variable-length field
+ private double standardDeviation;
+ // Bernoulli distribution with prob = largeRecordProb.
+ private boolean bernoulliDistribution;
+ private double probLargeRecord = 0.5;
+ private int minSizeSmall = 100;
+ private int maxSizeSmall = 32 * 1024 - 1;
+ private int minSizeLarge = 32 * 1024;
+ private int maxSizeLarge = 5 * 32 * 1024;
+ private boolean zipfDistribution = false;
+ private int zipfMinSize = 0;
+ private int zipfMaxSize = 30720;
+ private double zipfSkew = 2;
+
+ public boolean isZipfDistribution() {
+ return zipfDistribution;
+ }
+
+ public int getZipfMinSize() {
+ return zipfMinSize;
+ }
+
+ public int getZipfMaxSize() {
+ return zipfMaxSize;
+ }
+
+ public double getZipfSkew() {
+ return zipfSkew;
+ }
+
+ public int getRange() {
+ return range;
+ }
+
+ public double getMean() {
+ return mean;
+ }
+
+ public void setMean(double mean) {
+ this.mean = mean;
+ }
+
+ public void setRange(int range) {
+ this.range = range;
+ }
+
+ public DataTypes.DataType getType() {
+ return type;
+ }
+
+ public void setType(DataTypes.DataType type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Order.order getOrder() {
+ return order;
+ }
+
+ public void setOrder(Order.order order) {
+ this.order = order;
+ }
+
+ public int getStringLength() {
+ return stringLength;
+ }
+
+ public void setStringLength(int stringLength) {
+ if (stringLength != 0)
+ this.stringLength = stringLength;
+ }
+
+ public boolean isOptional() {
+ return optional;
+ }
+
+ public void setOptional(boolean optional) {
+ this.optional = optional;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ public void setNullable(boolean nullable) {
+ this.nullable = nullable;
+ }
+
+ public double getNulls() {
+ return nulls;
+ }
+
+ public void setNulls(double nulls) {
+ this.nulls = nulls;
+ }
+
+ public double getMissings() {
+ return missings;
+ }
+
+ public void setMissings(double missings) {
+ this.missings = missings;
+ }
+
+ public boolean isEven() {
+ return even;
+ }
+
+ public void setEven(boolean even) {
+ this.even = even;
+ }
+
+ public boolean isOdd() {
+ return odd;
+ }
+
+ public void setOdd(boolean odd) {
+ this.odd = odd;
+ }
+
+ public boolean isVariableLength() {
+ return variableLength;
+ }
+
+ public void setVariableLength(boolean variableLength) {
+ this.variableLength = variableLength;
+ }
+
+ public double getStandardDeviation() {
+ return standardDeviation;
+ }
+
+ public void setStandardDeviation(double standardDeviation) {
+ this.standardDeviation = standardDeviation;
+ }
+
+ public boolean isWord() {
+ return word;
+ }
+
+ public void setWord(boolean word) {
+ this.word = word;
+ }
+
+ public boolean isNormalDistribution() {
+ return normalDistribution;
+ }
+
+ public boolean isGammaDistribution() {
+ return gammaDistribution;
+ }
+
+ public void setNormalDistribution(boolean normalDistribution) {
+ this.normalDistribution = normalDistribution;
+ }
+
+ public void setGammaDistribution(boolean gammaDistribution) {
+ this.gammaDistribution = gammaDistribution;
+ }
+
+ public double getShape() {
+ return shape;
+ }
+
+ public void setShape(double shape) {
+ this.shape = shape;
+ }
+
+ public double getScale() {
+ return scale;
+ }
+
+ public void setScale(double scale) {
+ this.scale = scale;
+ }
+
+ public boolean isBernoulliDistribution() {
+ return bernoulliDistribution;
+ }
+
+ public void setBernoulliDistribution(boolean bernoulliDistribution) {
+ this.bernoulliDistribution = bernoulliDistribution;
+ }
+
+ public double getProbLargeRecord() {
+ return probLargeRecord;
+ }
+
+ public void setProbLargeRecord(double probLargeRecord) {
+ this.probLargeRecord = probLargeRecord;
+ }
+
+ public int getMinSizeSmall() {
+ return minSizeSmall;
+ }
+
+ public void setMinSizeSmall(int minSizeSmall) {
+ this.minSizeSmall = minSizeSmall;
+ }
+
+ public int getMinSizeLarge() {
+ return minSizeLarge;
+ }
+
+ public void setMinSizeLarge(int minSizeLarge) {
+ this.minSizeLarge = minSizeLarge;
+ }
+
+ public int getMaxSizeSmall() {
+ return maxSizeSmall;
+ }
+
+ public void setMaxSizeSmall(int maxSizeSmall) {
+ this.maxSizeSmall = maxSizeSmall;
+ }
+
+ public int getMaxSizeLarge() {
+ return maxSizeLarge;
+ }
+
+ public void setMaxSizeLarge(int maxSizeLarge) {
+ this.maxSizeLarge = maxSizeLarge;
+ }
+
+ public void setZipfDistribution(boolean zipfDistribution) {
+ this.zipfDistribution = zipfDistribution;
+ }
+
+ public void setZipfMinSize(int zipfMinSize) {
+ this.zipfMinSize = zipfMinSize;
+ }
+
+ public void setZipfMaxSize(int zipfMaxSize) {
+ this.zipfMaxSize = zipfMaxSize;
+ }
+
+ public void setZipfSkew(double zipfSkew) {
+ this.zipfSkew = zipfSkew;
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
new file mode 100644
index 0000000..4a484b4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class Schema {
+
+ private List<Field> fields;
+ private long prime;
+ private long generator;
+
+ public Schema() {
+ this.fields = new LinkedList<Field>();
+ }
+
+ public List<Field> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<Field> fields) {
+ this.fields = fields;
+ }
+
+ public long getPrime() {
+ return prime;
+ }
+
+ public void setPrime(long prime) {
+ this.prime = prime;
+ }
+
+ public long getGenerator() {
+ return generator;
+ }
+
+ public void setGenerator(long generator) {
+ this.generator = generator;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
new file mode 100644
index 0000000..b85fded
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Constants.DataTypes;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.AWisconsinGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinBinaryGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinNumberGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.FieldGenerators.WisconsinStringGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.AWisconsinOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.WisconsinAsterixDBLoadOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.OutputGenerator.WisconsinFileOutputGenerator;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Parser.Parser;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Field;
+import org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Schema.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class Server {
+
+ private static Map<String, String> env = System.getenv();
+ private static String DATAGEN_HOME = env.get("DATAGEN_HOME");
+ public static String workloadsFolder = DATAGEN_HOME != null ? DATAGEN_HOME + "/Workloads/" : "Workloads/";
+ private static final Logger LOGGER = LogManager.getRootLogger();
+
+ // Configuration file name
+ private static final String PROPERTIES_FILE_NAME = "wisconsin_datagen.properties";
+ private static final String PROPERTIES_FILE_PATH_FIELD_NAME = "propertiesfilepath";
+
+ // // Properties field names - Dataset
+ public static final String WORKLOAD_NAME = "workload";
+ public static final String FILESIZE_NAME = "filesize";
+ public static final String CARDINALITY_NAME = "cardinality";
+ public static final String BATCHSIZE_NAME = "batchsize";
+ public static final String PARTITIONS_NAME = "partitions";
+ public static final String PARTITION_NAME = "partition";
+ public static final String FILEOUTPUT_NAME = "fileoutput";
+ public static final String WRITER_NAME = "writer";
+ public static final String ASTERIXDB_LOAD_PORT = "AsterixDBLoadPort";
+ public static final String HOST_NAME_FIELD_NAME = "hostname";
+ public static final Map<String, String> JSONDataGenConfiguration = new HashMap<>();
+
+ static {
+ JSONDataGenConfiguration.put(PROPERTIES_FILE_PATH_FIELD_NAME, PROPERTIES_FILE_NAME);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Map<String, String> commandLineCfg = processCommandLineConfig(args);
+ processAndSetConfiguration(commandLineCfg);
+
+ String workload = workloadsFolder + (JSONDataGenConfiguration.containsKey(WORKLOAD_NAME)
+ ? JSONDataGenConfiguration.get("workload") : "Default.json");
+
+ Parser parser = new Parser();
+ Schema schema = parser.parseWisconsinConfigFile(workload);
+ List<AWisconsinGenerator> generators = new LinkedList<>();
+ int fieldId = 0;
+ AWisconsinGenerator wg;
+ for (Field field : schema.getFields()) {
+ //use field Id
+ if (field.getType() == DataTypes.DataType.INTEGER) {
+ wg = new WisconsinNumberGenerator(schema, fieldId);
+ } else if (field.getType() == DataTypes.DataType.STRING) {
+ wg = new WisconsinStringGenerator(schema, fieldId);
+ } else if (field.getType() == DataTypes.DataType.BINARY) {
+ wg = new WisconsinBinaryGenerator(schema, fieldId);
+ } else {
+ throw new IllegalArgumentException(
+ "Invalid data type has been entered.Currently we only support String and Integer.");
+ }
+ generators.add(wg);
+ fieldId++;
+ }
+
+ // Default outputwriter is filewriter.
+ AWisconsinOutputGenerator outputGenerators = null;
+ String writer = JSONDataGenConfiguration.get(WRITER_NAME);
+ if (writer != null && writer.equalsIgnoreCase("asterixdb")) {
+ outputGenerators = new WisconsinAsterixDBLoadOutputGenerator(schema, generators);
+ } else {
+ outputGenerators = new WisconsinFileOutputGenerator(schema, generators);
+ }
+ if (outputGenerators != null) {
+ outputGenerators.execute();
+ }
+ }
+
+ private static Map<String, String> processCommandLineConfig(String[] args) {
+ Map<String, String> commandLineConfig = new HashMap<>();
+ if (args != null) {
+ for (String arg : args) {
+ if (arg.contains("=")) {
+ commandLineConfig.put(arg.substring(0, arg.indexOf("=")).toLowerCase(),
+ arg.substring(arg.indexOf("=") + 1));
+ }
+ }
+ }
+ return commandLineConfig;
+ }
+
+ private static void processAndSetConfiguration(Map<String, String> cmdlineConfig) {
+ // Command line configurations
+ String propertiesFilePath = null;
+
+ // Get the properties file if it was provided in command line arguments
+ for (Map.Entry<String, String> entry : cmdlineConfig.entrySet()) {
+ if (entry.getKey().equalsIgnoreCase(PROPERTIES_FILE_PATH_FIELD_NAME)) {
+ propertiesFilePath = entry.getValue();
+ break;
+ }
+ }
+
+ // Properties file configurations
+ Map<String, String> propertiesFileConfig = readPropertiesFileConfiguration(propertiesFilePath);
+
+ // Make sure the command line configs override the properties file configs
+ JSONDataGenConfiguration.putAll(propertiesFileConfig);
+ JSONDataGenConfiguration.putAll(cmdlineConfig);
+ }
+
+ /**
+ * Reads the configuration from the properties file. Uses the default properties file if path for properties file
+ * is not provided.
+ *
+ * @param propertiesFilePath properties file path that is provided by the user
+ * @return a map containing the read configuration
+ */
+ private static Map<String, String> readPropertiesFileConfiguration(String propertiesFilePath) {
+ Map<String, String> configs = new HashMap<>();
+ boolean isPropertiesFilePathProvided = false;
+
+ // User provided file path
+ if (propertiesFilePath != null) {
+ isPropertiesFilePathProvided = true;
+ LOGGER.info("Loaded properties file: " + propertiesFilePath);
+ } else {
+ // No file provided, use default
+ LOGGER.info("No properties file provided, using default properties file");
+ }
+
+ try (InputStream inputStream = isPropertiesFilePathProvided
+ ? new FileInputStream(Paths.get(propertiesFilePath).toFile())
+ : WisconsinFileOutputGenerator.class.getClassLoader().getResourceAsStream(PROPERTIES_FILE_NAME)) {
+
+ Properties properties = new Properties();
+
+ // Load the properties from the configuration file
+ if (inputStream != null) {
+ properties.load(inputStream);
+ } else {
+ // This should never happen, default configuration file exists in resources
+ LOGGER.error("Properties configuration file not found.");
+ }
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ configs.put(entry.getKey().toString().toLowerCase(), entry.getValue().toString());
+ System.out.println(entry.getKey().toString().toLowerCase() + ":" + entry.getValue().toString() + "\n");
+ }
+ } catch (FileNotFoundException ex) {
+ LOGGER.error("Configuration file not found. " + ex.getMessage());
+ } catch (Exception ex) {
+ LOGGER.error("Failed to read configuration file. " + ex.getMessage());
+ }
+
+ return configs;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
new file mode 100644
index 0000000..71ed8d8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*Copyright (c) 2020 Shiva Jahangiri
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+package org.apache.asterix.wisconsindatagenerator.src.main.java.com.datagen.Utils;
+
+public class Utils {
+
+ public static String getPrintableTimeDifference(long startTime, long endTime) {
+ long duration = endTime - startTime;
+ long seconds = (duration / 1000) % 60;
+ long minutes = ((duration / (1000 * 60)) % 60);
+ long hours = ((duration / (1000 * 60 * 60)) % 24);
+ return hours + ":" + minutes + ":" + seconds;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..d5302bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
@@ -0,0 +1,57 @@
+<!--
+#/*
+ #* Licensed to the Apache Software Foundation (ASF) under one
+ #* or more contributor license agreements. See the NOTICE file
+ #* distributed with this work for additional information
+ #* regarding copyright ownership. The ASF licenses this file
+ #* to you under the Apache License, Version 2.0 (the
+ #* "License"); you may not use this file except in compliance
+ #* with the License. You may obtain a copy of the License at
+ #*
+ #* http://www.apache.org/licenses/LICENSE-2.0
+ #*
+ #* Unless required by applicable law or agreed to in writing,
+ #* software distributed under the License is distributed on an
+ #* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ #* KIND, either express or implied. See the License for the
+ #* specific language governing permissions and limitations
+ #* under the License.
+ #*/
+ #/*Copyright (c) 2020 Shiva Jahangiri
+
+ #Permission is hereby granted, free of charge, to any person obtaining a copy
+ #of this software and associated documentation files (the "Software"), to deal
+ #in the Software without restriction, including without limitation the rights
+ #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ #copies of the Software, and to permit persons to whom the Software is
+ #furnished to do so, subject to the following conditions:
+
+ #The above copyright notice and this permission notice shall be included in all
+ #copies or substantial portions of the Software.
+
+ #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ #SOFTWARE.
+ #*/-->
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level - %msg%n"/>
+ </Console>
+ <File name="File" fileName="logs/logs.log">
+ <PatternLayout>
+ <Pattern>%d %p %c{1.} [%t] %m%n</Pattern>
+ </PatternLayout>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="debug" additivity="false">
+ <AppenderRef ref="Console"/>
+ <AppenderRef ref="File"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
new file mode 100644
index 0000000..fe6357f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
@@ -0,0 +1,51 @@
+#/*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*/
+#/*Copyright (c) 2020 Shiva Jahangiri
+
+#Permission is hereby granted, free of charge, to any person obtaining a copy
+#of this software and associated documentation files (the "Software"), to deal
+#in the Software without restriction, including without limitation the rights
+#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#copies of the Software, and to permit persons to whom the Software is
+#furnished to do so, subject to the following conditions:
+
+#The above copyright notice and this permission notice shall be included in all
+#copies or substantial portions of the Software.
+
+#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#SOFTWARE.
+#*/
+#Workload
+workload=Default.json
+cardinality=1000000
+filesize=-1
+fileoutput=output.adm
+#options for writer:file,asterixdb
+writer=file
+# Connection details
+hostname=localhost
+AsterixDBLoadPort=10001
+partition=0
+partitions=1
+batchsize=1000
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
Gerrit-Change-Number: 12967
Gerrit-PatchSet: 1
Gerrit-Owner: Shiva Jahangiri <sh...@uci.edu>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [NO ISSUE] Added Wisconsin Data Generator for tests
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Jenkins <je...@fulliautomatix.ics.uci.edu>:
Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967 )
Change subject: [NO ISSUE] Added Wisconsin Data Generator for tests
......................................................................
Patch Set 4: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/12442/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
Gerrit-Change-Number: 12967
Gerrit-PatchSet: 4
Gerrit-Owner: Shiva Jahangiri <sh...@uci.edu>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Glenn Galvizo <gg...@uci.edu>
Gerrit-Reviewer: Ian Maxon <im...@uci.edu>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Comment-Date: Thu, 02 Sep 2021 21:51:52 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment
Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Shiva Jahangiri <sh...@uci.edu>:
Hello Glenn Galvizo, Anon. E. Moose #1000171, Till Westmann, Jenkins, Ian Maxon, Dmitry Lychagin,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
to look at the new patch set (#3).
Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................
PLEASE EDIT to provide a meaningful commit message!
[NO ISSUE] Added Wisconsin Data Generator for tests
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Wisconsin Data Generator has been added as
a separate package to generate data for related
test queries.
Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
---
M asterixdb/asterix-app/pom.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/LICENSE
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/README.md
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/pom.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/DataTypes.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/Order.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Constants/StringWordList.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/AWisconsinGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinBinaryGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinNumberGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/WisconsinStringGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/FieldGenerators/ZipfGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/AWisconsinOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinAsterixDBLoadOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/OutputGenerator/WisconsinFileOutputGenerator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/PField.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Parser/Parser.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Field.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Schema/Schema.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Server.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/java/com/datagen/Utils/Utils.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/log4j2.xml
A asterixdb/asterix-app/src/test/java/org/apache/asterix/wisconsindatagenerator/src/main/resources/wisconsin_datagen.properties
23 files changed, 2,367 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/67/12967/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12967
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie13be468ce2de60200864759a7eca4550132a2a3
Gerrit-Change-Number: 12967
Gerrit-PatchSet: 3
Gerrit-Owner: Shiva Jahangiri <sh...@uci.edu>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Glenn Galvizo <gg...@uci.edu>
Gerrit-Reviewer: Ian Maxon <im...@uci.edu>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-MessageType: newpatchset