You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/10/15 18:01:59 UTC
[5/8] asterixdb git commit: Move Twitter Firehose Datasource to Test
Source Folder
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
new file mode 100644
index 0000000..77b8df1
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IInputStreamFactory;
+import org.apache.asterix.external.input.stream.TwitterFirehoseInputStream;
+import org.apache.asterix.runtime.util.ClusterStateManager;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Factory class for creating @see{TwitterFirehoseFeedAdapter}. The adapter
+ * simulates a twitter firehose with tweets being "pushed" into Asterix at a
+ * configurable rate measured in terms of TPS (tweets/second). The stream of
+ * tweets lasts for a configurable duration (measured in seconds).
+ */
+public class TwitterFirehoseStreamFactory implements IInputStreamFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Degree of parallelism for feed ingestion activity. Defaults to 1. This
+ * determines the count constraint for the ingestion operator.
+ **/
+ private static final String KEY_INGESTION_CARDINALITY = "ingestion-cardinality";
+
+ /**
+ * The absolute locations where ingestion operator instances will be placed.
+ **/
+ private static final String KEY_INGESTION_LOCATIONS = "ingestion-location";
+
+ private Map<String, String> configuration;
+
+ @Override
+ public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
+ String ingestionCardinalityParam = configuration.get(KEY_INGESTION_CARDINALITY);
+ String ingestionLocationParam = configuration.get(KEY_INGESTION_LOCATIONS);
+ String[] locations = null;
+ if (ingestionLocationParam != null) {
+ locations = ingestionLocationParam.split(",");
+ }
+ int count = locations != null ? locations.length : 1;
+ if (ingestionCardinalityParam != null) {
+ count = Integer.parseInt(ingestionCardinalityParam);
+ }
+
+ List<String> chosenLocations = new ArrayList<>();
+ String[] availableLocations = locations != null ? locations
+ : ClusterStateManager.INSTANCE.getParticipantNodes().toArray(new String[] {});
+ for (int i = 0, k = 0; i < count; i++, k = (k + 1) % availableLocations.length) {
+ chosenLocations.add(availableLocations[k]);
+ }
+ return new AlgebricksAbsolutePartitionConstraint(chosenLocations.toArray(new String[] {}));
+ }
+
+ @Override
+ public DataSourceType getDataSourceType() {
+ return DataSourceType.STREAM;
+ }
+
+ @Override
+ public void configure(Map<String, String> configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public boolean isIndexible() {
+ return false;
+ }
+
+ @Override
+ public AsterixInputStream createInputStream(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ try {
+ return new TwitterFirehoseInputStream(configuration, ctx, partition);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/pom.xml b/asterixdb/asterix-tools/pom.xml
index 011d44b..c039c48 100644
--- a/asterixdb/asterix-tools/pom.xml
+++ b/asterixdb/asterix-tools/pom.xml
@@ -54,6 +54,14 @@
<include>**/DEPENDENCIES*</include>
</includes>
</configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
@@ -63,19 +71,20 @@
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-lang-aql</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-algebra</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-external-data</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
@@ -88,8 +97,7 @@
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-metadata</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>test</scope>
</dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdgClientDriver.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
deleted file mode 100644
index 8a1adb0..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.datagen;
-
-import java.io.File;
-
-import org.kohsuke.args4j.Argument;
-import org.kohsuke.args4j.CmdLineParser;
-
-public class AdgClientDriver {
-
- public static final int NUMBER_OF_ARGUMENTS = 2;
-
- public static class AdgClientConfig {
-
- @Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
- private File schemaFile;
-
- @Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
- private File outputDir;
- }
-
- public static void main(String[] args) throws Exception {
- AdgClientConfig acc = new AdgClientConfig();
- CmdLineParser cmdParser = new CmdLineParser(acc);
- try {
- cmdParser.parseArgument(args);
- } catch (Exception e) {
- cmdParser.printUsage(System.err);
- throw e;
- }
- AdmDataGen adg = new AdmDataGen(acc.schemaFile, acc.outputDir);
- adg.init();
- adg.dataGen();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdmDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdmDataGen.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdmDataGen.java
deleted file mode 100644
index 2918575..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/AdmDataGen.java
+++ /dev/null
@@ -1,1020 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.datagen;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.asterix.common.annotations.AutoDataGen;
-import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
-import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
-import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
-import org.apache.asterix.common.annotations.FieldIntervalDataGen;
-import org.apache.asterix.common.annotations.FieldValFileDataGen;
-import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
-import org.apache.asterix.common.annotations.IRecordFieldDataGen;
-import org.apache.asterix.common.annotations.IRecordTypeAnnotation;
-import org.apache.asterix.common.annotations.IRecordTypeAnnotation.Kind;
-import org.apache.asterix.common.annotations.InsertRandIntDataGen;
-import org.apache.asterix.common.annotations.ListDataGen;
-import org.apache.asterix.common.annotations.ListValFileDataGen;
-import org.apache.asterix.common.annotations.RecordDataGenAnnotation;
-import org.apache.asterix.common.annotations.TypeDataGen;
-import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.transactions.JobId;
-import org.apache.asterix.lang.aql.parser.AQLParserFactory;
-import org.apache.asterix.lang.aql.parser.ParseException;
-import org.apache.asterix.lang.common.base.IParser;
-import org.apache.asterix.lang.common.base.IParserFactory;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.metadata.MetadataException;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.AUnionType;
-import org.apache.asterix.om.types.AbstractCollectionType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeSignature;
-import org.apache.asterix.om.util.NonTaggedFormatUtil;
-import org.apache.asterix.tools.translator.ADGenDmlTranslator;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
-import org.apache.hyracks.algebricks.data.utils.WriteValueTools;
-
-public class AdmDataGen {
-
- class DataGeneratorContext {
- private Map<File, String[]> fileToLoadedDataMap = new HashMap<File, String[]>();
-
- public Map<File, String[]> getFileToLoadedDataMap() {
- return fileToLoadedDataMap;
- }
- }
-
- public static void printDate(int year, int month, int day, PrintStream out) throws IOException {
- WriteValueTools.writeInt(year, out);
- out.print("-");
- if (month < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(month, out);
- out.print("-");
- if (day < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(day, out);
- }
-
- abstract class AbstractValueGenerator {
- protected PrintStream out;
- protected DataGeneratorContext ctx;
-
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- this.out = out;
- this.ctx = ctx;
- }
-
- public abstract void generate() throws IOException;
- }
-
- abstract class RandomValueGenerator extends AbstractValueGenerator {
- protected Random rnd;
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- this.rnd = new Random(51);
- }
-
- }
-
- class IntAutoGenerator extends AbstractValueGenerator {
-
- private final int initValue;
- private int val;
-
- public IntAutoGenerator(int initValue) {
- this.initValue = initValue;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- val = initValue;
- }
-
- @Override
- public void generate() throws IOException {
- WriteValueTools.writeInt(val, out);
- val++;
- }
-
- }
-
- class LongAutoGenerator extends AbstractValueGenerator {
-
- private final long initValue;
- private long val;
-
- public LongAutoGenerator(long initValue) {
- this.initValue = initValue;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- val = initValue;
- }
-
- @Override
- public void generate() throws IOException {
- WriteValueTools.writeLong(val, out);
- out.print("i64");
- val++;
- }
-
- }
-
- class StringFromArrayGenerator extends RandomValueGenerator {
- private final String[][] array;
- private int lastIndex;
- private final String constructor;
- private String[] chosen;
-
- public StringFromArrayGenerator(String[][] array, String constructor) {
- this.array = array;
- this.constructor = constructor;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- this.lastIndex = -1;
- this.chosen = new String[array.length];
- }
-
- @Override
- public void generate() throws IOException {
- for (int i = 0; i < array.length; i++) {
- lastIndex = Math.abs(rnd.nextInt()) % array[i].length;
- chosen[i] = array[i][lastIndex];
- }
- if (constructor != null) {
- out.print(constructor);
- out.print("(");
- }
- out.print("\"");
- for (int i = 0; i < chosen.length; i++) {
- out.print(chosen[i]);
- }
- out.print("\"");
- if (constructor != null) {
- out.print(")");
- }
- }
-
- public int getLastIndex() {
- return lastIndex;
- }
- }
-
- abstract class AbstractCollectionDataGenerator extends RandomValueGenerator {
- protected final AbstractCollectionType act;
- protected final int min;
- protected final int max;
- protected final String startList;
- protected final String endList;
-
- public AbstractCollectionDataGenerator(AbstractCollectionType act, int min, int max) {
- this.act = act;
- this.min = min;
- this.max = max;
- if (act.getTypeTag() == ATypeTag.ORDEREDLIST) {
- startList = "[";
- endList = "]";
- } else {
- startList = "{{";
- endList = "}}";
- }
- }
-
- }
-
- class ListDataGenerator extends AbstractCollectionDataGenerator {
-
- private AbstractValueGenerator itemGen;
-
- public ListDataGenerator(AbstractCollectionType act, int min, int max) {
- super(act, min, max);
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- IAType t = act.getItemType();
- if (t.getTypeTag() != ATypeTag.RECORD) {
- throw new NotImplementedException("list annotation only works with record item types for now.");
- }
- ARecordType rt = (ARecordType) t;
- RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
- if (dga == null) {
- throw new Exception("No annotation on item type " + t);
- }
- itemGen = new RecordGenerator(rt, dga, null);
- itemGen.init(out, ctx);
- }
-
- @Override
- public void generate() throws IOException {
- out.print(startList);
- int numItems = rnd.nextInt(max - min + 1) + min;
- for (int i = 0; i < numItems; i++) {
- if (i > 0) {
- out.print(", ");
- }
- itemGen.generate();
- }
- out.print(endList);
- }
-
- }
-
- class ListFromArrayGenerator extends AbstractCollectionDataGenerator {
-
- private final String[] array;
- private String constructor;
-
- public ListFromArrayGenerator(AbstractCollectionType act, String[] array, int min, int max) {
- super(act, min, max);
- this.array = array;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- if (act.getItemType().getTypeTag() == ATypeTag.STRING) {
- constructor = null;
- } else {
- constructor = getConstructor(act.getItemType());
- }
- }
-
- @Override
- public void generate() throws IOException {
- out.print(startList);
- int numItems = rnd.nextInt(max - min + 1) + min;
- for (int i = 0; i < numItems; i++) {
- if (i > 0) {
- out.print(", ");
- }
- int c = Math.abs(rnd.nextInt()) % array.length;
- if (constructor != null) {
- out.print(constructor);
- out.print("(");
- }
- out.print("\"");
- out.print(array[c]);
- out.print("\"");
- if (constructor != null) {
- out.print(")");
- }
- }
- out.print(endList);
- }
-
- }
-
- class StringFromArraySameIndexGenerator extends AbstractValueGenerator {
- private final String[] array;
- private final StringFromArrayGenerator sfag;
- private final String constructor;
-
- public StringFromArraySameIndexGenerator(String[] array, StringFromArrayGenerator sfag, String constructor) {
- this.array = array;
- this.sfag = sfag;
- this.constructor = constructor;
- }
-
- @Override
- public void generate() throws IOException {
- String val = array[sfag.getLastIndex()];
- if (constructor != null) {
- out.print(constructor);
- out.print("(");
- }
- out.print("\"");
- out.print(val);
- out.print("\"");
- if (constructor != null) {
- out.print(")");
- }
- }
- }
-
- class IntIntervalGenerator extends RandomValueGenerator {
-
- private final int min;
- private final int max;
- private final String prefix;
- private final String suffix;
-
- public IntIntervalGenerator(int min, int max, String prefix, String suffix) {
- this.min = min;
- this.max = max;
- this.prefix = prefix;
- this.suffix = suffix;
- }
-
- @Override
- public void generate() throws IOException {
- int v = Math.abs(rnd.nextInt() % (max - min + 1)) + min;
- if (prefix != null) {
- out.print(prefix);
- }
- WriteValueTools.writeInt(v, out);
- if (suffix != null) {
- out.print(suffix);
- }
- }
-
- }
-
- class LongIntervalGenerator extends RandomValueGenerator {
-
- private final long min;
- private final long max;
- private final String prefix;
- private final String suffix;
-
- public LongIntervalGenerator(long min, long max, String prefix, String suffix) {
- this.min = min;
- this.max = max;
- this.prefix = prefix;
- this.suffix = suffix;
- }
-
- @Override
- public void generate() throws IOException {
- long v = Math.abs(rnd.nextLong() % (max - min + 1)) + min;
- if (prefix != null) {
- out.print(prefix);
- }
- WriteValueTools.writeLong(v, out);
- if (suffix != null) {
- out.print(suffix);
- }
- }
-
- }
-
- class DoubleIntervalGenerator extends RandomValueGenerator {
-
- private final double min;
- private final double max;
- private final String prefix;
- private final String suffix;
-
- public DoubleIntervalGenerator(double min, double max, String prefix, String suffix) {
- this.min = min;
- this.max = max;
- this.prefix = prefix;
- this.suffix = suffix;
- }
-
- @Override
- public void generate() throws IOException {
- double v = Math.abs(rnd.nextDouble() % (max - min)) + min;
- if (prefix != null) {
- out.print(prefix);
- }
- out.print(v);
- if (suffix != null) {
- out.print(suffix);
- }
- }
-
- }
-
- class InsertRandIntGenerator extends RandomValueGenerator {
-
- private final String str1;
- private final String str2;
-
- public InsertRandIntGenerator(String str1, String str2) {
- this.str1 = "\"" + str1;
- this.str2 = str2 + "\"";
- }
-
- @Override
- public void generate() throws IOException {
- int v = Math.abs(rnd.nextInt());
- out.print(str1);
- WriteValueTools.writeInt(v, out);
- out.print(str2);
- }
-
- }
-
- interface AccessibleDateGenerator {
- public int getYear();
-
- public int getMonth();
-
- public int getDay();
- }
-
- abstract class DateGenerator extends RandomValueGenerator implements AccessibleDateGenerator {
-
- private int year;
- private int month;
- private int day;
-
- protected void generateDate(int minYear, int maxYear) throws IOException {
- year = rnd.nextInt(maxYear - minYear + 1) + minYear;
- month = Math.abs(rnd.nextInt()) % 12 + 1;
- day = Math.abs(rnd.nextInt()) % 28 + 1;
- printDate(year, month, day, out);
- }
-
- @Override
- public int getYear() {
- return year;
- }
-
- @Override
- public int getMonth() {
- return month;
- }
-
- @Override
- public int getDay() {
- return day;
- }
- }
-
- class DateBetweenYearsGenerator extends DateGenerator {
-
- private final int minYear;
- private final int maxYear;
-
- public DateBetweenYearsGenerator(int minYear, int maxYear) {
- this.minYear = minYear;
- this.maxYear = maxYear;
- }
-
- @Override
- public void generate() throws IOException {
- out.print("date(\"");
- generateDate(minYear, maxYear);
- out.print("\")");
- }
-
- }
-
- interface AccessibleDatetimeGenerator extends AccessibleDateGenerator {
- public int getHour();
-
- public int getMinute();
-
- public int getSecond();
- }
-
- class DatetimeBetweenYearsGenerator extends DateGenerator implements AccessibleDatetimeGenerator {
- private final int minYear;
- private final int maxYear;
- private int hour;
-
- public DatetimeBetweenYearsGenerator(int minYear, int maxYear) {
- this.minYear = minYear;
- this.maxYear = maxYear;
- }
-
- @Override
- public void generate() throws IOException {
- out.print("datetime(\"");
- generateDate(minYear, maxYear);
- out.print("T");
- hour = rnd.nextInt(24);
- if (hour < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(hour, out);
- out.print(":00:00\")");
- }
-
- @Override
- public int getHour() {
- return hour;
- }
-
- @Override
- public int getMinute() {
- return 0;
- }
-
- @Override
- public int getSecond() {
- return 0;
- }
-
- }
-
- class DatetimeAddRandHoursGenerator extends RandomValueGenerator {
-
- private final int minHours;
- private final int maxHours;
- private final AccessibleDatetimeGenerator adtg;
-
- public DatetimeAddRandHoursGenerator(int minHours, int maxHours, AccessibleDatetimeGenerator adtg) {
- this.minHours = minHours;
- this.maxHours = maxHours;
- this.adtg = adtg;
- }
-
- @Override
- public void generate() throws IOException {
- int addHours = rnd.nextInt(maxHours - minHours + 1) + minHours;
-
- out.print("datetime(\"");
- printDate(adtg.getYear(), adtg.getMonth(), adtg.getDay(), out);
- out.print("T");
- int h = adtg.getHour() + addHours;
- if (h > 23) {
- h = 23;
- }
- if (h < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(h, out);
- out.print(":");
- int m = adtg.getMinute();
- if (m < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(m, out);
- out.print(":");
- int s = adtg.getSecond();
- if (s < 10) {
- out.print("0");
- }
- WriteValueTools.writeInt(s, out);
- out.print("\")");
- }
-
- }
-
- class GenFieldsIntGenerator extends RandomValueGenerator {
- private final int minFields;
- private final int maxFields;
- private final String prefix;
- private final int startIndex;
- private String[] fieldNames;
- private int[] id;
- private int[] x;
-
- public GenFieldsIntGenerator(int startIndex, int minFields, int maxFields, String prefix) {
- this.startIndex = startIndex;
- this.minFields = minFields;
- this.maxFields = maxFields;
- this.prefix = prefix;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- fieldNames = new String[maxFields];
- for (int i = 0; i < maxFields; i++) {
- fieldNames[i] = prefix + "_" + i;
- }
- id = new int[maxFields];
- for (int i = 0; i < maxFields; i++) {
- id[i] = i;
- }
- x = new int[maxFields];
- }
-
- @Override
- public void generate() throws IOException {
- int numFields = rnd.nextInt(maxFields - minFields + 1) + minFields;
- System.arraycopy(id, 0, x, 0, maxFields);
- int n = numFields;
- boolean first = startIndex < 1;
- while (n > 0) {
- int c = rnd.nextInt(n);
- if (first) {
- first = false;
- } else {
- out.print(",");
- }
- out.print(" \"");
- out.print(fieldNames[x[c]]);
- out.print("\": ");
- WriteValueTools.writeInt(Math.abs(rnd.nextInt()), out);
- x[c] = x[n - 1];
- n--;
- }
- }
- }
-
- class RecordGenerator extends RandomValueGenerator {
-
- private AbstractValueGenerator[] declaredFieldsGenerators;
- private boolean[] nullable;
- private AbstractValueGenerator undeclaredFieldsGenerator;
- private final ARecordType recType;
- private final RecordDataGenAnnotation annot;
- private final String printAfter;
-
- public RecordGenerator(ARecordType type, RecordDataGenAnnotation annot, String printAfter) {
- this.recType = type;
- this.annot = annot;
- this.printAfter = printAfter;
- }
-
- @Override
- public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
- super.init(out, ctx);
- if (declaredFieldsGenerators == null) {
- int m = annot.getDeclaredFieldsDatagen().length;
- declaredFieldsGenerators = new AbstractValueGenerator[m];
- nullable = new boolean[m];
- for (int i = 0; i < m; i++) {
- IAType ti = recType.getFieldTypes()[i];
- if (NonTaggedFormatUtil.isOptional(ti)) {
- ti = ((AUnionType) ti).getActualType();
- nullable[i] = true;
- }
- IRecordFieldDataGen rfdg = annot.getDeclaredFieldsDatagen()[i];
- if (rfdg == null) {
- if (ti.getTypeTag() == ATypeTag.RECORD) {
- ARecordType rt = (ARecordType) ti;
- RecordDataGenAnnotation dga = null;
- for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
- if (annot.getKind() == Kind.RECORD_DATA_GEN) {
- dga = (RecordDataGenAnnotation) annot;
- break;
- }
- }
- if (dga != null) {
- declaredFieldsGenerators[i] = new RecordGenerator(rt, dga, null);
- continue;
- }
- }
- if (declaredFieldsGenerators[i] == null) {
- throw new Exception("No data generator annotation for field " + recType.getFieldNames()[i]
- + " in type " + recType);
- }
- }
- switch (rfdg.getKind()) {
- case VALFILE: {
- FieldValFileDataGen vf = (FieldValFileDataGen) rfdg;
- int numFiles = vf.getFiles().length;
- String[][] a = new String[numFiles][];
- for (int k = 0; k < numFiles; k++) {
- File f = vf.getFiles()[k];
- a[k] = ctx.getFileToLoadedDataMap().get(f);
- if (a[k] == null) {
- a[k] = readFileAsStringArray(f);
- ctx.getFileToLoadedDataMap().put(f, a[k]);
- }
- }
- String constructor;
- if (ti.getTypeTag() == ATypeTag.STRING) {
- constructor = null;
- } else {
- constructor = getConstructor(ti);
- }
- declaredFieldsGenerators[i] = new StringFromArrayGenerator(a, constructor);
- break;
- }
- case LISTVALFILE: {
- ListValFileDataGen lvf = (ListValFileDataGen) rfdg;
- String[] a = ctx.getFileToLoadedDataMap().get(lvf.getFile());
- if (a == null) {
- a = readFileAsStringArray(lvf.getFile());
- ctx.getFileToLoadedDataMap().put(lvf.getFile(), a);
- }
- if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
- throw new Exception(
- "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
- }
- AbstractCollectionType act = (AbstractCollectionType) ti;
- declaredFieldsGenerators[i] = new ListFromArrayGenerator(act, a, lvf.getMin(),
- lvf.getMax());
- break;
- }
- case VALFILESAMEINDEX: {
- FieldValFileSameIndexDataGen vfsi = (FieldValFileSameIndexDataGen) rfdg;
- String[] a = ctx.getFileToLoadedDataMap().get(vfsi.getFile());
- if (a == null) {
- a = readFileAsStringArray(vfsi.getFile());
- ctx.getFileToLoadedDataMap().put(vfsi.getFile(), a);
- }
- StringFromArrayGenerator sfag = null;
- for (int j = 0; j < i; j++) {
- if (recType.getFieldNames()[j].equals(vfsi.getSameAsField())) {
- if (declaredFieldsGenerators[j] instanceof StringFromArrayGenerator) {
- sfag = (StringFromArrayGenerator) declaredFieldsGenerators[j];
- break;
- } else {
- throw new Exception("Data generator for field " + recType.getFieldNames()[j]
- + " is not based on values from a text file, as required by generator for field "
- + recType.getFieldNames()[i]);
- }
- }
- }
- if (sfag == null) {
- throw new Exception("Couldn't find field " + vfsi.getSameAsField() + " before field "
- + recType.getFieldNames()[i]);
- }
- String constructor;
- if (ti.getTypeTag() == ATypeTag.STRING) {
- constructor = null;
- } else {
- constructor = getConstructor(ti);
- }
- declaredFieldsGenerators[i] = new StringFromArraySameIndexGenerator(a, sfag, constructor);
- break;
- }
- case INTERVAL: {
- FieldIntervalDataGen fi = (FieldIntervalDataGen) rfdg;
- String prefix = null;
- String suffix = null;
- if (ti.getTypeTag() == ATypeTag.STRING) {
- prefix = "\"";
- suffix = "\"";
- }
- switch (fi.getValueType()) {
- case INT: {
- declaredFieldsGenerators[i] = new IntIntervalGenerator(
- Integer.parseInt(fi.getMin()), Integer.parseInt(fi.getMax()), prefix,
- suffix);
- break;
- }
- case LONG: {
- declaredFieldsGenerators[i] = new LongIntervalGenerator(Long.parseLong(fi.getMin()),
- Long.parseLong(fi.getMax()), prefix, suffix);
- break;
- }
- case DOUBLE: {
- declaredFieldsGenerators[i] = new DoubleIntervalGenerator(
- Double.parseDouble(fi.getMin()), Double.parseDouble(fi.getMax()), prefix,
- suffix);
- break;
- }
- default: {
- throw new NotImplementedException();
- }
- }
- break;
- }
- case INSERTRANDINT: {
- InsertRandIntDataGen iri = (InsertRandIntDataGen) rfdg;
- declaredFieldsGenerators[i] = new InsertRandIntGenerator(iri.getStr1(), iri.getStr2());
- break;
- }
- case LIST: {
- ListDataGen l = (ListDataGen) rfdg;
- if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
- throw new Exception(
- "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
- }
- AbstractCollectionType act = (AbstractCollectionType) ti;
- declaredFieldsGenerators[i] = new ListDataGenerator(act, l.getMin(), l.getMax());
- break;
- }
- case DATEBETWEENYEARS: {
- DateBetweenYearsDataGen dby = (DateBetweenYearsDataGen) rfdg;
- declaredFieldsGenerators[i] = new DateBetweenYearsGenerator(dby.getMinYear(),
- dby.getMaxYear());
- break;
- }
- case DATETIMEBETWEENYEARS: {
- DatetimeBetweenYearsDataGen dtby = (DatetimeBetweenYearsDataGen) rfdg;
- declaredFieldsGenerators[i] = new DatetimeBetweenYearsGenerator(dtby.getMinYear(),
- dtby.getMaxYear());
- break;
- }
- case DATETIMEADDRANDHOURS: {
- DatetimeAddRandHoursDataGen dtarh = (DatetimeAddRandHoursDataGen) rfdg;
- AccessibleDatetimeGenerator adtg = null;
- for (int j = 0; j < i; j++) {
- if (recType.getFieldNames()[j].equals(dtarh.getAddToField())) {
- if (declaredFieldsGenerators[j] instanceof AccessibleDatetimeGenerator) {
- adtg = (AccessibleDatetimeGenerator) declaredFieldsGenerators[j];
- break;
- } else {
- throw new Exception("Data generator for field " + recType.getFieldNames()[j]
- + " is not based on accessible datetime values, as required by generator for field "
- + recType.getFieldNames()[i]);
- }
- }
- }
- if (adtg == null) {
- throw new Exception("Couldn't find field " + dtarh.getAddToField() + " before field "
- + recType.getFieldNames()[i]);
- }
- declaredFieldsGenerators[i] = new DatetimeAddRandHoursGenerator(dtarh.getMinHour(),
- dtarh.getMaxHour(), adtg);
- break;
- }
- case AUTO: {
- AutoDataGen auto = (AutoDataGen) rfdg;
- switch (ti.getTypeTag()) {
- case INT32: {
- declaredFieldsGenerators[i] = new IntAutoGenerator(
- Integer.parseInt(auto.getInitValueStr()));
- break;
- }
- case INT64: {
- declaredFieldsGenerators[i] = new LongAutoGenerator(
- Long.parseLong(auto.getInitValueStr()));
- break;
- }
- default: {
- throw new IllegalStateException(rfdg.getKind()
- + " annotation is not implemented for type " + ti.getTypeTag());
- }
- }
- break;
- }
- default: {
- throw new NotImplementedException(rfdg.getKind() + " is not implemented");
- }
- }
- }
- }
- for (int i = 0; i < declaredFieldsGenerators.length; i++) {
- declaredFieldsGenerators[i].init(out, ctx);
- }
- if (undeclaredFieldsGenerator == null) {
- UndeclaredFieldsDataGen ufdg = annot.getUndeclaredFieldsDataGen();
- if (ufdg != null) {
- if (!recType.isOpen()) {
- throw new Exception("Cannot generate undeclared fields for closed type " + recType);
- }
- undeclaredFieldsGenerator = new GenFieldsIntGenerator(declaredFieldsGenerators.length,
- ufdg.getMinUndeclaredFields(), ufdg.getMaxUndeclaredFields(),
- ufdg.getUndeclaredFieldsPrefix());
- }
- }
- if (undeclaredFieldsGenerator != null) {
- undeclaredFieldsGenerator.init(out, ctx);
- }
- }
-
- @Override
- public void generate() throws IOException {
- out.print("{");
- boolean first = true;
- for (int i = 0; i < declaredFieldsGenerators.length; i++) {
- boolean pick;
- if (nullable[i]) {
- pick = rnd.nextBoolean();
- } else {
- pick = true;
- }
- if (pick) {
- if (first) {
- first = false;
- } else {
- out.print(",");
- }
- out.print(" \"");
- out.print(this.recType.getFieldNames()[i]);
- out.print("\": ");
- declaredFieldsGenerators[i].generate();
- }
- }
- if (undeclaredFieldsGenerator != null) {
- undeclaredFieldsGenerator.generate();
- }
- out.print(" }");
- if (printAfter != null) {
- out.print(printAfter);
- }
- }
-
- }
-
- private final File schemaFile;
- private final File outputDir;
- private Map<TypeSignature, IAType> typeMap;
- private Map<TypeSignature, TypeDataGen> typeAnnotMap;
- private DataGeneratorContext dgCtx;
- private final IParserFactory parserFactory = new AQLParserFactory();
-
- public AdmDataGen(File schemaFile, File outputDir) {
- this.schemaFile = schemaFile;
- this.outputDir = outputDir;
- }
-
- public void init() throws IOException, ParseException, AsterixException, ACIDException, MetadataException,
- AlgebricksException {
- FileReader aql = new FileReader(schemaFile);
- IParser parser = parserFactory.createParser(aql);
- List<Statement> statements = parser.parse();
- aql.close();
- // TODO: Need to fix how to use transactions here.
- MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(new JobId(-1));
- ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, statements);
- dmlt.translate();
- typeMap = dmlt.getTypeMap();
- typeAnnotMap = dmlt.getTypeDataGenMap();
- dgCtx = new DataGeneratorContext();
- }
-
- public void dataGen() throws Exception {
- for (Map.Entry<TypeSignature, IAType> me : typeMap.entrySet()) {
- TypeSignature tn = me.getKey();
- TypeDataGen tdg = typeAnnotMap.get(tn);
- if (tdg.isDataGen()) {
- IAType t = me.getValue();
-
- if (t.getTypeTag() != ATypeTag.RECORD) {
- throw new NotImplementedException();
- }
- ARecordType rt = (ARecordType) t;
- RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
- if (dga == null) {
- throw new Exception("No data generator annotations for type " + tn);
- }
- File outFile = new File(outputDir + File.separator + tdg.getOutputFileName());
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile)));
- RecordGenerator rg = new RecordGenerator(rt, dga, "\n");
- rg.init(outStream, dgCtx);
- for (long i = 0; i < tdg.getNumValues(); i++) {
- rg.generate();
- }
- outStream.close();
- }
- }
- }
-
- private static RecordDataGenAnnotation firstDataGenAnnotation(ARecordType rt) {
- for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
- if (annot.getKind() == Kind.RECORD_DATA_GEN) {
- return (RecordDataGenAnnotation) annot;
- }
- }
- return null;
- }
-
- private static String[] readFileAsStringArray(File file) throws IOException {
- List<String> tmp = new ArrayList<String>();
- FileInputStream fstream = new FileInputStream(file);
- DataInputStream in = new DataInputStream(fstream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- String strLine;
- while ((strLine = br.readLine()) != null) {
- tmp.add(strLine);
- }
- in.close();
- return tmp.toArray(new String[0]);
- }
-
- private static String getConstructor(IAType t) throws Exception {
- if (t instanceof BuiltinType) {
- String s = ((BuiltinType) t).getConstructor();
- if (s == null) {
- throw new Exception("Type " + t + " has no constructors.");
- }
- return s;
- } else {
- throw new Exception("No string constructor for type " + t);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
deleted file mode 100644
index b605f73..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
+++ /dev/null
@@ -1,476 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.datagen;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-
-public class CustOrdDataGen {
-
- // for customers
- private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
- private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
- private static final int MIN_AGE = 10;
- private static final int MAX_AGE = 90;
-
- private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
- "Lake St.", "Hill St.", "Park St.", "View St." };
- private static final int MIN_STREET_NUM = 1;
- private static final int MAX_STREET_NUM = 10000;
- private static final String[] CITIES = { "Seattle", "San Jose", "Mountain View", "Los Angeles", "Sunnyvale",
- "Portland" };
-
- private static final int MIN_INTERESTS = 0;
- private static final int MAX_INTERESTS = 5;
- private String[] INTERESTS = { "Bass", "Music", "Databases", "Fishing", "Tennis", "Squash", "Computers", "Books",
- "Movies", "Cigars", "Wine", "Running", "Walking", "Skiing", "Basketball", "Video Games", "Cooking",
- "Coffee", "Base Jumping", "Puzzles", "Chess", "Programming", "Reddit", "Soccer", "Hockey", "Money",
- "Dancing", "Brewing", "Gardening", "Hacking", "Reading" };
-
- private static final int MIN_CHILD_AGE = 0;
-
- private static final int MIN_CHILDREN = 0;
- private static final int MAX_CHILDREN = 5;
-
- // for orders
- private static final int MIN_ORDERS_PER_CUST = 1;
- private static final int MAX_ORDERS_PER_CUST = 10;
-
- private String[] CLERKS = { "Kathrin", "Katherine", "Cathryn", "Catherine", "Cat", "Kathryne", "Cathrin" };
- private String[] ORDER_PRIORITIES = { "LOW", "MEDIUM", "HIGH", "PREMIUM" };
- private String[] ORDER_STATUSES = { "ORDER_PLACED", "PAYMENT_RECEIVED", "ORDER_SHIPPED", "ORDER_DELIVERED" };
-
- private String[] firstNames = { "Joe", "John", "Jill", "Gill", "Bill", "William", "Kathy", "Cathey", "Jane",
- "Albert" };
- private String[] lastNames = { "Doe", "Smith", "Li", "Singh", "Williams", "Davis", "Brown", "Wilson", "Moore",
- "Thomas" };
-
- private static final String[] UNDECLARED_FIELD_NAMES = { "param1", "param2", "param3", "param4", "param5",
- "param6", "param7", "param8", "param9", "param10" };
-
- private int currentCID = 0;
- private int currentOID = 0;
-
- private Random rndValue = new Random(50);
-
- private Random rndChildAgeField = new Random(50);
- private Random rndCustAgeField = new Random(50);
- private Random rndCustAddressField = new Random(50);
-
- private Random[] rndUndeclaredOrderFields;
-
- private class Child {
- public String name;
- public int age;
-
- public void generateFieldValues(String lastName, int maxChildAge) {
- // name
- int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
- name = firstNames[firstNameIx] + " " + lastName;
-
- // age
- age = -1;
- if (rndChildAgeField.nextBoolean()) {
- if (maxChildAge >= 0) {
- if (maxChildAge == MIN_CHILD_AGE) {
- age = maxChildAge;
- } else {
- age = Math.abs((rndValue.nextInt()) % (maxChildAge - MIN_CHILD_AGE)) + MIN_AGE;
- }
- }
- }
- }
-
- public String getJSON() {
- StringBuilder jsonString = new StringBuilder();
-
- jsonString.append("{ "); // start child
-
- // name
- jsonString.append(" \"name\": ");
- jsonString.append("\"" + name + "\"");
-
- // age
- if (age >= 0) {
- jsonString.append(", ");
- jsonString.append(" \"age\": ");
- jsonString.append(age);
- }
-
- jsonString.append(" }"); // end child
-
- return jsonString.toString();
- }
-
- }
-
- private class Customer {
-
- private int cid;
- private String name;
- private int age;
-
- private String streetName;
- private int streetNumber;
- private String city;
-
- private int[] custInterests;
- private Child[] custChildren;
-
- public void generateFieldValues() {
- cid = currentCID++;
-
- int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
- int lastNameIx = Math.abs(rndValue.nextInt()) % lastNames.length;
- name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
-
- if (rndCustAgeField.nextBoolean()) {
- age = Math.abs((rndValue.nextInt()) % (MAX_AGE - MIN_AGE)) + MIN_AGE;
- } else {
- age = -1;
- }
-
- if (rndCustAddressField.nextBoolean()) {
- streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
-
- int streetIx = Math.abs(rndValue.nextInt()) % STREETS.length;
- streetName = STREETS[streetIx];
-
- int cityIx = Math.abs(rndValue.nextInt()) % CITIES.length;
- city = CITIES[cityIx];
- } else {
- streetNumber = -1;
- streetName = null;
- city = null;
- }
-
- int numInterests = Math.abs((rndValue.nextInt()) % (MAX_INTERESTS - MIN_INTERESTS)) + MIN_INTERESTS;
- custInterests = new int[numInterests];
- for (int i = 0; i < numInterests; i++) {
- custInterests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
- }
-
- int numChildren = Math.abs((rndValue.nextInt()) % (MAX_CHILDREN - MIN_CHILDREN)) + MIN_CHILDREN;
- custChildren = new Child[numChildren];
- for (int i = 0; i < numChildren; i++) {
- Child c = new Child();
- int maxChildAge = age <= 0 ? 50 : (age - 20);
- c.generateFieldValues(lastNames[lastNameIx], maxChildAge);
- custChildren[i] = c;
- }
- }
-
- public String getJSON() {
- StringBuilder jsonString = new StringBuilder();
-
- jsonString.append("{ "); // start customer
-
- // customer id
- jsonString.append(" \"cid\": ");
- jsonString.append(cid);
- jsonString.append(", ");
-
- // name
- jsonString.append(" \"name\": ");
- jsonString.append("\"" + name + "\"");
-
- // age
- if (age >= 0) {
- jsonString.append(", ");
- jsonString.append(" \"age\": ");
- jsonString.append(age);
- }
-
- // nested address
- if (streetNumber >= 0) {
- jsonString.append(", ");
- jsonString.append(" \"address\": ");
- jsonString.append("{ "); // start address
-
- // number
- jsonString.append(" \"number\": ");
- jsonString.append(streetNumber);
- jsonString.append(", ");
-
- // street
- jsonString.append(" \"street\": ");
- jsonString.append("\"" + streetName + "\"");
- jsonString.append(", ");
-
- // city
- jsonString.append(" \"city\": ");
- jsonString.append("\"" + city + "\"");
-
- jsonString.append(" }"); // end address
- }
-
- jsonString.append(", ");
-
- // interests
- jsonString.append(" \"interests\": ");
- jsonString.append("{{ "); // start interests
- for (int i = 0; i < custInterests.length; i++) {
- jsonString.append("\"" + INTERESTS[custInterests[i]] + "\"");
- if (i != custInterests.length - 1) {
- jsonString.append(", ");
- }
- }
- jsonString.append(" }}"); // end interests
- jsonString.append(", ");
-
- // children
- jsonString.append(" \"children\": ");
- jsonString.append("[ "); // start children
- for (int i = 0; i < custChildren.length; i++) {
- String jsonChild = custChildren[i].getJSON();
- jsonString.append(jsonChild);
- if (i != custChildren.length - 1) {
- jsonString.append(", ");
- }
- }
- jsonString.append(" ]"); // end children
-
- jsonString.append(" }"); // end customer
-
- return jsonString.toString();
- }
- }
-
- private class Order {
- public int oid;
- public int cid;
- public String orderStatus;
- public String orderPriority;
- public String clerk;
- public String total;
-
- private int[] undeclaredFields;
- private BitSet nullMap;
-
- public void generateFieldValues(Customer cust) {
- cid = cust.cid;
- oid = currentOID++;
- float t = Math.abs(rndValue.nextFloat()) * 100;
- total = t + "f";
-
- int orderStatusIx = Math.abs(rndValue.nextInt()) % ORDER_STATUSES.length;
- orderStatus = ORDER_STATUSES[orderStatusIx];
-
- int orderPriorityIx = Math.abs(rndValue.nextInt()) % ORDER_PRIORITIES.length;
- orderPriority = ORDER_PRIORITIES[orderPriorityIx];
-
- int clerkIx = Math.abs(rndValue.nextInt()) % CLERKS.length;
- clerk = CLERKS[clerkIx];
-
- int m = rndUndeclaredOrderFields.length;
- undeclaredFields = new int[m];
- nullMap = new BitSet(m);
- for (int i = 0; i < m; i++) {
- if (rndUndeclaredOrderFields[i].nextBoolean()) {
- undeclaredFields[i] = rndValue.nextInt();
- } else {
- nullMap.set(i);
- }
- }
- }
-
- public String getJSON() {
- StringBuilder jsonString = new StringBuilder();
-
- jsonString.append("{ "); // start order
-
- // oid
- jsonString.append(" \"oid\": ");
- jsonString.append(oid);
- jsonString.append(", ");
-
- // cid
- jsonString.append(" \"cid\": ");
- jsonString.append(cid);
- jsonString.append(", ");
-
- // orderStatus
- jsonString.append(" \"orderstatus\": ");
- jsonString.append("\"" + orderStatus + "\"");
- jsonString.append(", ");
-
- // orderPriority
- jsonString.append(" \"orderpriority\": ");
- jsonString.append("\"" + orderPriority + "\"");
- jsonString.append(", ");
-
- // clerk
- jsonString.append(" \"clerk\": ");
- jsonString.append("\"" + clerk + "\"");
- jsonString.append(", ");
-
- // / cid
- jsonString.append(" \"total\": ");
- jsonString.append(total);
-
- for (int i = 0; i < undeclaredFields.length; i++) {
- if (!nullMap.get(i)) {
- jsonString.append(", ");
- jsonString.append(" \"" + UNDECLARED_FIELD_NAMES[i] + "\": ");
- jsonString.append(undeclaredFields[i]);
- }
- }
-
- jsonString.append(" }"); // end order
- return jsonString.toString();
- }
- }
-
- public void init() {
- try {
- List<String> tmpFirstNames = new ArrayList<String>();
-
- FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
- DataInputStream in = new DataInputStream(fstream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- String strLine;
- while ((strLine = br.readLine()) != null) {
- String firstLetter = strLine.substring(0, 1);
- String remainder = strLine.substring(1);
- String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
- tmpFirstNames.add(capitalized);
- }
- in.close();
- firstNames = tmpFirstNames.toArray(firstNames);
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- try {
- List<String> tmpLastNames = new ArrayList<String>();
-
- FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
- DataInputStream in = new DataInputStream(fstream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- String strLine;
- while ((strLine = br.readLine()) != null) {
- String firstLetter = strLine.substring(0, 1);
- String remainder = strLine.substring(1);
- String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
- tmpLastNames.add(capitalized);
- }
- in.close();
- lastNames = tmpLastNames.toArray(firstNames);
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- rndUndeclaredOrderFields = new Random[UNDECLARED_FIELD_NAMES.length];
- for (int i = 0; i < rndUndeclaredOrderFields.length; i++) {
- rndUndeclaredOrderFields[i] = new Random(50);
- }
- }
-
- public void writeOrdersList(List<Order> ordersList, FileWriter ordersFile) throws IOException {
- while (!ordersList.isEmpty()) {
- int ix = Math.abs(rndValue.nextInt()) % ordersList.size();
- ordersFile.write(ordersList.get(ix).getJSON() + "\n");
- ordersList.remove(ix);
- }
- }
-
- public void writeCustomerList(List<Customer> customerList, Order[] ordersBatch, List<Order> ordersList,
- FileWriter customersFile, FileWriter ordersFile) throws IOException {
- while (!customerList.isEmpty()) {
- int ix = Math.abs(rndValue.nextInt()) % customerList.size();
- customersFile.write(customerList.get(ix).getJSON() + "\n");
-
- // generate orders
- int numOrders = Math.abs(rndValue.nextInt()) % (MAX_ORDERS_PER_CUST - MIN_ORDERS_PER_CUST)
- + MIN_ORDERS_PER_CUST;
- for (int i = 0; i < numOrders; i++) {
- ordersBatch[i].generateFieldValues(customerList.get(ix));
- ordersList.add(ordersBatch[i]);
- }
- writeOrdersList(ordersList, ordersFile);
-
- customerList.remove(ix);
- }
- }
-
- public static void main(String[] args) throws IOException {
- if (args.length != 2) {
- System.err.println("MUST PROVIDE 2 PARAMS, 1. output dir name and 2. number of records to generate.");
- System.exit(1);
- }
-
- String outputFile = args[0];
- int numRecords = Integer.parseInt(args[1]);
-
- FileWriter customersFile = new FileWriter(outputFile + File.separator + "customer.adm");
- FileWriter ordersFile = new FileWriter(outputFile + File.separator + "orders.adm");
-
- CustOrdDataGen dataGen = new CustOrdDataGen();
- dataGen.init();
-
- int batchSize = 1000;
- Customer[] customerBatch = new Customer[batchSize];
- for (int i = 0; i < batchSize; i++) {
- customerBatch[i] = dataGen.new Customer();
- }
-
- Order[] ordersBatch = new Order[MAX_ORDERS_PER_CUST];
- for (int i = 0; i < MAX_ORDERS_PER_CUST; i++) {
- ordersBatch[i] = dataGen.new Order();
- }
-
- List<Customer> customerList = new LinkedList<Customer>();
- List<Order> ordersList = new LinkedList<Order>();
- int custIx = 0;
- for (int i = 0; i < numRecords; i++) {
-
- customerBatch[custIx].generateFieldValues();
- customerList.add(customerBatch[custIx]);
- custIx++;
-
- if (customerList.size() >= batchSize) {
- dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
- custIx = 0;
- }
- }
- dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
- customersFile.flush();
- customersFile.close();
-
- ordersFile.flush();
- ordersFile.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/EventDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/EventDataGen.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/EventDataGen.java
deleted file mode 100644
index b175b42..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/datagen/EventDataGen.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.datagen;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-public class EventDataGen {
- private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
- private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
-
- private String[] firstNames = new String[10];
- private String[] lastNames = new String[10];
-
- private static final int MIN_USER_INTERESTS = 0;
- private static final int MAX_USER_INTERESTS = 7;
- private String[] INTERESTS = { "bass", "music", "databases", "fishing", "tennis", "squash", "computers", "books",
- "movies", "cigars", "wine", "running", "walking", "skiing", "basketball", "video games", "cooking",
- "coffee", "base jumping", "puzzles" };
-
- private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
- "Lake St.", "Hill St.", "Park St.", "View St." };
- private static final int MIN_STREET_NUM = 1;
- private static final int MAX_STREET_NUM = 10000;
- private static final String[] CITIES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
- "Huntington Beach", "Portland" };
- private static final int MIN_ZIP = 100000;
- private static final int MAX_ZIP = 999999;
- private static final String[] LAT_LONGS = { "47,-122", "33,-117", "33,-117", "34,-118", "33,-117", "33,-117",
- "45,-122" };
-
- private static final int MIN_MEMBERSHIPS = 1;
- private static final int MAX_MEMBERSHIPS = 10;
- private static final int MIN_SIG_ID = 1;
- private static final int MAX_SIG_ID = 100;
- private static final String[] CHAPTER_NAMES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
- "Huntington Beach", "Portland", "Newport Beach", "Kirkland" };
- private static final int MEMBER_SINCE_MIN_YEAR = 1970;
- private static final int MEMBER_SINCE_MAX_YEAR = 1998;
-
- private Random rndValue = new Random(50);
- private User user;
-
- private final class User {
- private int firstNameIdx;
- private int lastNameIdx;
- private int[] interests = new int[MAX_USER_INTERESTS - MIN_USER_INTERESTS];
- int numInterests;
- private int streetNumber;
- private String street;
- private String city;
- private int zip;
- private String latlong;
- int numMemberships;
- private int[] member_sigid = new int[MAX_MEMBERSHIPS];
- private String[] member_chap_name = new String[MAX_MEMBERSHIPS];
- private String[] member_since_date = new String[MAX_MEMBERSHIPS];
-
- public void generateFieldValues() {
- firstNameIdx = Math.abs(rndValue.nextInt()) % firstNames.length;
- lastNameIdx = Math.abs(rndValue.nextInt()) % lastNames.length;
- // name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
- numInterests = Math.abs((rndValue.nextInt()) % (MAX_USER_INTERESTS - MIN_USER_INTERESTS))
- + MIN_USER_INTERESTS;
- for (int i = 0; i < numInterests; i++) {
- interests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
- }
- streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
- street = STREETS[Math.abs(rndValue.nextInt()) % STREETS.length];
- int cityIdx = Math.abs(rndValue.nextInt()) % CITIES.length;
- city = CITIES[cityIdx];
- zip = Math.abs(rndValue.nextInt() % (MAX_ZIP - MIN_ZIP)) + MIN_ZIP;
- latlong = LAT_LONGS[cityIdx];
- numMemberships = Math.abs(rndValue.nextInt()) % (MAX_MEMBERSHIPS - MIN_MEMBERSHIPS) + MIN_MEMBERSHIPS;
- for (int i = 0; i < numMemberships; i++) {
- member_sigid[i] = Math.abs(rndValue.nextInt()) % (MAX_SIG_ID - MIN_SIG_ID) + MIN_SIG_ID;
- int cnIdx = Math.abs(rndValue.nextInt()) % CHAPTER_NAMES.length;
- member_chap_name[i] = CHAPTER_NAMES[cnIdx];
- int msYear = Math.abs(rndValue.nextInt()) % (MEMBER_SINCE_MAX_YEAR - MEMBER_SINCE_MIN_YEAR)
- + MEMBER_SINCE_MIN_YEAR;
- int msMo = Math.abs(rndValue.nextInt()) % 12 + 1;
- int msDay = Math.abs(rndValue.nextInt()) % 28 + 1;
- member_since_date[i] = msYear + "-" + (msMo < 10 ? "0" : "") + msMo + "-" + (msDay < 10 ? "0" : "")
- + msDay;
- }
- }
-
- public void write(Writer writer) throws IOException {
- writer.append("{");
- writer.append(" \"name\": \"");
- writer.append(firstNames[firstNameIdx]);
- writer.append(" ");
- writer.append(lastNames[lastNameIdx]);
- writer.append("\", ");
-
- writer.append(" \"email\": \"");
- writer.append(firstNames[firstNameIdx]);
- writer.append(".");
- writer.append(lastNames[lastNameIdx]);
- writer.append("@example.com\", ");
-
- writer.append(" \"interests\": <");
- for (int i = 0; i < numInterests; i++) {
- if (i > 0) {
- writer.append(", ");
- }
- writer.append("\"");
- writer.append(INTERESTS[interests[i]]);
- writer.append("\"");
- }
- writer.append(">, ");
-
- writer.append(" \"address\": {");
- writer.append(" \"street\": \"");
- writer.append(streetNumber + " " + street);
- writer.append("\",");
- writer.append(" \"city\": \"");
- writer.append(city);
- writer.append("\",");
- writer.append(" \"zip\": \"");
- writer.append(zip + "\",");
- writer.append(" \"latlong\": point(\"");
- writer.append(latlong);
- writer.append("\")");
- writer.append("}, ");
-
- writer.append(" \"member_of\": <");
- for (int i = 0; i < numMemberships; i++) {
- if (i > 0) {
- writer.append(", ");
- }
- writer.append("{");
- writer.append(" \"sig_id\": ");
- writer.append(member_sigid[i] + ",");
- writer.append(" \"chapter_name\": \"");
- writer.append(member_chap_name[i]);
- writer.append("\",");
- writer.append(" \"member_since\": date(\"");
- writer.append(member_since_date[i]);
- writer.append("\") }");
- }
- writer.append(">");
-
- writer.append(" }\n");
- }
- }
-
- public void init() throws IOException {
- {
- List<String> tmpFirstNames = new ArrayList<String>();
-
- FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
- DataInputStream in = new DataInputStream(fstream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- String strLine;
- while ((strLine = br.readLine()) != null) {
- String firstLetter = strLine.substring(0, 1);
- String remainder = strLine.substring(1);
- String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
- tmpFirstNames.add(capitalized);
- }
- in.close();
- firstNames = tmpFirstNames.toArray(firstNames);
- }
- {
- List<String> tmpLastNames = new ArrayList<String>();
-
- FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
- DataInputStream in = new DataInputStream(fstream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- String strLine;
- while ((strLine = br.readLine()) != null) {
- String firstLetter = strLine.substring(0, 1);
- String remainder = strLine.substring(1);
- String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
- tmpLastNames.add(capitalized);
- }
- in.close();
- lastNames = tmpLastNames.toArray(firstNames);
- }
- user = new User();
- }
-
- public void generate() {
- user.generateFieldValues();
- }
-
- public void write(Writer w) throws IOException {
- user.write(w);
- }
-
- public static void main(String[] args) throws IOException {
- if (args.length != 2) {
- System.err
- .println("MUST PROVIDE 2 PARAMETERS, 1. output directory path and 2. number of records to generate.");
- System.exit(1);
- }
- String outputFile = args[0];
- int numRecords = Integer.parseInt(args[1]);
- Writer userFile = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile + File.separator
- + "user.adm")));
- EventDataGen dgen = new EventDataGen();
- dgen.init();
- for (int i = 0; i < numRecords; i++) {
- dgen.generate();
- dgen.write(userFile);
- }
- userFile.close();
- }
-}