You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/10/15 18:01:57 UTC
[3/8] asterixdb git commit: Move Twitter Firehose Datasource to Test
Source Folder
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
deleted file mode 100644
index fcaa2a4..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.tools.external.data;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessage;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator;
-
-public class TweetGeneratorForSpatialIndexEvaluation {
-
- private static Logger LOGGER = Logger.getLogger(TweetGeneratorForSpatialIndexEvaluation.class.getName());
-
- public static final String KEY_DURATION = "duration";
- public static final String KEY_TPS = "tps";
- public static final String KEY_GUID_SEED = "guid-seed";
- public static final String KEY_OPENSTREETMAP_FILEPATH = "open-street-map-filepath";
- public static final String KEY_LOCATION_SAMPLE_INTERVAL = "location-sample-interval";
-
- public static final String OUTPUT_FORMAT = "output-format";
- public static final String OUTPUT_FORMAT_ARECORD = "arecord";
- public static final String OUTPUT_FORMAT_ADM_STRING = "adm-string";
-
- private static final int DEFAULT_DURATION = 60; //seconds
- private static final int DEFAULT_GUID_SEED = 0;
- private static final int DEFAULT_SAMPLE_INTERVAL = 1;
-
- private int duration;
- private TweetMessageIterator tweetIterator = null;
- private int partition;
- private int tweetCount = 0;
- private int frameTweetCount = 0;
- private int numFlushedTweets = 0;
- private OutputStream os;
- private DataGeneratorForSpatialIndexEvaluation dataGenerator = null;
- private ByteBuffer outputBuffer = ByteBuffer.allocate(32 * 1024);
- private GULongIDGenerator uidGenerator;
- private String openStreetMapFilePath;
- private int locationSampleInterval;
-
- public int getTweetCount() {
- return tweetCount;
- }
-
- public TweetGeneratorForSpatialIndexEvaluation(Map<String, String> configuration, int partition, String format,
- OutputStream os) throws Exception {
- this.partition = partition;
- String value = configuration.get(KEY_DURATION);
- this.duration = value != null ? Integer.parseInt(value) : DEFAULT_DURATION;
- openStreetMapFilePath = configuration.get(KEY_OPENSTREETMAP_FILEPATH);
- String lsi = configuration.get(KEY_LOCATION_SAMPLE_INTERVAL);
- this.locationSampleInterval = lsi != null ? Integer.parseInt(lsi) : DEFAULT_SAMPLE_INTERVAL;
- int guidSeed = configuration.get(KEY_GUID_SEED) != null ? Integer.parseInt(configuration.get(KEY_GUID_SEED))
- : DEFAULT_GUID_SEED;
- uidGenerator = new GULongIDGenerator(partition, (byte) (guidSeed));
- dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), openStreetMapFilePath,
- locationSampleInterval);
- tweetIterator = dataGenerator.new TweetMessageIterator(duration, uidGenerator);
- this.os = os;
- }
-
- private void writeTweetString(TweetMessage tweetMessage) throws IOException {
- String tweet = tweetMessage.toString() + "\n";
- tweetCount++;
- byte[] b = tweet.getBytes();
- if (outputBuffer.position() + b.length > outputBuffer.limit()) {
- flush();
- numFlushedTweets += frameTweetCount;
- frameTweetCount = 0;
- outputBuffer.put(b);
- } else {
- outputBuffer.put(b);
- }
- frameTweetCount++;
- }
-
- public int getNumFlushedTweets() {
- return numFlushedTweets;
- }
-
- private void flush() throws IOException {
- outputBuffer.flip();
- os.write(outputBuffer.array(), 0, outputBuffer.limit());
- outputBuffer.position(0);
- outputBuffer.limit(32 * 1024);
- }
-
- public boolean setNextRecordBatch(int numTweetsInBatch) throws Exception {
- boolean moreData = tweetIterator.hasNext();
- if (!moreData) {
- if (outputBuffer.position() > 0) {
- flush();
- numFlushedTweets += frameTweetCount;
- frameTweetCount = 0;
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Reached end of batch. Tweet Count: [" + partition + "]" + tweetCount);
- }
- return false;
- } else {
- int count = 0;
- while (count < numTweetsInBatch) {
- writeTweetString(tweetIterator.next());
- count++;
- }
- return true;
- }
- }
-
- public void resetDurationAndFlushedTweetCount(int duration) {
- tweetIterator.resetDuration(duration);
- numFlushedTweets = 0;
- tweetCount = 0;
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
deleted file mode 100644
index 26656f6..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.tbltoadm;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-public class TblToAdm {
-
- private static final String SEPARATOR_STRING = "\\|\\s*";
- private static final char SEPARATOR_CHAR = '|';
-
- private static void convertFile(String inputFileName, String outputFileName, String scmFileName) throws IOException {
- File scmFile = new File(scmFileName);
- File inFile = new File(inputFileName);
- File outFile = new File(outputFileName);
-
- BufferedReader scmReader = new BufferedReader(new FileReader(scmFile));
- String scmStr = scmReader.readLine();
- String[] fieldInfo = scmStr.split(SEPARATOR_STRING);
- String[] fieldNames = new String[fieldInfo.length];
- boolean[] fieldIsString = new boolean[fieldInfo.length];
- for (int i = 0; i < fieldInfo.length; i++) {
- String[] f = fieldInfo[i].split(":\\s*");
- fieldNames[i] = f[0];
- fieldIsString[i] = f[1].equals("string");
- }
- scmReader.close();
-
- BufferedReader reader = new BufferedReader(new FileReader(inFile));
- PrintWriter writer = new PrintWriter(outFile);
- String row;
- while ((row = reader.readLine()) != null) {
- // String[] fields = row.split(SEPARATOR);
- if (row.length() == 0 || row.charAt(0) == ' ' || row.charAt(0) == '\n') {
- continue;
- }
- writer.write("{ ");
- int pos = 0;
- int strlen = row.length();
- for (int i = 0; i < fieldNames.length; i++) {
- if (i > 0) {
- writer.write(", ");
- }
- writer.print('"');
- writer.write(fieldNames[i]);
- writer.write("\": ");
- if (fieldIsString[i]) {
- writer.print('"');
- }
- for (; pos < strlen; pos++) {
- char c = row.charAt(pos);
- if (c != SEPARATOR_CHAR) {
- writer.print(c);
-
- } else {
- ++pos;
- break;
- }
- }
- if (fieldIsString[i]) {
- writer.print('"');
- }
- }
- writer.write(" }\n");
- }
- writer.close();
- reader.close();
- }
-
- public static void main(String args[]) throws Exception {
- if (args.length != 3) {
- System.err.println("Usage: bash tbl2adm <srcfile.tbl> <destfile.adm> <schema-file>");
- return;
- }
- convertFile(args[0], args[1], args[2]);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
deleted file mode 100644
index a525b1b..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.translator;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.common.annotations.TypeDataGen;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.statement.DataverseDecl;
-import org.apache.asterix.lang.common.statement.TypeDecl;
-import org.apache.asterix.metadata.MetadataException;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeSignature;
-import org.apache.asterix.translator.AbstractLangTranslator;
-import org.apache.asterix.translator.TypeTranslator;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public class ADGenDmlTranslator extends AbstractLangTranslator {
-
- private final MetadataTransactionContext mdTxnCtx;
- private final List<Statement> aqlStatements;
- private Map<TypeSignature, IAType> types;
- private Map<TypeSignature, TypeDataGen> typeDataGenMap;
-
- public ADGenDmlTranslator(MetadataTransactionContext mdTxnCtx, List<Statement> aqlStatements) {
- this.mdTxnCtx = mdTxnCtx;
- this.aqlStatements = aqlStatements;
- }
-
- public void translate() throws AsterixException, MetadataException, AlgebricksException {
- String defaultDataverse = getDefaultDataverse();
- types = new HashMap<TypeSignature, IAType>();
- typeDataGenMap = new HashMap<TypeSignature, TypeDataGen>();
-
- for (Statement stmt : aqlStatements) {
- if (stmt.getKind() == Statement.Kind.TYPE_DECL) {
- TypeDecl td = (TypeDecl) stmt;
- String typeDataverse =
- td.getDataverseName() == null ? defaultDataverse : td.getDataverseName().getValue();
-
- Map<TypeSignature, IAType> typeInStmt = TypeTranslator.computeTypes(mdTxnCtx, td.getTypeDef(),
- td.getIdent().getValue(), typeDataverse, types);
- types.putAll(typeInStmt);
-
- TypeSignature signature = new TypeSignature(typeDataverse, td.getIdent().getValue());
- TypeDataGen tdg = td.getDatagenAnnotation();
- if (tdg != null) {
- typeDataGenMap.put(signature, tdg);
- }
- }
- }
- }
-
- private String getDefaultDataverse() {
- for (Statement stmt : aqlStatements) {
- if (stmt.getKind() == Statement.Kind.DATAVERSE_DECL) {
- return ((DataverseDecl) stmt).getDataverseName().getValue();
- }
- }
- return null;
- }
-
- public Map<TypeSignature, IAType> getTypeMap() {
- return types;
- }
-
- public Map<TypeSignature, TypeDataGen> getTypeDataGenMap() {
- return typeDataGenMap;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
new file mode 100644
index 0000000..8a1adb0
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.File;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineParser;
+
+public class AdgClientDriver {
+
+ public static final int NUMBER_OF_ARGUMENTS = 2;
+
+ public static class AdgClientConfig {
+
+ @Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
+ private File schemaFile;
+
+ @Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
+ private File outputDir;
+ }
+
+ public static void main(String[] args) throws Exception {
+ AdgClientConfig acc = new AdgClientConfig();
+ CmdLineParser cmdParser = new CmdLineParser(acc);
+ try {
+ cmdParser.parseArgument(args);
+ } catch (Exception e) {
+ cmdParser.printUsage(System.err);
+ throw e;
+ }
+ AdmDataGen adg = new AdmDataGen(acc.schemaFile, acc.outputDir);
+ adg.init();
+ adg.dataGen();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
new file mode 100644
index 0000000..2918575
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
@@ -0,0 +1,1020 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.asterix.common.annotations.AutoDataGen;
+import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.FieldIntervalDataGen;
+import org.apache.asterix.common.annotations.FieldValFileDataGen;
+import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import org.apache.asterix.common.annotations.IRecordFieldDataGen;
+import org.apache.asterix.common.annotations.IRecordTypeAnnotation;
+import org.apache.asterix.common.annotations.IRecordTypeAnnotation.Kind;
+import org.apache.asterix.common.annotations.InsertRandIntDataGen;
+import org.apache.asterix.common.annotations.ListDataGen;
+import org.apache.asterix.common.annotations.ListValFileDataGen;
+import org.apache.asterix.common.annotations.RecordDataGenAnnotation;
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.lang.aql.parser.AQLParserFactory;
+import org.apache.asterix.lang.aql.parser.ParseException;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeSignature;
+import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import org.apache.asterix.tools.translator.ADGenDmlTranslator;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
+import org.apache.hyracks.algebricks.data.utils.WriteValueTools;
+
+public class AdmDataGen {
+
+ class DataGeneratorContext {
+ private Map<File, String[]> fileToLoadedDataMap = new HashMap<File, String[]>();
+
+ public Map<File, String[]> getFileToLoadedDataMap() {
+ return fileToLoadedDataMap;
+ }
+ }
+
+ public static void printDate(int year, int month, int day, PrintStream out) throws IOException {
+ WriteValueTools.writeInt(year, out);
+ out.print("-");
+ if (month < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(month, out);
+ out.print("-");
+ if (day < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(day, out);
+ }
+
+ abstract class AbstractValueGenerator {
+ protected PrintStream out;
+ protected DataGeneratorContext ctx;
+
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ this.out = out;
+ this.ctx = ctx;
+ }
+
+ public abstract void generate() throws IOException;
+ }
+
+ abstract class RandomValueGenerator extends AbstractValueGenerator {
+ protected Random rnd;
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ this.rnd = new Random(51);
+ }
+
+ }
+
+ class IntAutoGenerator extends AbstractValueGenerator {
+
+ private final int initValue;
+ private int val;
+
+ public IntAutoGenerator(int initValue) {
+ this.initValue = initValue;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ val = initValue;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ WriteValueTools.writeInt(val, out);
+ val++;
+ }
+
+ }
+
+ class LongAutoGenerator extends AbstractValueGenerator {
+
+ private final long initValue;
+ private long val;
+
+ public LongAutoGenerator(long initValue) {
+ this.initValue = initValue;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ val = initValue;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ WriteValueTools.writeLong(val, out);
+ out.print("i64");
+ val++;
+ }
+
+ }
+
+ class StringFromArrayGenerator extends RandomValueGenerator {
+ private final String[][] array;
+ private int lastIndex;
+ private final String constructor;
+ private String[] chosen;
+
+ public StringFromArrayGenerator(String[][] array, String constructor) {
+ this.array = array;
+ this.constructor = constructor;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ this.lastIndex = -1;
+ this.chosen = new String[array.length];
+ }
+
+ @Override
+ public void generate() throws IOException {
+ for (int i = 0; i < array.length; i++) {
+ lastIndex = Math.abs(rnd.nextInt()) % array[i].length;
+ chosen[i] = array[i][lastIndex];
+ }
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ for (int i = 0; i < chosen.length; i++) {
+ out.print(chosen[i]);
+ }
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+
+ public int getLastIndex() {
+ return lastIndex;
+ }
+ }
+
+ abstract class AbstractCollectionDataGenerator extends RandomValueGenerator {
+ protected final AbstractCollectionType act;
+ protected final int min;
+ protected final int max;
+ protected final String startList;
+ protected final String endList;
+
+ public AbstractCollectionDataGenerator(AbstractCollectionType act, int min, int max) {
+ this.act = act;
+ this.min = min;
+ this.max = max;
+ if (act.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ startList = "[";
+ endList = "]";
+ } else {
+ startList = "{{";
+ endList = "}}";
+ }
+ }
+
+ }
+
+ class ListDataGenerator extends AbstractCollectionDataGenerator {
+
+ private AbstractValueGenerator itemGen;
+
+ public ListDataGenerator(AbstractCollectionType act, int min, int max) {
+ super(act, min, max);
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ IAType t = act.getItemType();
+ if (t.getTypeTag() != ATypeTag.RECORD) {
+ throw new NotImplementedException("list annotation only works with record item types for now.");
+ }
+ ARecordType rt = (ARecordType) t;
+ RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+ if (dga == null) {
+ throw new Exception("No annotation on item type " + t);
+ }
+ itemGen = new RecordGenerator(rt, dga, null);
+ itemGen.init(out, ctx);
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print(startList);
+ int numItems = rnd.nextInt(max - min + 1) + min;
+ for (int i = 0; i < numItems; i++) {
+ if (i > 0) {
+ out.print(", ");
+ }
+ itemGen.generate();
+ }
+ out.print(endList);
+ }
+
+ }
+
+ class ListFromArrayGenerator extends AbstractCollectionDataGenerator {
+
+ private final String[] array;
+ private String constructor;
+
+ public ListFromArrayGenerator(AbstractCollectionType act, String[] array, int min, int max) {
+ super(act, min, max);
+ this.array = array;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ if (act.getItemType().getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(act.getItemType());
+ }
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print(startList);
+ int numItems = rnd.nextInt(max - min + 1) + min;
+ for (int i = 0; i < numItems; i++) {
+ if (i > 0) {
+ out.print(", ");
+ }
+ int c = Math.abs(rnd.nextInt()) % array.length;
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ out.print(array[c]);
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+ out.print(endList);
+ }
+
+ }
+
+ class StringFromArraySameIndexGenerator extends AbstractValueGenerator {
+ private final String[] array;
+ private final StringFromArrayGenerator sfag;
+ private final String constructor;
+
+ public StringFromArraySameIndexGenerator(String[] array, StringFromArrayGenerator sfag, String constructor) {
+ this.array = array;
+ this.sfag = sfag;
+ this.constructor = constructor;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ String val = array[sfag.getLastIndex()];
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ out.print(val);
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+ }
+
+ class IntIntervalGenerator extends RandomValueGenerator {
+
+ private final int min;
+ private final int max;
+ private final String prefix;
+ private final String suffix;
+
+ public IntIntervalGenerator(int min, int max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int v = Math.abs(rnd.nextInt() % (max - min + 1)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ WriteValueTools.writeInt(v, out);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class LongIntervalGenerator extends RandomValueGenerator {
+
+ private final long min;
+ private final long max;
+ private final String prefix;
+ private final String suffix;
+
+ public LongIntervalGenerator(long min, long max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ long v = Math.abs(rnd.nextLong() % (max - min + 1)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ WriteValueTools.writeLong(v, out);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class DoubleIntervalGenerator extends RandomValueGenerator {
+
+ private final double min;
+ private final double max;
+ private final String prefix;
+ private final String suffix;
+
+ public DoubleIntervalGenerator(double min, double max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ double v = Math.abs(rnd.nextDouble() % (max - min)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ out.print(v);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class InsertRandIntGenerator extends RandomValueGenerator {
+
+ private final String str1;
+ private final String str2;
+
+ public InsertRandIntGenerator(String str1, String str2) {
+ this.str1 = "\"" + str1;
+ this.str2 = str2 + "\"";
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int v = Math.abs(rnd.nextInt());
+ out.print(str1);
+ WriteValueTools.writeInt(v, out);
+ out.print(str2);
+ }
+
+ }
+
+ interface AccessibleDateGenerator {
+ public int getYear();
+
+ public int getMonth();
+
+ public int getDay();
+ }
+
+ abstract class DateGenerator extends RandomValueGenerator implements AccessibleDateGenerator {
+
+ private int year;
+ private int month;
+ private int day;
+
+ protected void generateDate(int minYear, int maxYear) throws IOException {
+ year = rnd.nextInt(maxYear - minYear + 1) + minYear;
+ month = Math.abs(rnd.nextInt()) % 12 + 1;
+ day = Math.abs(rnd.nextInt()) % 28 + 1;
+ printDate(year, month, day, out);
+ }
+
+ @Override
+ public int getYear() {
+ return year;
+ }
+
+ @Override
+ public int getMonth() {
+ return month;
+ }
+
+ @Override
+ public int getDay() {
+ return day;
+ }
+ }
+
+ class DateBetweenYearsGenerator extends DateGenerator {
+
+ private final int minYear;
+ private final int maxYear;
+
+ public DateBetweenYearsGenerator(int minYear, int maxYear) {
+ this.minYear = minYear;
+ this.maxYear = maxYear;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("date(\"");
+ generateDate(minYear, maxYear);
+ out.print("\")");
+ }
+
+ }
+
+ interface AccessibleDatetimeGenerator extends AccessibleDateGenerator {
+ public int getHour();
+
+ public int getMinute();
+
+ public int getSecond();
+ }
+
+ class DatetimeBetweenYearsGenerator extends DateGenerator implements AccessibleDatetimeGenerator {
+ private final int minYear;
+ private final int maxYear;
+ private int hour;
+
+ public DatetimeBetweenYearsGenerator(int minYear, int maxYear) {
+ this.minYear = minYear;
+ this.maxYear = maxYear;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("datetime(\"");
+ generateDate(minYear, maxYear);
+ out.print("T");
+ hour = rnd.nextInt(24);
+ if (hour < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(hour, out);
+ out.print(":00:00\")");
+ }
+
+ @Override
+ public int getHour() {
+ return hour;
+ }
+
+ @Override
+ public int getMinute() {
+ return 0;
+ }
+
+ @Override
+ public int getSecond() {
+ return 0;
+ }
+
+ }
+
+ class DatetimeAddRandHoursGenerator extends RandomValueGenerator {
+
+ private final int minHours;
+ private final int maxHours;
+ private final AccessibleDatetimeGenerator adtg;
+
+ public DatetimeAddRandHoursGenerator(int minHours, int maxHours, AccessibleDatetimeGenerator adtg) {
+ this.minHours = minHours;
+ this.maxHours = maxHours;
+ this.adtg = adtg;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int addHours = rnd.nextInt(maxHours - minHours + 1) + minHours;
+
+ out.print("datetime(\"");
+ printDate(adtg.getYear(), adtg.getMonth(), adtg.getDay(), out);
+ out.print("T");
+ int h = adtg.getHour() + addHours;
+ if (h > 23) {
+ h = 23;
+ }
+ if (h < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(h, out);
+ out.print(":");
+ int m = adtg.getMinute();
+ if (m < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(m, out);
+ out.print(":");
+ int s = adtg.getSecond();
+ if (s < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(s, out);
+ out.print("\")");
+ }
+
+ }
+
+ class GenFieldsIntGenerator extends RandomValueGenerator {
+ private final int minFields;
+ private final int maxFields;
+ private final String prefix;
+ private final int startIndex;
+ private String[] fieldNames;
+ private int[] id;
+ private int[] x;
+
+ public GenFieldsIntGenerator(int startIndex, int minFields, int maxFields, String prefix) {
+ this.startIndex = startIndex;
+ this.minFields = minFields;
+ this.maxFields = maxFields;
+ this.prefix = prefix;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ fieldNames = new String[maxFields];
+ for (int i = 0; i < maxFields; i++) {
+ fieldNames[i] = prefix + "_" + i;
+ }
+ id = new int[maxFields];
+ for (int i = 0; i < maxFields; i++) {
+ id[i] = i;
+ }
+ x = new int[maxFields];
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int numFields = rnd.nextInt(maxFields - minFields + 1) + minFields;
+ System.arraycopy(id, 0, x, 0, maxFields);
+ int n = numFields;
+ boolean first = startIndex < 1;
+ while (n > 0) {
+ int c = rnd.nextInt(n);
+ if (first) {
+ first = false;
+ } else {
+ out.print(",");
+ }
+ out.print(" \"");
+ out.print(fieldNames[x[c]]);
+ out.print("\": ");
+ WriteValueTools.writeInt(Math.abs(rnd.nextInt()), out);
+ x[c] = x[n - 1];
+ n--;
+ }
+ }
+ }
+
+ class RecordGenerator extends RandomValueGenerator {
+
+ private AbstractValueGenerator[] declaredFieldsGenerators;
+ private boolean[] nullable;
+ private AbstractValueGenerator undeclaredFieldsGenerator;
+ private final ARecordType recType;
+ private final RecordDataGenAnnotation annot;
+ private final String printAfter;
+
+ public RecordGenerator(ARecordType type, RecordDataGenAnnotation annot, String printAfter) {
+ this.recType = type;
+ this.annot = annot;
+ this.printAfter = printAfter;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ if (declaredFieldsGenerators == null) {
+ int m = annot.getDeclaredFieldsDatagen().length;
+ declaredFieldsGenerators = new AbstractValueGenerator[m];
+ nullable = new boolean[m];
+ for (int i = 0; i < m; i++) {
+ IAType ti = recType.getFieldTypes()[i];
+ if (NonTaggedFormatUtil.isOptional(ti)) {
+ ti = ((AUnionType) ti).getActualType();
+ nullable[i] = true;
+ }
+ IRecordFieldDataGen rfdg = annot.getDeclaredFieldsDatagen()[i];
+ if (rfdg == null) {
+ if (ti.getTypeTag() == ATypeTag.RECORD) {
+ ARecordType rt = (ARecordType) ti;
+ RecordDataGenAnnotation dga = null;
+ for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+ if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+ dga = (RecordDataGenAnnotation) annot;
+ break;
+ }
+ }
+ if (dga != null) {
+ declaredFieldsGenerators[i] = new RecordGenerator(rt, dga, null);
+ continue;
+ }
+ }
+ if (declaredFieldsGenerators[i] == null) {
+ throw new Exception("No data generator annotation for field " + recType.getFieldNames()[i]
+ + " in type " + recType);
+ }
+ }
+ switch (rfdg.getKind()) {
+ case VALFILE: {
+ FieldValFileDataGen vf = (FieldValFileDataGen) rfdg;
+ int numFiles = vf.getFiles().length;
+ String[][] a = new String[numFiles][];
+ for (int k = 0; k < numFiles; k++) {
+ File f = vf.getFiles()[k];
+ a[k] = ctx.getFileToLoadedDataMap().get(f);
+ if (a[k] == null) {
+ a[k] = readFileAsStringArray(f);
+ ctx.getFileToLoadedDataMap().put(f, a[k]);
+ }
+ }
+ String constructor;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(ti);
+ }
+ declaredFieldsGenerators[i] = new StringFromArrayGenerator(a, constructor);
+ break;
+ }
+ case LISTVALFILE: {
+ ListValFileDataGen lvf = (ListValFileDataGen) rfdg;
+ String[] a = ctx.getFileToLoadedDataMap().get(lvf.getFile());
+ if (a == null) {
+ a = readFileAsStringArray(lvf.getFile());
+ ctx.getFileToLoadedDataMap().put(lvf.getFile(), a);
+ }
+ if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+ throw new Exception(
+ "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
+ }
+ AbstractCollectionType act = (AbstractCollectionType) ti;
+ declaredFieldsGenerators[i] = new ListFromArrayGenerator(act, a, lvf.getMin(),
+ lvf.getMax());
+ break;
+ }
+ case VALFILESAMEINDEX: {
+ FieldValFileSameIndexDataGen vfsi = (FieldValFileSameIndexDataGen) rfdg;
+ String[] a = ctx.getFileToLoadedDataMap().get(vfsi.getFile());
+ if (a == null) {
+ a = readFileAsStringArray(vfsi.getFile());
+ ctx.getFileToLoadedDataMap().put(vfsi.getFile(), a);
+ }
+ StringFromArrayGenerator sfag = null;
+ for (int j = 0; j < i; j++) {
+ if (recType.getFieldNames()[j].equals(vfsi.getSameAsField())) {
+ if (declaredFieldsGenerators[j] instanceof StringFromArrayGenerator) {
+ sfag = (StringFromArrayGenerator) declaredFieldsGenerators[j];
+ break;
+ } else {
+ throw new Exception("Data generator for field " + recType.getFieldNames()[j]
+ + " is not based on values from a text file, as required by generator for field "
+ + recType.getFieldNames()[i]);
+ }
+ }
+ }
+ if (sfag == null) {
+ throw new Exception("Couldn't find field " + vfsi.getSameAsField() + " before field "
+ + recType.getFieldNames()[i]);
+ }
+ String constructor;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(ti);
+ }
+ declaredFieldsGenerators[i] = new StringFromArraySameIndexGenerator(a, sfag, constructor);
+ break;
+ }
+ case INTERVAL: {
+ FieldIntervalDataGen fi = (FieldIntervalDataGen) rfdg;
+ String prefix = null;
+ String suffix = null;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ prefix = "\"";
+ suffix = "\"";
+ }
+ switch (fi.getValueType()) {
+ case INT: {
+ declaredFieldsGenerators[i] = new IntIntervalGenerator(
+ Integer.parseInt(fi.getMin()), Integer.parseInt(fi.getMax()), prefix,
+ suffix);
+ break;
+ }
+ case LONG: {
+ declaredFieldsGenerators[i] = new LongIntervalGenerator(Long.parseLong(fi.getMin()),
+ Long.parseLong(fi.getMax()), prefix, suffix);
+ break;
+ }
+ case DOUBLE: {
+ declaredFieldsGenerators[i] = new DoubleIntervalGenerator(
+ Double.parseDouble(fi.getMin()), Double.parseDouble(fi.getMax()), prefix,
+ suffix);
+ break;
+ }
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ break;
+ }
+ case INSERTRANDINT: {
+ InsertRandIntDataGen iri = (InsertRandIntDataGen) rfdg;
+ declaredFieldsGenerators[i] = new InsertRandIntGenerator(iri.getStr1(), iri.getStr2());
+ break;
+ }
+ case LIST: {
+ ListDataGen l = (ListDataGen) rfdg;
+ if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+ throw new Exception(
+ "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
+ }
+ AbstractCollectionType act = (AbstractCollectionType) ti;
+ declaredFieldsGenerators[i] = new ListDataGenerator(act, l.getMin(), l.getMax());
+ break;
+ }
+ case DATEBETWEENYEARS: {
+ DateBetweenYearsDataGen dby = (DateBetweenYearsDataGen) rfdg;
+ declaredFieldsGenerators[i] = new DateBetweenYearsGenerator(dby.getMinYear(),
+ dby.getMaxYear());
+ break;
+ }
+ case DATETIMEBETWEENYEARS: {
+ DatetimeBetweenYearsDataGen dtby = (DatetimeBetweenYearsDataGen) rfdg;
+ declaredFieldsGenerators[i] = new DatetimeBetweenYearsGenerator(dtby.getMinYear(),
+ dtby.getMaxYear());
+ break;
+ }
+ case DATETIMEADDRANDHOURS: {
+ DatetimeAddRandHoursDataGen dtarh = (DatetimeAddRandHoursDataGen) rfdg;
+ AccessibleDatetimeGenerator adtg = null;
+ for (int j = 0; j < i; j++) {
+ if (recType.getFieldNames()[j].equals(dtarh.getAddToField())) {
+ if (declaredFieldsGenerators[j] instanceof AccessibleDatetimeGenerator) {
+ adtg = (AccessibleDatetimeGenerator) declaredFieldsGenerators[j];
+ break;
+ } else {
+ throw new Exception("Data generator for field " + recType.getFieldNames()[j]
+ + " is not based on accessible datetime values, as required by generator for field "
+ + recType.getFieldNames()[i]);
+ }
+ }
+ }
+ if (adtg == null) {
+ throw new Exception("Couldn't find field " + dtarh.getAddToField() + " before field "
+ + recType.getFieldNames()[i]);
+ }
+ declaredFieldsGenerators[i] = new DatetimeAddRandHoursGenerator(dtarh.getMinHour(),
+ dtarh.getMaxHour(), adtg);
+ break;
+ }
+ case AUTO: {
+ AutoDataGen auto = (AutoDataGen) rfdg;
+ switch (ti.getTypeTag()) {
+ case INT32: {
+ declaredFieldsGenerators[i] = new IntAutoGenerator(
+ Integer.parseInt(auto.getInitValueStr()));
+ break;
+ }
+ case INT64: {
+ declaredFieldsGenerators[i] = new LongAutoGenerator(
+ Long.parseLong(auto.getInitValueStr()));
+ break;
+ }
+ default: {
+ throw new IllegalStateException(rfdg.getKind()
+ + " annotation is not implemented for type " + ti.getTypeTag());
+ }
+ }
+ break;
+ }
+ default: {
+ throw new NotImplementedException(rfdg.getKind() + " is not implemented");
+ }
+ }
+ }
+ }
+ for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+ declaredFieldsGenerators[i].init(out, ctx);
+ }
+ if (undeclaredFieldsGenerator == null) {
+ UndeclaredFieldsDataGen ufdg = annot.getUndeclaredFieldsDataGen();
+ if (ufdg != null) {
+ if (!recType.isOpen()) {
+ throw new Exception("Cannot generate undeclared fields for closed type " + recType);
+ }
+ undeclaredFieldsGenerator = new GenFieldsIntGenerator(declaredFieldsGenerators.length,
+ ufdg.getMinUndeclaredFields(), ufdg.getMaxUndeclaredFields(),
+ ufdg.getUndeclaredFieldsPrefix());
+ }
+ }
+ if (undeclaredFieldsGenerator != null) {
+ undeclaredFieldsGenerator.init(out, ctx);
+ }
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("{");
+ boolean first = true;
+ for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+ boolean pick;
+ if (nullable[i]) {
+ pick = rnd.nextBoolean();
+ } else {
+ pick = true;
+ }
+ if (pick) {
+ if (first) {
+ first = false;
+ } else {
+ out.print(",");
+ }
+ out.print(" \"");
+ out.print(this.recType.getFieldNames()[i]);
+ out.print("\": ");
+ declaredFieldsGenerators[i].generate();
+ }
+ }
+ if (undeclaredFieldsGenerator != null) {
+ undeclaredFieldsGenerator.generate();
+ }
+ out.print(" }");
+ if (printAfter != null) {
+ out.print(printAfter);
+ }
+ }
+
+ }
+
+ private final File schemaFile;
+ private final File outputDir;
+ private Map<TypeSignature, IAType> typeMap;
+ private Map<TypeSignature, TypeDataGen> typeAnnotMap;
+ private DataGeneratorContext dgCtx;
+ private final IParserFactory parserFactory = new AQLParserFactory();
+
+ public AdmDataGen(File schemaFile, File outputDir) {
+ this.schemaFile = schemaFile;
+ this.outputDir = outputDir;
+ }
+
+ public void init() throws IOException, ParseException, AsterixException, ACIDException, MetadataException,
+ AlgebricksException {
+ FileReader aql = new FileReader(schemaFile);
+ IParser parser = parserFactory.createParser(aql);
+ List<Statement> statements = parser.parse();
+ aql.close();
+ // TODO: Need to fix how to use transactions here.
+ MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(new JobId(-1));
+ ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, statements);
+ dmlt.translate();
+ typeMap = dmlt.getTypeMap();
+ typeAnnotMap = dmlt.getTypeDataGenMap();
+ dgCtx = new DataGeneratorContext();
+ }
+
+ public void dataGen() throws Exception {
+ for (Map.Entry<TypeSignature, IAType> me : typeMap.entrySet()) {
+ TypeSignature tn = me.getKey();
+ TypeDataGen tdg = typeAnnotMap.get(tn);
+ if (tdg.isDataGen()) {
+ IAType t = me.getValue();
+
+ if (t.getTypeTag() != ATypeTag.RECORD) {
+ throw new NotImplementedException();
+ }
+ ARecordType rt = (ARecordType) t;
+ RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+ if (dga == null) {
+ throw new Exception("No data generator annotations for type " + tn);
+ }
+ File outFile = new File(outputDir + File.separator + tdg.getOutputFileName());
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile)));
+ RecordGenerator rg = new RecordGenerator(rt, dga, "\n");
+ rg.init(outStream, dgCtx);
+ for (long i = 0; i < tdg.getNumValues(); i++) {
+ rg.generate();
+ }
+ outStream.close();
+ }
+ }
+ }
+
+ private static RecordDataGenAnnotation firstDataGenAnnotation(ARecordType rt) {
+ for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+ if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+ return (RecordDataGenAnnotation) annot;
+ }
+ }
+ return null;
+ }
+
+ private static String[] readFileAsStringArray(File file) throws IOException {
+ List<String> tmp = new ArrayList<String>();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ tmp.add(strLine);
+ }
+ in.close();
+ return tmp.toArray(new String[0]);
+ }
+
+ private static String getConstructor(IAType t) throws Exception {
+ if (t instanceof BuiltinType) {
+ String s = ((BuiltinType) t).getConstructor();
+ if (s == null) {
+ throw new Exception("Type " + t + " has no constructors.");
+ }
+ return s;
+ } else {
+ throw new Exception("No string constructor for type " + t);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
new file mode 100644
index 0000000..b605f73
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class CustOrdDataGen {
+
+ // for customers
+ private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+ private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+ private static final int MIN_AGE = 10;
+ private static final int MAX_AGE = 90;
+
+ private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+ "Lake St.", "Hill St.", "Park St.", "View St." };
+ private static final int MIN_STREET_NUM = 1;
+ private static final int MAX_STREET_NUM = 10000;
+ private static final String[] CITIES = { "Seattle", "San Jose", "Mountain View", "Los Angeles", "Sunnyvale",
+ "Portland" };
+
+ private static final int MIN_INTERESTS = 0;
+ private static final int MAX_INTERESTS = 5;
+ private String[] INTERESTS = { "Bass", "Music", "Databases", "Fishing", "Tennis", "Squash", "Computers", "Books",
+ "Movies", "Cigars", "Wine", "Running", "Walking", "Skiing", "Basketball", "Video Games", "Cooking",
+ "Coffee", "Base Jumping", "Puzzles", "Chess", "Programming", "Reddit", "Soccer", "Hockey", "Money",
+ "Dancing", "Brewing", "Gardening", "Hacking", "Reading" };
+
+ private static final int MIN_CHILD_AGE = 0;
+
+ private static final int MIN_CHILDREN = 0;
+ private static final int MAX_CHILDREN = 5;
+
+ // for orders
+ private static final int MIN_ORDERS_PER_CUST = 1;
+ private static final int MAX_ORDERS_PER_CUST = 10;
+
+ private String[] CLERKS = { "Kathrin", "Katherine", "Cathryn", "Catherine", "Cat", "Kathryne", "Cathrin" };
+ private String[] ORDER_PRIORITIES = { "LOW", "MEDIUM", "HIGH", "PREMIUM" };
+ private String[] ORDER_STATUSES = { "ORDER_PLACED", "PAYMENT_RECEIVED", "ORDER_SHIPPED", "ORDER_DELIVERED" };
+
+ private String[] firstNames = { "Joe", "John", "Jill", "Gill", "Bill", "William", "Kathy", "Cathey", "Jane",
+ "Albert" };
+ private String[] lastNames = { "Doe", "Smith", "Li", "Singh", "Williams", "Davis", "Brown", "Wilson", "Moore",
+ "Thomas" };
+
+ private static final String[] UNDECLARED_FIELD_NAMES = { "param1", "param2", "param3", "param4", "param5",
+ "param6", "param7", "param8", "param9", "param10" };
+
+ private int currentCID = 0;
+ private int currentOID = 0;
+
+ private Random rndValue = new Random(50);
+
+ private Random rndChildAgeField = new Random(50);
+ private Random rndCustAgeField = new Random(50);
+ private Random rndCustAddressField = new Random(50);
+
+ private Random[] rndUndeclaredOrderFields;
+
+ private class Child {
+ public String name;
+ public int age;
+
+ public void generateFieldValues(String lastName, int maxChildAge) {
+ // name
+ int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ name = firstNames[firstNameIx] + " " + lastName;
+
+ // age
+ age = -1;
+ if (rndChildAgeField.nextBoolean()) {
+ if (maxChildAge >= 0) {
+ if (maxChildAge == MIN_CHILD_AGE) {
+ age = maxChildAge;
+ } else {
+ age = Math.abs((rndValue.nextInt()) % (maxChildAge - MIN_CHILD_AGE)) + MIN_AGE;
+ }
+ }
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start child
+
+ // name
+ jsonString.append(" \"name\": ");
+ jsonString.append("\"" + name + "\"");
+
+ // age
+ if (age >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"age\": ");
+ jsonString.append(age);
+ }
+
+ jsonString.append(" }"); // end child
+
+ return jsonString.toString();
+ }
+
+ }
+
+ private class Customer {
+
+ private int cid;
+ private String name;
+ private int age;
+
+ private String streetName;
+ private int streetNumber;
+ private String city;
+
+ private int[] custInterests;
+ private Child[] custChildren;
+
+ public void generateFieldValues() {
+ cid = currentCID++;
+
+ int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ int lastNameIx = Math.abs(rndValue.nextInt()) % lastNames.length;
+ name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+
+ if (rndCustAgeField.nextBoolean()) {
+ age = Math.abs((rndValue.nextInt()) % (MAX_AGE - MIN_AGE)) + MIN_AGE;
+ } else {
+ age = -1;
+ }
+
+ if (rndCustAddressField.nextBoolean()) {
+ streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+
+ int streetIx = Math.abs(rndValue.nextInt()) % STREETS.length;
+ streetName = STREETS[streetIx];
+
+ int cityIx = Math.abs(rndValue.nextInt()) % CITIES.length;
+ city = CITIES[cityIx];
+ } else {
+ streetNumber = -1;
+ streetName = null;
+ city = null;
+ }
+
+ int numInterests = Math.abs((rndValue.nextInt()) % (MAX_INTERESTS - MIN_INTERESTS)) + MIN_INTERESTS;
+ custInterests = new int[numInterests];
+ for (int i = 0; i < numInterests; i++) {
+ custInterests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+ }
+
+ int numChildren = Math.abs((rndValue.nextInt()) % (MAX_CHILDREN - MIN_CHILDREN)) + MIN_CHILDREN;
+ custChildren = new Child[numChildren];
+ for (int i = 0; i < numChildren; i++) {
+ Child c = new Child();
+ int maxChildAge = age <= 0 ? 50 : (age - 20);
+ c.generateFieldValues(lastNames[lastNameIx], maxChildAge);
+ custChildren[i] = c;
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start customer
+
+ // customer id
+ jsonString.append(" \"cid\": ");
+ jsonString.append(cid);
+ jsonString.append(", ");
+
+ // name
+ jsonString.append(" \"name\": ");
+ jsonString.append("\"" + name + "\"");
+
+ // age
+ if (age >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"age\": ");
+ jsonString.append(age);
+ }
+
+ // nested address
+ if (streetNumber >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"address\": ");
+ jsonString.append("{ "); // start address
+
+ // number
+ jsonString.append(" \"number\": ");
+ jsonString.append(streetNumber);
+ jsonString.append(", ");
+
+ // street
+ jsonString.append(" \"street\": ");
+ jsonString.append("\"" + streetName + "\"");
+ jsonString.append(", ");
+
+ // city
+ jsonString.append(" \"city\": ");
+ jsonString.append("\"" + city + "\"");
+
+ jsonString.append(" }"); // end address
+ }
+
+ jsonString.append(", ");
+
+ // interests
+ jsonString.append(" \"interests\": ");
+ jsonString.append("{{ "); // start interests
+ for (int i = 0; i < custInterests.length; i++) {
+ jsonString.append("\"" + INTERESTS[custInterests[i]] + "\"");
+ if (i != custInterests.length - 1) {
+ jsonString.append(", ");
+ }
+ }
+ jsonString.append(" }}"); // end interests
+ jsonString.append(", ");
+
+ // children
+ jsonString.append(" \"children\": ");
+ jsonString.append("[ "); // start children
+ for (int i = 0; i < custChildren.length; i++) {
+ String jsonChild = custChildren[i].getJSON();
+ jsonString.append(jsonChild);
+ if (i != custChildren.length - 1) {
+ jsonString.append(", ");
+ }
+ }
+ jsonString.append(" ]"); // end children
+
+ jsonString.append(" }"); // end customer
+
+ return jsonString.toString();
+ }
+ }
+
+ private class Order {
+ public int oid;
+ public int cid;
+ public String orderStatus;
+ public String orderPriority;
+ public String clerk;
+ public String total;
+
+ private int[] undeclaredFields;
+ private BitSet nullMap;
+
+ public void generateFieldValues(Customer cust) {
+ cid = cust.cid;
+ oid = currentOID++;
+ float t = Math.abs(rndValue.nextFloat()) * 100;
+ total = t + "f";
+
+ int orderStatusIx = Math.abs(rndValue.nextInt()) % ORDER_STATUSES.length;
+ orderStatus = ORDER_STATUSES[orderStatusIx];
+
+ int orderPriorityIx = Math.abs(rndValue.nextInt()) % ORDER_PRIORITIES.length;
+ orderPriority = ORDER_PRIORITIES[orderPriorityIx];
+
+ int clerkIx = Math.abs(rndValue.nextInt()) % CLERKS.length;
+ clerk = CLERKS[clerkIx];
+
+ int m = rndUndeclaredOrderFields.length;
+ undeclaredFields = new int[m];
+ nullMap = new BitSet(m);
+ for (int i = 0; i < m; i++) {
+ if (rndUndeclaredOrderFields[i].nextBoolean()) {
+ undeclaredFields[i] = rndValue.nextInt();
+ } else {
+ nullMap.set(i);
+ }
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start order
+
+ // oid
+ jsonString.append(" \"oid\": ");
+ jsonString.append(oid);
+ jsonString.append(", ");
+
+ // cid
+ jsonString.append(" \"cid\": ");
+ jsonString.append(cid);
+ jsonString.append(", ");
+
+ // orderStatus
+ jsonString.append(" \"orderstatus\": ");
+ jsonString.append("\"" + orderStatus + "\"");
+ jsonString.append(", ");
+
+ // orderPriority
+ jsonString.append(" \"orderpriority\": ");
+ jsonString.append("\"" + orderPriority + "\"");
+ jsonString.append(", ");
+
+ // clerk
+ jsonString.append(" \"clerk\": ");
+ jsonString.append("\"" + clerk + "\"");
+ jsonString.append(", ");
+
+ // / cid
+ jsonString.append(" \"total\": ");
+ jsonString.append(total);
+
+ for (int i = 0; i < undeclaredFields.length; i++) {
+ if (!nullMap.get(i)) {
+ jsonString.append(", ");
+ jsonString.append(" \"" + UNDECLARED_FIELD_NAMES[i] + "\": ");
+ jsonString.append(undeclaredFields[i]);
+ }
+ }
+
+ jsonString.append(" }"); // end order
+ return jsonString.toString();
+ }
+ }
+
+ public void init() {
+ try {
+ List<String> tmpFirstNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpFirstNames.add(capitalized);
+ }
+ in.close();
+ firstNames = tmpFirstNames.toArray(firstNames);
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ List<String> tmpLastNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpLastNames.add(capitalized);
+ }
+ in.close();
+ lastNames = tmpLastNames.toArray(firstNames);
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ rndUndeclaredOrderFields = new Random[UNDECLARED_FIELD_NAMES.length];
+ for (int i = 0; i < rndUndeclaredOrderFields.length; i++) {
+ rndUndeclaredOrderFields[i] = new Random(50);
+ }
+ }
+
+ public void writeOrdersList(List<Order> ordersList, FileWriter ordersFile) throws IOException {
+ while (!ordersList.isEmpty()) {
+ int ix = Math.abs(rndValue.nextInt()) % ordersList.size();
+ ordersFile.write(ordersList.get(ix).getJSON() + "\n");
+ ordersList.remove(ix);
+ }
+ }
+
+ public void writeCustomerList(List<Customer> customerList, Order[] ordersBatch, List<Order> ordersList,
+ FileWriter customersFile, FileWriter ordersFile) throws IOException {
+ while (!customerList.isEmpty()) {
+ int ix = Math.abs(rndValue.nextInt()) % customerList.size();
+ customersFile.write(customerList.get(ix).getJSON() + "\n");
+
+ // generate orders
+ int numOrders = Math.abs(rndValue.nextInt()) % (MAX_ORDERS_PER_CUST - MIN_ORDERS_PER_CUST)
+ + MIN_ORDERS_PER_CUST;
+ for (int i = 0; i < numOrders; i++) {
+ ordersBatch[i].generateFieldValues(customerList.get(ix));
+ ordersList.add(ordersBatch[i]);
+ }
+ writeOrdersList(ordersList, ordersFile);
+
+ customerList.remove(ix);
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.err.println("MUST PROVIDE 2 PARAMS, 1. output dir name and 2. number of records to generate.");
+ System.exit(1);
+ }
+
+ String outputFile = args[0];
+ int numRecords = Integer.parseInt(args[1]);
+
+ FileWriter customersFile = new FileWriter(outputFile + File.separator + "customer.adm");
+ FileWriter ordersFile = new FileWriter(outputFile + File.separator + "orders.adm");
+
+ CustOrdDataGen dataGen = new CustOrdDataGen();
+ dataGen.init();
+
+ int batchSize = 1000;
+ Customer[] customerBatch = new Customer[batchSize];
+ for (int i = 0; i < batchSize; i++) {
+ customerBatch[i] = dataGen.new Customer();
+ }
+
+ Order[] ordersBatch = new Order[MAX_ORDERS_PER_CUST];
+ for (int i = 0; i < MAX_ORDERS_PER_CUST; i++) {
+ ordersBatch[i] = dataGen.new Order();
+ }
+
+ List<Customer> customerList = new LinkedList<Customer>();
+ List<Order> ordersList = new LinkedList<Order>();
+ int custIx = 0;
+ for (int i = 0; i < numRecords; i++) {
+
+ customerBatch[custIx].generateFieldValues();
+ customerList.add(customerBatch[custIx]);
+ custIx++;
+
+ if (customerList.size() >= batchSize) {
+ dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+ custIx = 0;
+ }
+ }
+ dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+ customersFile.flush();
+ customersFile.close();
+
+ ordersFile.flush();
+ ordersFile.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
new file mode 100644
index 0000000..b175b42
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class EventDataGen {
+ private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+ private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+
+ private String[] firstNames = new String[10];
+ private String[] lastNames = new String[10];
+
+ private static final int MIN_USER_INTERESTS = 0;
+ private static final int MAX_USER_INTERESTS = 7;
+ private String[] INTERESTS = { "bass", "music", "databases", "fishing", "tennis", "squash", "computers", "books",
+ "movies", "cigars", "wine", "running", "walking", "skiing", "basketball", "video games", "cooking",
+ "coffee", "base jumping", "puzzles" };
+
+ private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+ "Lake St.", "Hill St.", "Park St.", "View St." };
+ private static final int MIN_STREET_NUM = 1;
+ private static final int MAX_STREET_NUM = 10000;
+ private static final String[] CITIES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+ "Huntington Beach", "Portland" };
+ private static final int MIN_ZIP = 100000;
+ private static final int MAX_ZIP = 999999;
+ private static final String[] LAT_LONGS = { "47,-122", "33,-117", "33,-117", "34,-118", "33,-117", "33,-117",
+ "45,-122" };
+
+ private static final int MIN_MEMBERSHIPS = 1;
+ private static final int MAX_MEMBERSHIPS = 10;
+ private static final int MIN_SIG_ID = 1;
+ private static final int MAX_SIG_ID = 100;
+ private static final String[] CHAPTER_NAMES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+ "Huntington Beach", "Portland", "Newport Beach", "Kirkland" };
+ private static final int MEMBER_SINCE_MIN_YEAR = 1970;
+ private static final int MEMBER_SINCE_MAX_YEAR = 1998;
+
+ private Random rndValue = new Random(50);
+ private User user;
+
+ private final class User {
+ private int firstNameIdx;
+ private int lastNameIdx;
+ private int[] interests = new int[MAX_USER_INTERESTS - MIN_USER_INTERESTS];
+ int numInterests;
+ private int streetNumber;
+ private String street;
+ private String city;
+ private int zip;
+ private String latlong;
+ int numMemberships;
+ private int[] member_sigid = new int[MAX_MEMBERSHIPS];
+ private String[] member_chap_name = new String[MAX_MEMBERSHIPS];
+ private String[] member_since_date = new String[MAX_MEMBERSHIPS];
+
+ public void generateFieldValues() {
+ firstNameIdx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ lastNameIdx = Math.abs(rndValue.nextInt()) % lastNames.length;
+ // name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+ numInterests = Math.abs((rndValue.nextInt()) % (MAX_USER_INTERESTS - MIN_USER_INTERESTS))
+ + MIN_USER_INTERESTS;
+ for (int i = 0; i < numInterests; i++) {
+ interests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+ }
+ streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+ street = STREETS[Math.abs(rndValue.nextInt()) % STREETS.length];
+ int cityIdx = Math.abs(rndValue.nextInt()) % CITIES.length;
+ city = CITIES[cityIdx];
+ zip = Math.abs(rndValue.nextInt() % (MAX_ZIP - MIN_ZIP)) + MIN_ZIP;
+ latlong = LAT_LONGS[cityIdx];
+ numMemberships = Math.abs(rndValue.nextInt()) % (MAX_MEMBERSHIPS - MIN_MEMBERSHIPS) + MIN_MEMBERSHIPS;
+ for (int i = 0; i < numMemberships; i++) {
+ member_sigid[i] = Math.abs(rndValue.nextInt()) % (MAX_SIG_ID - MIN_SIG_ID) + MIN_SIG_ID;
+ int cnIdx = Math.abs(rndValue.nextInt()) % CHAPTER_NAMES.length;
+ member_chap_name[i] = CHAPTER_NAMES[cnIdx];
+ int msYear = Math.abs(rndValue.nextInt()) % (MEMBER_SINCE_MAX_YEAR - MEMBER_SINCE_MIN_YEAR)
+ + MEMBER_SINCE_MIN_YEAR;
+ int msMo = Math.abs(rndValue.nextInt()) % 12 + 1;
+ int msDay = Math.abs(rndValue.nextInt()) % 28 + 1;
+ member_since_date[i] = msYear + "-" + (msMo < 10 ? "0" : "") + msMo + "-" + (msDay < 10 ? "0" : "")
+ + msDay;
+ }
+ }
+
+ public void write(Writer writer) throws IOException {
+ writer.append("{");
+ writer.append(" \"name\": \"");
+ writer.append(firstNames[firstNameIdx]);
+ writer.append(" ");
+ writer.append(lastNames[lastNameIdx]);
+ writer.append("\", ");
+
+ writer.append(" \"email\": \"");
+ writer.append(firstNames[firstNameIdx]);
+ writer.append(".");
+ writer.append(lastNames[lastNameIdx]);
+ writer.append("@example.com\", ");
+
+ writer.append(" \"interests\": <");
+ for (int i = 0; i < numInterests; i++) {
+ if (i > 0) {
+ writer.append(", ");
+ }
+ writer.append("\"");
+ writer.append(INTERESTS[interests[i]]);
+ writer.append("\"");
+ }
+ writer.append(">, ");
+
+ writer.append(" \"address\": {");
+ writer.append(" \"street\": \"");
+ writer.append(streetNumber + " " + street);
+ writer.append("\",");
+ writer.append(" \"city\": \"");
+ writer.append(city);
+ writer.append("\",");
+ writer.append(" \"zip\": \"");
+ writer.append(zip + "\",");
+ writer.append(" \"latlong\": point(\"");
+ writer.append(latlong);
+ writer.append("\")");
+ writer.append("}, ");
+
+ writer.append(" \"member_of\": <");
+ for (int i = 0; i < numMemberships; i++) {
+ if (i > 0) {
+ writer.append(", ");
+ }
+ writer.append("{");
+ writer.append(" \"sig_id\": ");
+ writer.append(member_sigid[i] + ",");
+ writer.append(" \"chapter_name\": \"");
+ writer.append(member_chap_name[i]);
+ writer.append("\",");
+ writer.append(" \"member_since\": date(\"");
+ writer.append(member_since_date[i]);
+ writer.append("\") }");
+ }
+ writer.append(">");
+
+ writer.append(" }\n");
+ }
+ }
+
+ public void init() throws IOException {
+ {
+ List<String> tmpFirstNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpFirstNames.add(capitalized);
+ }
+ in.close();
+ firstNames = tmpFirstNames.toArray(firstNames);
+ }
+ {
+ List<String> tmpLastNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpLastNames.add(capitalized);
+ }
+ in.close();
+ lastNames = tmpLastNames.toArray(firstNames);
+ }
+ user = new User();
+ }
+
+ public void generate() {
+ user.generateFieldValues();
+ }
+
+ public void write(Writer w) throws IOException {
+ user.write(w);
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.err
+ .println("MUST PROVIDE 2 PARAMETERS, 1. output directory path and 2. number of records to generate.");
+ System.exit(1);
+ }
+ String outputFile = args[0];
+ int numRecords = Integer.parseInt(args[1]);
+ Writer userFile = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile + File.separator
+ + "user.adm")));
+ EventDataGen dgen = new EventDataGen();
+ dgen.init();
+ for (int i = 0; i < numRecords; i++) {
+ dgen.generate();
+ dgen.write(userFile);
+ }
+ userFile.close();
+ }
+}