You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/16 10:17:01 UTC
[1/2] kylin git commit: halfway
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2283 e79626e7c -> 5a27f0331
halfway
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32b00d14
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32b00d14
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32b00d14
Branch: refs/heads/KYLIN-2283
Commit: 32b00d14589e940063c01df7e0b63ac42098d3bf
Parents: e79626e
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 16 17:45:03 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 17:45:03 2016 +0800
----------------------------------------------------------------------
.../kylin/source/datagen/ColumnGenerator.java | 83 ++++++++++++++++++--
1 file changed, 76 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/32b00d14/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
index 409a1e9..9ffc9f6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -26,9 +26,13 @@ import java.util.List;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
+import com.google.common.base.Preconditions;
+
public class ColumnGenerator {
final private ColumnGenConfig conf;
@@ -48,7 +52,7 @@ public class ColumnGenerator {
if (conf.isID) {
result = new IDIter(conf.idStart);
} else if (conf.isRandom) {
- result = new RandomIter(targetCol.getType(), conf.randFormat, conf.randStart, Math.max(conf.randEnd, conf.randStart + conf.cardinality));
+ result = new RandomIter(targetCol.getType(), conf.randFormat, conf.randStart, conf.randEnd, conf.cardinality));
} else {
result = new DiscreteIter(conf.values);
}
@@ -70,20 +74,85 @@ public class ColumnGenerator {
private static class RandomIter implements Iterator<String> {
- public RandomIter(DataType type, String format, int randStart, int randEnd) {
- // TODO Auto-generated constructor stub
+ private DataType type;
+ private String format;
+ private int randStart;
+ private int randEnd;
+
+ public RandomIter(DataType type, String format, int randStart, int randEnd, int cardinality) {
+ if (type.isStringFamily()) {
+ // string
+ Preconditions.checkArgument(format.contains(ColumnGenConfig.$RANDOM));
+ initStartEndRange(randStart, randEnd, cardinality);
+ } else if (type.isTimeFamily()) {
+ // time
+ format = StringUtil.noBlank(format, DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ initDateTimeRange(randStart, randEnd, 0);
+ } else if (type.isDateTimeFamily()) {
+ // date
+ format = StringUtil.noBlank(format, DateFormat.DEFAULT_DATE_PATTERN);
+ initDateTimeRange(randStart, randEnd, cardinality);
+ } else if (type.isIntegerFamily()) {
+ // integer
+ initStartEndRange(randStart, randEnd, cardinality);
+ } else if (type.isNumberFamily()) {
+ // double
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ this.type = type;
+ this.format = format;
+ }
+
+ private void initDateTimeRange(int randStart, int randEnd, int days) {
+ if (randStart == 0 && randEnd == 0) {
+ randStart = 2010;
+ randEnd = 2015;
+ }
+ randEnd = Math.max(randEnd, randStart + (days / 365) + 1);
+
+ Preconditions.checkArgument(randStart < randEnd);
+ Preconditions.checkArgument((randEnd - randStart) * 365 >= days);
+
+ this.randStart = randStart;
+ this.randEnd = randEnd;
+ }
+
+ private void initStartEndRange(int randStart, int randEnd, int cardinality) {
+ if (randStart == 0 && randEnd == 0) {
+ randStart = 10000;
+ randEnd = 100000;
+ }
+ randEnd = Math.max(randEnd, randStart + cardinality);
+
+ Preconditions.checkArgument(randStart < randEnd);
+ Preconditions.checkArgument(randEnd - randStart >= cardinality);
+
+ this.randStart = randStart;
+ this.randEnd = randEnd;
}
@Override
public boolean hasNext() {
- // TODO Auto-generated method stub
- return false;
+ return true;
}
@Override
public String next() {
- // TODO Auto-generated method stub
- return null;
+ if (type.isStringFamily()) {
+ // string
+ } else if (type.isTimeFamily()) {
+ // time
+ } else if (type.isDateTimeFamily()) {
+ // date
+ } else if (type.isIntegerFamily()) {
+ // integer
+ } else if (type.isNumberFamily()) {
+ // double
+ } else {
+ throw new IllegalStateException();
+ }
}
@Override
[2/2] kylin git commit: pending PK/FK
Posted by li...@apache.org.
pending PK/FK
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a27f033
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a27f033
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a27f033
Branch: refs/heads/KYLIN-2283
Commit: 5a27f0331f3cadf772926ce45a13731b7a7b6bdd
Parents: 32b00d1
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 16 18:16:56 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 18:16:56 2016 +0800
----------------------------------------------------------------------
.../kylin/source/datagen/ColumnGenConfig.java | 3 +-
.../kylin/source/datagen/ColumnGenerator.java | 35 +++++++++++++++-----
2 files changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a27f033/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
index f174eef..1aac954 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
public class ColumnGenConfig {
@@ -48,7 +49,7 @@ public class ColumnGenConfig {
double genNullPct;
boolean order;
- public ColumnGenConfig(ColumnDesc col) {
+ public ColumnGenConfig(ColumnDesc col, DataModelDesc model) {
init(col.getName(), col.getDataGen());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a27f033/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
index 9ffc9f6..55cbcf9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -20,7 +20,6 @@ package org.apache.kylin.source.datagen;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -38,13 +37,11 @@ public class ColumnGenerator {
final private ColumnGenConfig conf;
final private ColumnDesc targetCol;
final private int targetRows;
- final private ModelDataGenerator modelGen;
public ColumnGenerator(ColumnDesc col, int nRows, ModelDataGenerator modelGen) {
- this.conf = new ColumnGenConfig(col);
+ this.conf = new ColumnGenConfig(col, modelGen.getModle());
this.targetCol = col;
this.targetRows = nRows;
- this.modelGen = modelGen;
}
public Iterator<String> generate() {
@@ -52,7 +49,7 @@ public class ColumnGenerator {
if (conf.isID) {
result = new IDIter(conf.idStart);
} else if (conf.isRandom) {
- result = new RandomIter(targetCol.getType(), conf.randFormat, conf.randStart, conf.randEnd, conf.cardinality));
+ result = new RandomIter(targetCol.getType(), conf.randFormat, conf.randStart, conf.randEnd, conf.cardinality);
} else {
result = new DiscreteIter(conf.values);
}
@@ -78,12 +75,13 @@ public class ColumnGenerator {
private String format;
private int randStart;
private int randEnd;
+ private Random rand = new Random();
public RandomIter(DataType type, String format, int randStart, int randEnd, int cardinality) {
if (type.isStringFamily()) {
// string
Preconditions.checkArgument(format.contains(ColumnGenConfig.$RANDOM));
- initStartEndRange(randStart, randEnd, cardinality);
+ initNumberRange(randStart, randEnd, cardinality);
} else if (type.isTimeFamily()) {
// time
format = StringUtil.noBlank(format, DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
@@ -94,9 +92,10 @@ public class ColumnGenerator {
initDateTimeRange(randStart, randEnd, cardinality);
} else if (type.isIntegerFamily()) {
// integer
- initStartEndRange(randStart, randEnd, cardinality);
+ initNumberRange(randStart, randEnd, cardinality);
} else if (type.isNumberFamily()) {
// double
+ initNumberRange(randStart, randEnd, 0);
} else {
throw new IllegalArgumentException();
}
@@ -119,7 +118,7 @@ public class ColumnGenerator {
this.randEnd = randEnd;
}
- private void initStartEndRange(int randStart, int randEnd, int cardinality) {
+ private void initNumberRange(int randStart, int randEnd, int cardinality) {
if (randStart == 0 && randEnd == 0) {
randStart = 10000;
randEnd = 100000;
@@ -142,19 +141,39 @@ public class ColumnGenerator {
public String next() {
if (type.isStringFamily()) {
// string
+ return format.replace(ColumnGenConfig.$RANDOM, "" + randomInt());
} else if (type.isTimeFamily()) {
// time
+ return DateFormat.formatToTimeStr(randomMillis(), format);
} else if (type.isDateTimeFamily()) {
// date
+ return DateFormat.formatToDateStr(randomMillis(), format);
} else if (type.isIntegerFamily()) {
// integer
+ return "" + randomInt();
} else if (type.isNumberFamily()) {
// double
+ return "" + randomDouble();
} else {
throw new IllegalStateException();
}
}
+ private int randomInt() {
+ return randStart + rand.nextInt(randEnd - randStart);
+ }
+
+ private double randomDouble() {
+ return randStart + (double) rand.nextInt(100 * (randEnd - randStart)) / 100D;
+ }
+
+ private long randomMillis() {
+ int secondsInYear = 3600 * 24 * 365;
+ long year = randStart + rand.nextInt(randEnd - randStart) - 1970;
+ long second = year * secondsInYear + rand.nextInt(secondsInYear);
+ return second * 1000;
+ }
+
@Override
public void remove() {
throw new UnsupportedOperationException();