You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:49:43 UTC
[04/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
deleted file mode 100644
index fa7e51b..0000000
--- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ /dev/null
@@ -1,646 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by hongbin on 5/20/14.
- */
-public class FactTableGenerator {
- CubeInstance cube = null;
- CubeDesc desc = null;
- ResourceStore store = null;
- String factTableName = null;
-
- GenConfig genConf = null;
-
- Random r = null;
-
- String cubeName;
- long randomSeed;
- int rowCount;
- int unlinkableRowCount;
- int unlinkableRowCountMax;
- double conflictRatio;
- double linkableRatio;
-
- // the names of lookup table columns which is in relation with fact
- // table(appear as fk in fact table)
- TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // possible values of lookupTableKeys, extracted from existing lookup
- // tables.
- // The key is in the format of tablename/columnname
- TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // lookup table name -> sets of all composite keys
- TreeMap<String, HashSet<Array<String>>> lookupTableCompositeKeyValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- private void init(String cubeName, int rowCount, double conflictRaio, double linkableRatio, long randomSeed) {
- this.rowCount = rowCount;
- this.conflictRatio = conflictRaio;
- this.cubeName = cubeName;
- this.randomSeed = randomSeed;
- this.linkableRatio = linkableRatio;
-
- this.unlinkableRowCountMax = (int) (this.rowCount * (1 - linkableRatio));
- this.unlinkableRowCount = 0;
-
- r = new Random(randomSeed);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- cube = CubeManager.getInstance(config).getCube(cubeName);
- desc = cube.getDescriptor();
- factTableName = desc.getFactTable();
- store = ResourceStore.getStore(config);
- }
-
- /*
- * users can specify the value preference for each column
- */
- private void loadConfig() {
- try {
- InputStream configStream = store.getResource("/data/data_gen_config.json").inputStream;
- this.genConf = GenConfig.loadConfig(configStream);
-
- if (configStream != null)
- configStream.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void loadLookupTableValues(String lookupTableName, LinkedList<String> columnNames, int distinctRowCount) throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- // only deal with composite keys
- if (columnNames.size() > 1 && !lookupTableCompositeKeyValues.containsKey(lookupTableName)) {
- lookupTableCompositeKeyValues.put(lookupTableName, new HashSet<Array<String>>());
- }
-
- InputStream tableStream = null;
- BufferedReader tableReader = null;
- try {
- TreeMap<String, Integer> zeroBasedInice = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- for (String columnName : columnNames) {
- ColumnDesc cDesc = MetadataManager.getInstance(config).getTableDesc(lookupTableName).findColumnByName(columnName);
- zeroBasedInice.put(columnName, cDesc.getZeroBasedIndex());
- }
-
- String path = "/data/" + lookupTableName + ".csv";
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
- tableReader.mark(0);
- int rowCount = 0;
- int curRowNum = 0;
- String curRow;
-
- while (tableReader.readLine() != null)
- rowCount++;
-
- HashSet<Integer> rows = new HashSet<Integer>();
- distinctRowCount = (distinctRowCount < rowCount) ? distinctRowCount : rowCount;
- while (rows.size() < distinctRowCount) {
- rows.add(r.nextInt(rowCount));
- }
-
- // reopen the stream
- tableReader.close();
- tableStream.close();
- tableStream = null;
- tableReader = null;
-
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
-
- while ((curRow = tableReader.readLine()) != null) {
- if (rows.contains(curRowNum)) {
- String[] tokens = curRow.split(",");
-
- String[] comboKeys = null;
- int index = 0;
- if (columnNames.size() > 1)
- comboKeys = new String[columnNames.size()];
-
- for (String columnName : columnNames) {
- int zeroBasedIndex = zeroBasedInice.get(columnName);
- if (!feasibleValues.containsKey(lookupTableName + "/" + columnName))
- feasibleValues.put(lookupTableName + "/" + columnName, new ArrayList<String>());
- feasibleValues.get(lookupTableName + "/" + columnName).add(tokens[zeroBasedIndex]);
-
- if (columnNames.size() > 1) {
- comboKeys[index] = tokens[zeroBasedIndex];
- index++;
- }
- }
-
- if (columnNames.size() > 1) {
- Array<String> wrap = new Array<String>(comboKeys);
- if (lookupTableCompositeKeyValues.get(lookupTableName).contains(wrap)) {
- throw new Exception("The composite key already exist in the lookup table");
- }
- lookupTableCompositeKeyValues.get(lookupTableName).add(wrap);
- }
- }
- curRowNum++;
- }
-
- if (tableStream != null)
- tableStream.close();
- if (tableReader != null)
- tableReader.close();
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- // prepare the candidate values for each joined column
- private void prepare() throws Exception {
- // load config
- loadConfig();
-
- TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- for (DimensionDesc dim : desc.getDimensions()) {
- for (TblColRef col : dim.getColumnRefs()) {
- if (col.getTable().equals(factTableName))
- factTableColumns.add(col.getName());
- }
-
- JoinDesc join = dim.getJoin();
- if (join != null) {
- String lookupTable = dim.getTable();
- for (String column : join.getPrimaryKey()) {
- if (!lookupTableKeys.containsKey(lookupTable)) {
- lookupTableKeys.put(lookupTable, new LinkedList<String>());
- }
-
- if (!lookupTableKeys.get(lookupTable).contains(column))
- lookupTableKeys.get(lookupTable).add(column);
- }
- }
- }
-
- int distinctRowCount = (int) (this.rowCount / this.conflictRatio);
- distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount;
- // lookup tables
- for (String lookupTable : lookupTableKeys.keySet()) {
- this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount);
- }
- }
-
- private List<DimensionDesc> getSortedDimentsionDescs() {
- List<DimensionDesc> dimensions = desc.getDimensions();
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc o1, DimensionDesc o2) {
- JoinDesc j1 = o2.getJoin();
- JoinDesc j2 = o1.getJoin();
- return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0);
- }
- });
- return dimensions;
- }
-
- /**
- * Generate the fact table and return it as text
- *
- * @return
- * @throws Exception
- */
- private String cookData() throws Exception {
- // the columns on the fact table can be classified into three groups:
- // 1. foreign keys
- TreeMap<String, String> factTableCol2LookupCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- // 2. metrics or directly used dimensions
- TreeSet<String> usedCols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- // 3. others, not referenced anywhere
-
- TreeMap<String, String> lookupCol2factTableCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // find fact table columns in fks
- List<DimensionDesc> dimensions = getSortedDimentsionDescs();
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc != null) {
- String[] fks = jDesc.getForeignKey();
- String[] pks = jDesc.getPrimaryKey();
- int num = fks.length;
- for (int i = 0; i < num; ++i) {
- String value = dim.getTable() + "/" + pks[i];
-
- lookupCol2factTableCol.put(value, fks[i]);
-
- if (factTableCol2LookupCol.containsKey(fks[i])) {
- if (!factTableCol2LookupCol.get(fks[i]).equals(value)) {
- System.out.println("Warning: Disambiguation on the mapping of column " + fks[i] + ", " + factTableCol2LookupCol.get(fks[i]) + "(chosen) or " + value);
- continue;
- }
- }
- factTableCol2LookupCol.put(fks[i], value);
- }
- }
- //else, deal with it in next roung
- }
-
- // find fact table columns in direct dimension
- // DO NOT merge this with the previous loop
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc == null) {
- // column on fact table used directly as a dimension
- for (String aColumn : dim.getColumn()) {
- if (!factTableCol2LookupCol.containsKey(aColumn))
- usedCols.add(aColumn);
- }
- }
- }
-
- // find fact table columns in measures
- for (MeasureDesc mDesc : desc.getMeasures()) {
- List<TblColRef> pcols = mDesc.getFunction().getParameter().getColRefs();
- if (pcols != null) {
- for (TblColRef col : pcols) {
- if (!factTableCol2LookupCol.containsKey(col.getName()))
- usedCols.add(col.getName());
- }
- }
- }
-
- return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols);
- }
-
- private String normToTwoDigits(int v) {
- if (v < 10)
- return "0" + v;
- else
- return Integer.toString(v);
- }
-
- private String randomPick(ArrayList<String> candidates) {
- int index = r.nextInt(candidates.size());
- return candidates.get(index);
- }
-
- private String createRandomCell(ColumnDesc cDesc, ArrayList<String> range) throws Exception {
- DataType type = cDesc.getType();
- if (type.isStringFamily()) {
- throw new Exception("Can't handle range values for string");
-
- } else if (type.isIntegerFamily()) {
- int low = Integer.parseInt(range.get(0));
- int high = Integer.parseInt(range.get(1));
- return Integer.toString(r.nextInt(high - low) + low);
-
- } else if (type.isDouble()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isFloat()) {
- float low = Float.parseFloat(range.get(0));
- float high = Float.parseFloat(range.get(1));
- return String.format("%.4f", r.nextFloat() * (high - low) + low);
-
- } else if (type.isDecimal()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isDateTimeFamily()) {
-
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
- Date start = format.parse(range.get(0));
- Date end = format.parse(range.get(1));
- long diff = end.getTime() - start.getTime();
- Date temp = new Date(start.getTime() + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createRandomCell(ColumnDesc cDesc) {
- String type = cDesc.getTypeName();
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 2; i++) {
- sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
- // possible strings
- }
- return sb.toString();
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return Integer.toString(r.nextInt(128));
- } else if (s.equals("double")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("float")) {
- return String.format("%.4f", r.nextFloat() * 100);
- } else if (s.equals("decimal")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("date")) {
- long date20131231 = 61349312153265L;
- long date20010101 = 60939158400000L;
- long diff = date20131231 - date20010101;
- Date temp = new Date(date20010101 + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createDefaultsCell(String type) {
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
- return "abcde";
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return "0";
- } else if (s.equals("double")) {
- return "0";
- } else if (s.equals("float")) {
- return "0";
- } else if (s.equals("decimal")) {
- return "0";
- } else if (s.equals("date")) {
- return "1970-01-01";
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private void printColumnMappings(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) {
-
- System.out.println("=======================================================================");
- System.out.format("%-30s %s", "FACT_TABLE_COLUMN", "MAPPING");
- System.out.println();
- System.out.println();
- for (Map.Entry<String, String> entry : factTableCol2LookupCol.entrySet()) {
- System.out.format("%-30s %s", entry.getKey(), entry.getValue());
- System.out.println();
- }
- for (String key : usedCols) {
- System.out.format("%-30s %s", key, "Random Values");
- System.out.println();
- }
- for (String key : defaultColumns) {
- System.out.format("%-30s %s", key, "Default Values");
- System.out.println();
- }
- System.out.println("=======================================================================");
-
- System.out.println("Parameters:");
- System.out.println();
- System.out.println("CubeName: " + cubeName);
- System.out.println("RowCount: " + rowCount);
- System.out.println("ConflictRatio: " + conflictRatio);
- System.out.println("LinkableRatio: " + linkableRatio);
- System.out.println("Seed: " + randomSeed);
- System.out.println();
- System.out.println("The number of actual unlinkable fact rows is: " + this.unlinkableRowCount);
- System.out.println("You can vary the above parameters to generate different datasets.");
- System.out.println();
- }
-
- // Any row in the column must finally appear in the flatten big table.
- // for single-column joins the generated row is guaranteed to have a match
- // in lookup table
- // for composite keys we'll need an extra check
- private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- for (String lookupTable : lookupTableKeys.keySet()) {
- if (lookupTableKeys.get(lookupTable).size() == 1)
- continue;
-
- String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()];
- int index = 0;
- for (String column : lookupTableKeys.get(lookupTable)) {
- String key = lookupTable + "/" + column;
- String factTableCol = lookupCol2FactTableCol.get(key);
- int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex();
- comboKey[index] = columnValues.get(cardinal);
-
- index++;
- }
- Array<String> wrap = new Array<String>(comboKey);
- if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) {
- // System.out.println("Try " + wrap + " Failed, continue...");
- return false;
- }
- }
- return true;
- }
-
- private String createCell(ColumnDesc cDesc) throws Exception {
- ColumnConfig cConfig = null;
-
- if ((cConfig = genConf.getColumnConfigByName(cDesc.getName())) == null) {
- // if the column is not configured, use random values
- return (createRandomCell(cDesc));
-
- } else {
- // the column has a configuration
- if (!cConfig.isAsRange() && !cConfig.isExclusive() && r.nextBoolean()) {
- // if the column still allows random values
- return (createRandomCell(cDesc));
-
- } else {
- // use specified values
- ArrayList<String> valueSet = cConfig.getValueSet();
- if (valueSet == null || valueSet.size() == 0)
- throw new Exception("Did you forget to specify value set for " + cDesc.getName());
-
- if (!cConfig.isAsRange()) {
- return (randomPick(valueSet));
- } else {
- if (valueSet.size() != 2)
- throw new Exception("Only two values can be set for range values, the column: " + cDesc.getName());
-
- return (createRandomCell(cDesc, valueSet));
- }
- }
-
- }
- }
-
- private LinkedList<String> createRow(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- LinkedList<String> columnValues = new LinkedList<String>();
-
- for (ColumnDesc cDesc : MetadataManager.getInstance(config).getTableDesc(factTableName).getColumns()) {
-
- String colName = cDesc.getName();
-
- if (factTableCol2LookupCol.containsKey(colName)) {
-
- // if the current column is a fk column in fact table
- ArrayList<String> candidates = this.feasibleValues.get(factTableCol2LookupCol.get(colName));
-
- columnValues.add(candidates.get(r.nextInt(candidates.size())));
- } else if (usedCols.contains(colName)) {
-
- // if the current column is a metric column in fact table
- columnValues.add(createCell(cDesc));
- } else {
-
- // otherwise this column is not useful in OLAP
- columnValues.add(createDefaultsCell(cDesc.getTypeName()));
- defaultColumns.add(colName);
- }
- }
-
- return columnValues;
- }
-
- /**
- * return the text of table contents(one line one row)
- *
- * @param rowCount
- * @param factTableCol2LookupCol
- * @param lookupCol2FactTableCol
- * @param usedCols
- * @return
- * @throws Exception
- */
- private String createTable(int rowCount, TreeMap<String, String> factTableCol2LookupCol, TreeMap<String, String> lookupCol2FactTableCol, TreeSet<String> usedCols) throws Exception {
- try {
- TreeSet<String> defaultColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < rowCount;) {
-
- LinkedList<String> columnValues = createRow(factTableCol2LookupCol, usedCols, defaultColumns);
-
- if (!matchAllCompositeKeys(lookupCol2FactTableCol, columnValues)) {
- if (unlinkableRowCount < unlinkableRowCountMax) {
- unlinkableRowCount++;
- } else {
- continue;
- }
- }
-
- for (String c : columnValues)
- sb.append(c + ",");
- sb.deleteCharAt(sb.length() - 1);
- sb.append(System.getProperty("line.separator"));
-
- i++;
-
- // System.out.println("Just generated the " + i + "th record");
- }
-
- printColumnMappings(factTableCol2LookupCol, usedCols, defaultColumns);
-
- return sb.toString();
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- return null;
- }
-
- /**
- * Randomly create a fact table and return the table content
- *
- * @param cubeName name of the cube
- * @param rowCount expected row count generated
- * @param linkableRatio the percentage of fact table rows that can be linked with all
- * lookup table by INNER join
- * @param randomSeed random seed
- */
- public static String generate(String cubeName, String rowCount, String linkableRatio, String randomSeed, String joinType) throws Exception {
-
- if (cubeName == null)
- cubeName = "test_kylin_cube_with_slr_ready";
- if (rowCount == null)
- rowCount = "10000";
- if (linkableRatio == null)
- linkableRatio = "0.6";
-
- //if (randomSeed == null)
- // don't give it value
-
- // String conflictRatio = "5";//this parameter do not allow configuring
- // any more
-
- FactTableGenerator generator = new FactTableGenerator();
- long seed;
- if (randomSeed != null) {
- seed = Long.parseLong(randomSeed);
- } else {
- Random r = new Random();
- seed = r.nextLong();
- }
-
- generator.init(cubeName, Integer.parseInt(rowCount), 5, Double.parseDouble(linkableRatio), seed);
- generator.prepare();
- return generator.cookData();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
deleted file mode 100644
index f7d7341..0000000
--- a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.JsonUtil;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-
-/**
- * Created by honma on 5/29/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class GenConfig {
-
- @JsonProperty("columnConfigs")
- private ArrayList<ColumnConfig> columnConfigs;
-
- private HashMap<String, ColumnConfig> cache = new HashMap<String, ColumnConfig>();
-
- public ArrayList<ColumnConfig> getColumnConfigs() {
- return columnConfigs;
- }
-
- public void setColumnConfigs(ArrayList<ColumnConfig> columnConfigs) {
- this.columnConfigs = columnConfigs;
- }
-
- public ColumnConfig getColumnConfigByName(String columnName) {
- columnName = columnName.toLowerCase();
-
- if (cache.containsKey(columnName))
- return cache.get(columnName);
-
- for (ColumnConfig cConfig : columnConfigs) {
- if (cConfig.getColumnName().toLowerCase().equals(columnName)) {
- cache.put(columnName, cConfig);
- return cConfig;
- }
- }
- cache.put(columnName, null);
- return null;
- }
-
- public static GenConfig loadConfig(InputStream stream) {
- try {
- GenConfig config = JsonUtil.readValue(stream, GenConfig.class);
- return config;
- } catch (JsonMappingException e) {
- e.printStackTrace();
- } catch (JsonParseException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
deleted file mode 100644
index 8d22efb..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
-
- public static void setLocalMR(Configuration conf) {
- conf.set("fs.default.name", "file:///");
- conf.set("mapreduce.framework.name", "local");
- conf.set("mapreduce.application.framework.path", "");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
- }
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/flat_table/";
- String output = "target/test-output/base_cuboid/";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "base_cuboid_job";
- String level = "0";
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
deleted file mode 100644
index f87bb0a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class BaseCuboidMapperPerformanceTest {
-
- String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure";
- String cubeName = "test_kylin_cube_with_slr";
- Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq");
-
- @Ignore("convenient trial tool for dev")
- @Test
- public void test() throws IOException, InterruptedException {
- Configuration hconf = new Configuration();
- BaseCuboidMapper mapper = new BaseCuboidMapper();
- Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
-
- mapper.setup(context);
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- while (reader.next(key, value)) {
- mapper.map(key, value, context);
- }
-
- reader.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
deleted file mode 100644
index 58d093a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowKeyDecoder;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
-
- MapDriver<Text, Text, Text, Text> mapDriver;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapperWithHeader() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.3322"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", "1", "22");
- }
-
- private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String... valueStr) {
- MeasureCodec codec = new MeasureCodec(measures);
- Object[] values = new Object[measures.size()];
- codec.decode(ByteBuffer.wrap(valueBytes.getBytes()), values);
- assertTrue(new BigDecimal(valueStr[0]).equals(values[0]));
- assertTrue(new BigDecimal(valueStr[1]).equals(values[1]));
- assertTrue(new BigDecimal(valueStr[2]).equals(values[2]));
- assertTrue(new LongWritable(Long.valueOf(valueStr[3])).equals(values[3]));
- assertTrue(new LongWritable(Long.valueOf(valueStr[4])).equals(values[4]));
- }
-
- @Test
- public void testMapperWithNull() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N22"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", "1", "22");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
deleted file mode 100644
index a8e29e6..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * @author yangli9
- *
- */
-public class CopySeq {
-
- public static void main(String[] args) throws IOException {
- copyTo64MB(args[0], args[1]);
- }
-
- public static void copyTo64MB(String src, String dst) throws IOException {
- Configuration hconf = new Configuration();
- Path srcPath = new Path(src);
- Path dstPath = new Path(dst);
-
- FileSystem fs = FileSystem.get(hconf);
- long srcSize = fs.getFileStatus(srcPath).getLen();
- int copyTimes = (int) (67108864 / srcSize); // 64 MB
- System.out.println("Copy " + copyTimes + " times");
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf)));
-
- int count = 0;
- while (reader.next(key, value)) {
- for (int i = 0; i < copyTimes; i++) {
- writer.append(key, value);
- count++;
- }
- }
-
- System.out.println("Len: " + writer.getLength());
- System.out.println("Rows: " + count);
-
- reader.close();
- writer.close();
- }
-
- static CompressionCodec getLZOCodec(Configuration hconf) {
- CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
- return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
deleted file mode 100644
index c9b7eba..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.measure.MeasureCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
-
- String cubeName = "test_kylin_cube_with_slr_ready";
-
- MeasureCodec codec;
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- Object[] outKV = new Object[2];
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
- CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor();
- codec = new MeasureCodec(desc.getMeasures());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testBasic() throws Exception {
-
- Configuration hconf = new Configuration();
- Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV);
-
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapper.setup(context);
-
- Text key = new Text("not important");
- Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
-
- mapper.map(key, value, context);
-
- ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
- KeyValue outValue = (KeyValue) outKV[1];
-
- assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0);
-
- assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
deleted file mode 100644
index 3555782..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeHFileMapperTest {
-
- MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver;
-
- private String cube_name = "FLAT_ITEM_CUBE";
-
- @Before
- public void setUp() {
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- @Ignore("not maintaining")
- public void testMapper2() throws IOException {
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);
-
- mapDriver.addInput(new Text("52010tech"), new Text("35.432"));
-
- List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run();
-
- assertEquals(2, result.size());
-
- byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
- ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);
-
- Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0);
- Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
-
- assertEquals(key, p1.getFirst());
- assertEquals("cf1", new String(p1.getSecond().getFamily()));
- assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
- assertEquals("35.43", new String(p1.getSecond().getValue()));
-
- assertEquals(key, p2.getFirst());
- assertEquals("cf1", new String(p2.getSecond().getFamily()));
- assertEquals("item_count", new String(p2.getSecond().getQualifier()));
- assertEquals("2", new String(p2.getSecond().getValue()));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
deleted file mode 100644
index 51f3990..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CubeReducerTest extends LocalFileMetadataTestCase {
-
- ReduceDriver<Text, Text, Text, Text> reduceDriver;
-
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- CuboidReducer reducer = new CuboidReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testReducer() throws Exception {
-
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
-
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 22));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 23));
- values1.add(newValueText(codec, "10", "10", "10", 1, 24));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 12));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 13));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4, 11));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 69));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 25));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4, 11));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- @Test
- public void testReducerOnlyAggrInBaseCuboid() throws Exception {
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
- reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);
-
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
- FunctionDesc functionDesc = measureDesc.getFunction();
- Field field = FunctionDesc.class.getDeclaredField("measureType");
- field.setAccessible(true);
- MeasureType origMeasureType = functionDesc.getMeasureType();
- field.set(functionDesc, new MockUpMeasureType(origMeasureType));
-
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
- values1.add(newValueText(codec, "10", "10", "10", 1, 300));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count, int itemcount) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count), new LongWritable(itemcount) };
-
- buf.clear();
- codec.encode(values, buf);
-
- Text t = new Text();
- t.set(buf.array(), 0, buf.position());
- return t;
- }
-
- class MockUpMeasureType extends MeasureType {
- MeasureType origMeasureType;
-
- public MockUpMeasureType(MeasureType origMeasureType) {
- this.origMeasureType = origMeasureType;
- }
-
- @Override
- public boolean onlyAggrInBaseCuboid() {
- return true;
- }
-
- @Override
- public MeasureIngester newIngester() {
- return origMeasureType.newIngester();
- }
-
- @Override
- public MeasureAggregator newAggregator() {
- return origMeasureType.newAggregator();
- }
-
- @Override
- public boolean needRewrite() {
- return origMeasureType.needRewrite();
- }
-
- @Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return origMeasureType.getRewriteCalciteAggrFunctionClass();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
deleted file mode 100644
index 6f7db6a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileUtil;
-//import org.apache.hadoop.util.ToolRunner;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionJobTest {
-//
-// private Configuration conf;
-//
-// @Before
-// public void setup() throws IOException {
-// conf = new Configuration();
-// BaseCuboidJobTest.setLocalMR(conf);
-// }
-//
-// @Test
-// public void testJob() throws Exception {
-// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
-// final String output = "target/test-output/key_distribution/";
-// final String cubeName = "test_kylin_cube_with_slr";
-// final String metadata = MetadataManager.getMetadataUrlFromEnv();
-//
-// FileUtil.fullyDelete(new File(output));
-//
-// String[] args =
-// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata,
-// "-columnpercentage", "50", "-splitnumber", "10" };
-// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args));
-// }
-//
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
deleted file mode 100644
index b4577f7..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.util.List;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-//import org.apache.hadoop.mrunit.types.Pair;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.index.BatchConstants;
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author ysong1
-// *
-// */
-//public class KeyDistributionMapperTest {
-// @SuppressWarnings("rawtypes")
-// MapDriver mapDriver;
-// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-//
-// @Before
-// public void setUp() {
-// KeyDistributionMapper mapper = new KeyDistributionMapper();
-// mapDriver = MapDriver.newMapDriver(mapper);
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithoutHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2,
-// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3,
-// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4,
-// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5,
-// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6,
-// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
deleted file mode 100644
index 6c17c69..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- // String input =
- // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid";
- String output = "target/test-output/merged_cuboid";
- String cubeName = "test_kylin_cube_with_slr_ready";
- String jobname = "merge_cuboid";
-
- File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- FileUtils.forceDelete(baseFolder);
- baseFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- FileUtils.forceDeleteOnExit(baseFolder);
-
- File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d");
- FileUtils.forceDelete(eightFolder);
- eightFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder);
- FileUtils.forceDeleteOnExit(eightFolder);
-
- FileUtil.fullyDelete(new File(output));
-
- // CubeManager cubeManager =
- // CubeManager.getInstanceFromEnv(getTestConfig());
-
- String[] args = { "-input", baseFolder.getAbsolutePath() + "," + eightFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
deleted file mode 100644
index ea5c163..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.ListDictionaryValueEnumerator;
-import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-@SuppressWarnings("rawtypes")
-public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
-
- private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class);
-
- MapDriver<Text, Text, Text, Text> mapDriver;
- CubeManager cubeManager;
- CubeInstance cube;
- DictionaryManager dictionaryManager;
-
- TblColRef lfn;
- TblColRef lsi;
- TblColRef ssc;
-
- private DictionaryInfo makeSharedDict() throws IOException {
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_common_dict");
-
- DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 101, 101, 101 });
- values.add(new byte[] { 102, 102, 102 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values));
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- return newDictInfo;
- }
-
- @Before
- public void setUp() throws Exception {
-
- createTestMetadata();
-
- logger.info("The metadataUrl is : " + getTestConfig());
-
- MetadataManager.clearCache();
- CubeManager.clearCache();
- ProjectManager.clearCache();
- DictionaryManager.clearCache();
-
- // hack for distributed cache
- // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
- // make sure the following mapper could get latest CubeManger
- FileUtils.deleteDirectory(new File("../job/meta"));
-
- MergeCuboidMapper mapper = new MergeCuboidMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
-
- cubeManager = CubeManager.getInstance(getTestConfig());
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
- dictionaryManager = DictionaryManager.getInstance(getTestConfig());
- lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
- lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
- ssc = cube.getDescriptor().findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
-
- DictionaryInfo sharedDict = makeSharedDict();
-
- boolean isFirstSegment = true;
- for (CubeSegment segment : cube.getSegments()) {
-
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
-
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 97, 97, 97 });
- if (isFirstSegment)
- values.add(new byte[] { 99, 99, 99 });
- else
- values.add(new byte[] { 98, 98, 98 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values));
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- segment.putDictResPath(lfn, newDictInfo.getResourcePath());
- segment.putDictResPath(lsi, sharedDict.getResourcePath());
- segment.putDictResPath(ssc, sharedDict.getResourcePath());
-
- // cubeManager.saveResource(segment.getCubeInstance());
- // cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
-
- isFirstSegment = false;
- }
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void test() throws IOException, ParseException {
-
- String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";
-
- CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, 1386835200000L, false);
- String segmentName = newSeg.getName();
-
- final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn);
- assertTrue(dictionary == null);
- // ((TrieDictionary) dictionary).dump(System.out);
-
- // hack for distributed cache
- // File metaDir = new File("../job/meta");
- // FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), metaDir);
- //
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
- // // "../job/meta");
- //
- // byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
- // byte[] value = new byte[] { 1, 2, 3 };
- // byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
- // byte[] newvalue = new byte[] { 1, 2, 3 };
- //
- // mapDriver.withInput(new Text(key), new Text(value));
- // mapDriver.withOutput(new Text(newkey), new Text(newvalue));
- // mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
- //
- // mapDriver.runTest();
- }
-}