You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/13 07:44:52 UTC
incubator-kylin git commit: KYLIN-877 Simply interface ReadtableTable
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging 83d05800d -> 024cc7d82
KYLIN-877 Simply interface ReadtableTable
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/024cc7d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/024cc7d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/024cc7d8
Branch: refs/heads/0.7-staging
Commit: 024cc7d82956f06744f628cb4f0b579f52a5a765
Parents: 83d0580
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Jul 13 13:44:43 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 13 13:44:43 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryGenerator.java | 9 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 23 +---
.../apache/kylin/dict/DictionaryManager.java | 13 +--
.../org/apache/kylin/dict/lookup/FileTable.java | 8 +-
.../kylin/dict/lookup/FileTableReader.java | 11 +-
.../org/apache/kylin/dict/lookup/HiveTable.java | 5 -
.../kylin/dict/lookup/HiveTableReader.java | 6 +-
.../apache/kylin/dict/lookup/LookupTable.java | 2 +
.../apache/kylin/dict/lookup/ReadableTable.java | 106 +++++++++++++++++-
.../kylin/dict/lookup/SnapshotManager.java | 6 +-
.../apache/kylin/dict/lookup/SnapshotTable.java | 14 ---
.../apache/kylin/dict/lookup/TableReader.java | 36 ------
.../kylin/dict/lookup/TableSignature.java | 111 -------------------
.../apache/kylin/dict/SnapshotManagerTest.java | 9 +-
.../org/apache/kylin/dict/TableReaderTest.java | 5 +-
.../job/hadoop/cube/MergeCuboidMapperTest.java | 21 ++--
16 files changed, 144 insertions(+), 241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 9a52235..fe9f52a 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -29,16 +29,15 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.dict.lookup.TableReader;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.dict.lookup.ReadableTable;
+import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
+import org.apache.kylin.metadata.model.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.dict.lookup.ReadableTable;
-import org.apache.kylin.metadata.model.DataType;
-
/**
* @author yangli9
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index b6109c7..2806ade 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -18,12 +18,13 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.dict.lookup.TableSignature;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class DictionaryInfo extends RootPersistentEntity {
@@ -38,8 +39,6 @@ public class DictionaryInfo extends RootPersistentEntity {
private String dataType;
@JsonProperty("input")
private TableSignature input;
- @JsonProperty("input_delimeter")
- private String inputDelimeter;
@JsonProperty("dictionary_class")
private String dictionaryClass;
@JsonProperty("cardinality")
@@ -50,7 +49,7 @@ public class DictionaryInfo extends RootPersistentEntity {
public DictionaryInfo() {
}
- public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input, String inputDelimeter) {
+ public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) {
this.updateRandomUuid();
@@ -59,7 +58,6 @@ public class DictionaryInfo extends RootPersistentEntity {
this.sourceColumnIndex = sourceColumnIndex;
this.dataType = dataType;
this.input = input;
- this.inputDelimeter = inputDelimeter;
}
public DictionaryInfo(DictionaryInfo other) {
@@ -71,7 +69,6 @@ public class DictionaryInfo extends RootPersistentEntity {
this.sourceColumnIndex = other.sourceColumnIndex;
this.dataType = other.dataType;
this.input = other.input;
- this.inputDelimeter = other.inputDelimeter;
}
// ----------------------------------------------------------------------------
@@ -89,7 +86,7 @@ public class DictionaryInfo extends RootPersistentEntity {
// to decide if two dictionaries are built on the same table/column,
// regardless of their signature
public boolean isDictOnSameColumn(DictionaryInfo other) {
- return this.sourceTable.equalsIgnoreCase(other.sourceTable) && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) && this.inputDelimeter.equalsIgnoreCase(other.inputDelimeter) && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
+ return this.sourceTable.equalsIgnoreCase(other.sourceTable) && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
}
public String getSourceTable() {
@@ -132,14 +129,6 @@ public class DictionaryInfo extends RootPersistentEntity {
this.input = input;
}
- public String getInputDelimeter() {
- return inputDelimeter;
- }
-
- public void setInputDelimeter(String inputDelimeter) {
- this.inputDelimeter = inputDelimeter;
- }
-
public String getDictionaryClass() {
return dictionaryClass;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 6560a51..7f274de 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -30,19 +30,18 @@ import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.TableSignature;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.dict.lookup.FileTable;
+import org.apache.kylin.dict.lookup.HiveTable;
import org.apache.kylin.dict.lookup.ReadableTable;
+import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DictionaryManager {
@@ -174,7 +173,7 @@ public class DictionaryManager {
int srcColIdx = (Integer) tmp[2];
ReadableTable inpTable = (ReadableTable) tmp[3];
- DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature(), inpTable.getColumnDelimeter());
+ DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature());
String dupDict = checkDupByInfo(dictInfo);
if (dupDict != null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
index a659ca3..b1b0e4d 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
@@ -32,6 +32,9 @@ import org.apache.kylin.common.util.HadoopUtil;
*/
public class FileTable implements ReadableTable {
+ public static final String DELIM_AUTO = "auto";
+ public static final String DELIM_COMMA = ",";
+
String path;
String delim;
int nColumns;
@@ -56,11 +59,6 @@ public class FileTable implements ReadableTable {
}
@Override
- public String getColumnDelimeter() {
- return delim;
- }
-
- @Override
public TableReader getReader() throws IOException {
return new FileTableReader(path, delim, nColumns);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
index 1226d72..8674852 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
@@ -37,9 +37,9 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
/**
* Tables are typically CSV or SEQ file.
@@ -89,11 +89,6 @@ public class FileTableReader implements TableReader {
}
@Override
- public void setExpectedColumnNumber(int expectedColumnNumber) {
- this.expectedColumnNumber = expectedColumnNumber;
- }
-
- @Override
public boolean next() throws IOException {
curLine = reader.nextLine();
curColumns = null;
@@ -107,7 +102,7 @@ public class FileTableReader implements TableReader {
@Override
public String[] getRow() {
if (curColumns == null) {
- if (ReadableTable.DELIM_AUTO.equals(delim))
+ if (FileTable.DELIM_AUTO.equals(delim))
delim = autoDetectDelim(curLine);
if (delim == null)
@@ -123,7 +118,7 @@ public class FileTableReader implements TableReader {
String str[] = StringSplitter.split(line, delim);
// un-escape CSV
- if (ReadableTable.DELIM_COMMA.equals(delim)) {
+ if (FileTable.DELIM_COMMA.equals(delim)) {
for (int i = 0; i < str.length; i++) {
str[i] = unescapeCsv(str[i]);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
index aa22814..e926a89 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
@@ -59,11 +59,6 @@ public class HiveTable implements ReadableTable {
}
@Override
- public String getColumnDelimeter() throws IOException {
- return getFileTable().getColumnDelimeter();
- }
-
- @Override
public TableReader getReader() throws IOException {
return new HiveTableReader(database, hiveTable);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
index 870935b..07492cc 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
@@ -33,6 +33,7 @@ import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
import org.apache.hive.hcatalog.data.transfer.HCatReader;
import org.apache.hive.hcatalog.data.transfer.ReadEntity;
import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
/**
* An implementation of TableReader with HCatalog for Hive table.
@@ -114,11 +115,6 @@ public class HiveTableReader implements TableReader {
}
@Override
- public void setExpectedColumnNumber(int expectedColumnNumber) {
-
- }
-
- @Override
public void close() throws IOException {
this.readCntxt = null;
this.currentHCatRecordItr = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 50d4cbc..7e83197 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -28,7 +28,9 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Sets;
+
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
import org.apache.kylin.metadata.model.TableDesc;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
index 321d832..2e6af14 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
@@ -18,20 +18,116 @@
package org.apache.kylin.dict.lookup;
+import java.io.Closeable;
import java.io.IOException;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+
/**
- * @author yangli9
*/
public interface ReadableTable {
- public static final String DELIM_AUTO = "auto";
- public static final String DELIM_COMMA = ",";
-
+ /** Returns a reader to read the table. */
public TableReader getReader() throws IOException;
+ /** Used to detect table modifications mainly. Return null in case table does not exist. */
public TableSignature getSignature() throws IOException;
- public String getColumnDelimeter() throws IOException;
+ public interface TableReader extends Closeable {
+
+ /** Move to the next row, return false if no more record. */
+ public boolean next() throws IOException;
+
+ /** Get the current row. */
+ public String[] getRow();
+
+ }
+
+ // ============================================================================
+
+ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+ public class TableSignature {
+
+ @JsonProperty("path")
+ private String path;
+ @JsonProperty("size")
+ private long size;
+ @JsonProperty("last_modified_time")
+ private long lastModifiedTime;
+
+ // for JSON serialization
+ public TableSignature() {
+ }
+
+ public TableSignature(String path, long size, long lastModifiedTime) {
+ super();
+ this.path = path;
+ this.size = size;
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+
+ public void setLastModifiedTime(long lastModifiedTime) {
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public long getLastModifiedTime() {
+ return lastModifiedTime;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (lastModifiedTime ^ (lastModifiedTime >>> 32));
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
+ result = prime * result + (int) (size ^ (size >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ TableSignature other = (TableSignature) obj;
+ if (lastModifiedTime != other.lastModifiedTime)
+ return false;
+ if (path == null) {
+ if (other.path != null)
+ return false;
+ } else if (!path.equals(other.path))
+ return false;
+ if (size != other.size)
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "FileSignature [path=" + path + ", size=" + size + ", lastModifiedTime=" + lastModifiedTime + "]";
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index e8d0bbc..51d4094 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -24,9 +24,9 @@ import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
@@ -123,9 +123,7 @@ public class SnapshotManager {
TableSignature sig = snapshot.getSignature();
for (String existing : existings) {
SnapshotTable existingTable = load(existing, false); // skip cache,
- // direct
- // load from
- // store
+ // direct load from store
if (existingTable != null && sig.equals(existingTable.getSignature()))
return existing;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index fb1feb1..9f29446 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -46,8 +46,6 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
@JsonProperty("signature")
private TableSignature signature;
- @JsonProperty("column_delimeter")
- private String columnDelimeter;
@JsonProperty("useDictionary")
private boolean useDictionary;
@@ -60,13 +58,11 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
SnapshotTable(ReadableTable table) throws IOException {
this.signature = table.getSignature();
- this.columnDelimeter = table.getColumnDelimeter();
this.useDictionary = true;
}
public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
this.signature = table.getSignature();
- this.columnDelimeter = table.getColumnDelimeter();
int maxIndex = tableDesc.getMaxColumnIndex();
@@ -133,11 +129,6 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
@Override
public void close() throws IOException {
}
-
- @Override
- public void setExpectedColumnNumber(int expectedColumnNumber) {
- // noop
- }
};
}
@@ -146,11 +137,6 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
return signature;
}
- @Override
- public String getColumnDelimeter() throws IOException {
- return columnDelimeter;
- }
-
/**
* a naive implementation
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableReader.java
deleted file mode 100644
index e31ff96..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableReader.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Tables are typically CSV or SEQ file.
- *
- * @author yangli9
- */
-public interface TableReader extends Closeable {
-
- public boolean next() throws IOException;
-
- public String[] getRow();
-
- public void setExpectedColumnNumber(int expectedColumnNumber);
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableSignature.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableSignature.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableSignature.java
deleted file mode 100644
index b9ecc71..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/TableSignature.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class TableSignature {
-
- @JsonProperty("path")
- private String path;
- @JsonProperty("size")
- private long size;
- @JsonProperty("last_modified_time")
- private long lastModifiedTime;
-
- // for JSON serialization
- public TableSignature() {
- }
-
- public TableSignature(String path, long size, long lastModifiedTime) {
- super();
- this.path = path;
- this.size = size;
- this.lastModifiedTime = lastModifiedTime;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public void setSize(long size) {
- this.size = size;
- }
-
- public void setLastModifiedTime(long lastModifiedTime) {
- this.lastModifiedTime = lastModifiedTime;
- }
-
- public String getPath() {
- return path;
- }
-
- public long getSize() {
- return size;
- }
-
- public long getLastModifiedTime() {
- return lastModifiedTime;
- }
-
- // ============================================================================
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (lastModifiedTime ^ (lastModifiedTime >>> 32));
- result = prime * result + ((path == null) ? 0 : path.hashCode());
- result = prime * result + (int) (size ^ (size >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableSignature other = (TableSignature) obj;
- if (lastModifiedTime != other.lastModifiedTime)
- return false;
- if (path == null) {
- if (other.path != null)
- return false;
- } else if (!path.equals(other.path))
- return false;
- if (size != other.size)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "FileSignature [path=" + path + ", size=" + size + ", lastModifiedTime=" + lastModifiedTime + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
index 0ae8e95..5176038 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
@@ -20,18 +20,17 @@ package org.apache.kylin.dict;
import static org.junit.Assert.*;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.dict.lookup.HiveTable;
+import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.dict.lookup.TableReader;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-
/**
* @author yangli9
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
index 2650aad..cfecaee 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
@@ -24,11 +24,10 @@ import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.kylin.dict.lookup.FileTable;
import org.apache.kylin.dict.lookup.FileTableReader;
import org.junit.Test;
-import org.apache.kylin.dict.lookup.ReadableTable;
-
/**
* @author yangli9
*
@@ -38,7 +37,7 @@ public class TableReaderTest {
@Test
public void testBasicReader() throws IOException {
File f = new File("src/test/resources/dict/DW_SITES");
- FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), ReadableTable.DELIM_AUTO, 10);
+ FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), FileTable.DELIM_AUTO, 10);
while (reader.next()) {
assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow()));
break;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/024cc7d8/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 1150bc4..60451cd 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.job.hadoop.cube;
+import static org.junit.Assert.*;
+
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
@@ -27,12 +29,6 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -42,12 +38,15 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.lookup.TableSignature;
+import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
-
-import static org.junit.Assert.assertTrue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author honma
@@ -72,7 +71,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_common_dict");
- DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature, "");
+ DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature);
List<byte[]> values = new ArrayList<byte[]>();
values.add(new byte[] { 101, 101, 101 });
@@ -121,7 +120,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature);
List<byte[]> values = new ArrayList<byte[]>();
values.add(new byte[] { 97, 97, 97 });