You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/05 05:23:09 UTC
[21/67] [abbrv] kylin git commit: Revert "reformat code"
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java
index 8af633e..94b6e9d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java
@@ -17,8 +17,6 @@
*/
package org.apache.kylin.dict.global;
-import static org.apache.kylin.dict.global.GlobalDictHDFSStore.BUFFER_SIZE;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -30,6 +28,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
+import static org.apache.kylin.dict.global.GlobalDictHDFSStore.BUFFER_SIZE;
+
/**
* Created by sunyerui on 16/11/15.
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
index a5ba46c..b30d5b9 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
@@ -18,15 +18,6 @@
package org.apache.kylin.dict.global;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -42,6 +33,15 @@ import org.apache.kylin.dict.BytesConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
public class GlobalDictHDFSStore extends GlobalDictStore {
static final Logger logger = LoggerFactory.getLogger(GlobalDictHDFSStore.class);
@@ -241,12 +241,9 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
@Override
public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
- checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()),
- "Please check why current directory {} doesn't belong to source working directory {}", baseDir,
- srcConfig.getHdfsWorkingDirectory());
+ checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), "Please check why current directory {} doesn't belong to source working directory {}", baseDir, srcConfig.getHdfsWorkingDirectory());
- final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(),
- dstConfig.getHdfsWorkingDirectory());
+ final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory());
Long[] versions = listAllVersions();
if (versions.length == 0) { // empty dict, nothing to copy
@@ -254,8 +251,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
}
Path srcVersionDir = getVersionDir(versions[versions.length - 1]);
- Path dstVersionDir = new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(),
- dstConfig.getHdfsWorkingDirectory()));
+ Path dstVersionDir = new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()));
FileSystem dstFS = dstVersionDir.getFileSystem(conf);
if (dstFS.exists(dstVersionDir)) {
dstFS.delete(dstVersionDir, true);
@@ -405,8 +401,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
public void sanityCheck(Path dir, GlobalDictMetadata metadata) throws IOException {
for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) {
if (!fs.exists(new Path(dir, entry.getValue()))) {
- throw new RuntimeException("The slice file " + entry.getValue() + " for the key: " + entry.getKey()
- + " must be existed!");
+ throw new RuntimeException("The slice file " + entry.getValue() + " for the key: " + entry.getKey() + " must be existed!");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
index d35380c..7c89ea2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
@@ -18,12 +18,11 @@
package org.apache.kylin.dict.global;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
+import com.google.common.base.Preconditions;
import org.apache.kylin.dict.BytesConverter;
-import com.google.common.base.Preconditions;
+import java.util.NavigableMap;
+import java.util.TreeMap;
/**
* Encapsulates the metadata for a particular version of the global dictionary.
@@ -37,8 +36,7 @@ public class GlobalDictMetadata {
public final BytesConverter bytesConverter;
public final TreeMap<AppendDictSliceKey, String> sliceFileMap; // slice key -> slice file name
- public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter,
- NavigableMap<AppendDictSliceKey, String> sliceFileMap) {
+ public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, NavigableMap<AppendDictSliceKey, String> sliceFileMap) {
Preconditions.checkNotNull(bytesConverter, "bytesConverter");
Preconditions.checkNotNull(sliceFileMap, "sliceFileMap");
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
index c370afc..eaf0729 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
@@ -18,12 +18,11 @@
package org.apache.kylin.dict.global;
-import java.io.IOException;
-
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
public abstract class GlobalDictStore {
@@ -74,8 +73,7 @@ public abstract class GlobalDictStore {
* @return file name of the new written slice
* @throws IOException on I/O error
*/
- public abstract String writeSlice(String workingDir, AppendDictSliceKey key, AppendDictNode slice)
- throws IOException;
+ public abstract String writeSlice(String workingDir, AppendDictSliceKey key, AppendDictNode slice) throws IOException;
/**
* Delete a slice with the specified file name.
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index a22878f..a99ef29 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -83,8 +83,7 @@ abstract public class LookupTable<T> {
Array<T> key = new Array<T>(keyCols);
if (data.containsKey(key))
- throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key="
- + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));
+ throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));
data.put(key, value);
}
@@ -148,8 +147,7 @@ abstract public class LookupTable<T> {
}
private boolean between(T beginValue, T v, T endValue, Comparator<T> comp) {
- return (beginValue == null || comp.compare(beginValue, v) <= 0)
- && (endValue == null || comp.compare(v, endValue) <= 0);
+ return (beginValue == null || comp.compare(beginValue, v) <= 0) && (endValue == null || comp.compare(v, endValue) <= 0);
}
abstract protected Comparator<T> getComparator(int colIdx);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
index 07659a8..45fd56b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
@@ -41,8 +41,7 @@ public class SnapshotCLI {
if (tableDesc == null)
throw new IllegalArgumentException("Not table found by " + table);
- SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceFactory.createReadableTable(tableDesc), tableDesc,
- overwriteUUID);
+ SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceFactory.createReadableTable(tableDesc), tableDesc, overwriteUUID);
System.out.println("resource path updated: " + snapshot.getResourcePath());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index d7c1b68..b997a88 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -86,8 +86,7 @@ public class SnapshotManager {
this.snapshotCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, SnapshotTable>() {
@Override
public void onRemoval(RemovalNotification<String, SnapshotTable> notification) {
- SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey()
- + " is removed due to " + notification.getCause());
+ SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey() + " is removed due to " + notification.getCause());
}
}).maximumSize(config.getCachedSnapshotMaxEntrySize())//
.expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, SnapshotTable>() {
@@ -142,8 +141,7 @@ public class SnapshotManager {
return trySaveNewSnapshot(snapshot);
}
- public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID)
- throws IOException {
+ public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException {
SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity());
snapshot.setUuid(overwriteUUID);
@@ -216,8 +214,7 @@ public class SnapshotManager {
logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData);
ResourceStore store = MetadataManager.getInstance(this.config).getStore();
- SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class,
- loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);
+ SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);
if (loadData)
logger.debug("Loaded snapshot at " + resourcePath);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index ed0cb7a..1d7e474 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import com.google.common.base.Strings;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -42,7 +43,6 @@ import org.apache.kylin.source.IReadableTable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Strings;
/**
* @author yangli9
@@ -83,8 +83,7 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl
while (reader.next()) {
String[] row = reader.getRow();
if (row.length <= maxIndex) {
- throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1)
- + " columns, but got " + Arrays.toString(row));
+ throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row));
}
for (ColumnDesc column : tableDesc.getColumns()) {
String cell = row[column.getZeroBasedIndex()];
@@ -169,7 +168,7 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl
public boolean exists() throws IOException {
return true;
}
-
+
/**
* a naive implementation
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
index cd86936..4820318 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
@@ -61,14 +61,14 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
assertEquals(lookupDate, dictMgr.decideSourceData(innerModel, lookupDate));
assertEquals(formatName, dictMgr.decideSourceData(innerModel, formatName));
}
-
+
{
DataModelDesc outerModel = metaMgr.getDataModelDesc("test_kylin_left_join_model_desc");
TblColRef factDate = outerModel.findColumn("TEST_KYLIN_FACT.CAL_DT");
assertEquals(factDate, dictMgr.decideSourceData(outerModel, factDate));
}
}
-
+
@Test
public void testBuildSaveDictionary() throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -80,32 +80,25 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
// non-exist input returns null;
DictionaryInfo nullInfo = dictMgr.buildDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"));
assertEquals(null, nullInfo);
-
- DictionaryInfo info1 = dictMgr.buildDictionary(model, col,
- MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
+
+ DictionaryInfo info1 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
assertEquals(3, info1.getDictionaryObject().getSize());
// same input returns same dict
- DictionaryInfo info2 = dictMgr.buildDictionary(model, col,
- MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
+ DictionaryInfo info2 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
assertTrue(info1 == info2);
-
+
// same input values (different path) returns same dict
- DictionaryInfo info3 = dictMgr.buildDictionary(model, col,
- MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3"));
+ DictionaryInfo info3 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3"));
assertTrue(info1 == info3);
-
+
// save dictionary works in spite of non-exist table
- Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(),
- new IterableDictionaryValueEnumerator("1", "2", "3"));
- DictionaryInfo info4 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"),
- dict);
+ Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3"));
+ DictionaryInfo info4 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict);
assertTrue(info1 == info4);
-
- Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(),
- new IterableDictionaryValueEnumerator("1", "2", "3", "4"));
- DictionaryInfo info5 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"),
- dict2);
+
+ Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3", "4"));
+ DictionaryInfo info5 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict2);
assertTrue(info1 != info5);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
index 794f334..4b386a7 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -37,7 +37,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-public class DictionaryProviderTest extends LocalFileMetadataTestCase {
+public class DictionaryProviderTest extends LocalFileMetadataTestCase{
@Before
public void setUp() throws Exception {
@@ -52,28 +52,23 @@ public class DictionaryProviderTest extends LocalFileMetadataTestCase {
@Test
public void testReadWrite() throws Exception {
//string dict
- Dictionary<String> dict = getDict(DataType.getType("string"),
- Arrays.asList(new String[] { "a", "b" }).iterator());
+ Dictionary<String> dict = getDict(DataType.getType("string"), Arrays.asList(new String[] { "a", "b" }).iterator());
readWriteTest(dict);
//number dict
- Dictionary<String> dict2 = getDict(DataType.getType("long"),
- Arrays.asList(new String[] { "1", "2" }).iterator());
+ Dictionary<String> dict2 = getDict(DataType.getType("long"), Arrays.asList(new String[] { "1", "2" }).iterator());
readWriteTest(dict2);
//date dict
- Dictionary<String> dict3 = getDict(DataType.getType("datetime"),
- Arrays.asList(new String[] { "20161122", "20161123" }).iterator());
+ Dictionary<String> dict3 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "20161122", "20161123" }).iterator());
readWriteTest(dict3);
//date dict
- Dictionary<String> dict4 = getDict(DataType.getType("datetime"),
- Arrays.asList(new String[] { "2016-11-22", "2016-11-23" }).iterator());
+ Dictionary<String> dict4 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "2016-11-22", "2016-11-23" }).iterator());
readWriteTest(dict4);
//date dict
try {
- Dictionary<String> dict5 = getDict(DataType.getType("date"),
- Arrays.asList(new String[] { "2016-11-22", "20161122" }).iterator());
+ Dictionary<String> dict5 = getDict(DataType.getType("date"), Arrays.asList(new String[] { "2016-11-22", "20161122" }).iterator());
readWriteTest(dict5);
fail("Date format not correct.Should throw exception");
} catch (IllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
index a9d2398..53c9476 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
@@ -34,7 +34,7 @@ public class MockupReadableTable implements IReadableTable {
}
return new MockupReadableTable(content, sig, true);
}
-
+
public static IReadableTable newNonExistTable(String path) {
TableSignature sig = new TableSignature(path, -1, 0);
return new MockupReadableTable(null, sig, false);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index d25f2f4..2e90bcf 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -56,52 +56,52 @@ public class MultipleDictionaryValueEnumeratorTest {
@Test
public void testNormalDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
- dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
- dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 }));
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[] { 0, 1, 2, 4, 5, 6 }, values);
+ assertArrayEquals(new Integer[]{0, 1, 2, 4, 5, 6}, values);
}
@Test
public void testFirstEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
- dictionaryInfoList.add(createDictInfo(new int[] {}));
- dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 }));
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
+ dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(3, values.length);
- assertArrayEquals(new Integer[] { 4, 5, 6 }, values);
+ assertArrayEquals(new Integer[]{4, 5, 6}, values);
}
@Test
public void testMiddleEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
- dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
- dictionaryInfoList.add(createDictInfo(new int[] {}));
- dictionaryInfoList.add(createDictInfo(new int[] { 7, 8, 9 }));
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
+ dictionaryInfoList.add(createDictInfo(new int[]{7, 8, 9}));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[] { 0, 1, 2, 7, 8, 9 }, values);
+ assertArrayEquals(new Integer[]{0, 1, 2, 7, 8, 9}, values);
}
@Test
public void testLastEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
- dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
- dictionaryInfoList.add(createDictInfo(new int[] { 6, 7, 8 }));
- dictionaryInfoList.add(createDictInfo(new int[] {}));
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{6, 7, 8}));
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[] { 0, 1, 2, 6, 7, 8 }, values);
+ assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values);
}
public static class MockDictionary extends Dictionary<String> {
private static final long serialVersionUID = 1L;
-
+
public int[] values;
@Override
@@ -111,7 +111,7 @@ public class MultipleDictionaryValueEnumeratorTest {
@Override
public int getMaxId() {
- return values.length - 1;
+ return values.length-1;
}
@Override
@@ -134,17 +134,16 @@ public class MultipleDictionaryValueEnumeratorTest {
return "" + values[id];
}
+
@Override
- public void dump(PrintStream out) {
- }
+ public void dump(PrintStream out) {}
@Override
- public void write(DataOutput out) throws IOException {
- }
+ public void write(DataOutput out) throws IOException {}
@Override
- public void readFields(DataInput in) throws IOException {
- }
+ public void readFields(DataInput in) throws IOException {}
+
@Override
public boolean contains(Dictionary another) {
@@ -152,4 +151,4 @@ public class MultipleDictionaryValueEnumeratorTest {
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 9bdb174..8da7208 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -45,8 +45,7 @@ import com.google.common.collect.Sets;
*/
public class NumberDictionaryTest extends LocalFileMetadataTestCase {
- Number2BytesConverter.NumberBytesCodec codec = new Number2BytesConverter.NumberBytesCodec(
- MAX_DIGITS_BEFORE_DECIMAL_POINT);
+ Number2BytesConverter.NumberBytesCodec codec = new Number2BytesConverter.NumberBytesCodec(MAX_DIGITS_BEFORE_DECIMAL_POINT);
Random rand = new Random();
@Before
@@ -79,8 +78,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
String[] ints = new String[] { "", "0", "5", "100", "13" };
// check "" is treated as NULL, not a code of dictionary
- Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType("integer"),
- new IterableDictionaryValueEnumerator(ints));
+ Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType("integer"), new IterableDictionaryValueEnumerator(ints));
assertEquals(4, dict.getSize());
final int id = ((NumberDictionary<String>) dict).getIdFromValue("");
@@ -97,8 +95,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
//test resolved jira-1800
checkCodec("-0.0045454354354354359999999999877218", "-9999999999999999999.9954545645645645640000000000122781;");
checkCodec("-0.009999999999877218", "-9999999999999999999.990000000000122781;");
- checkCodec("12343434372493274.438403840384023840253554345345345345",
- "00012343434372493274.438403840384023840253554345345345345");
+ checkCodec("12343434372493274.438403840384023840253554345345345345", "00012343434372493274.438403840384023840253554345345345345");
assertEquals("00000000000000000052.57", encodeNumber("52.5700"));
assertEquals("00000000000000000000", encodeNumber("0.00"));
assertEquals("00000000000000000000", encodeNumber("0.0"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
index e1eb119..adc1074 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
@@ -18,16 +18,16 @@
package org.apache.kylin.dict;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Random;
-import java.util.UUID;
-
import org.apache.kylin.common.util.Dictionary;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Random;
+import java.util.UUID;
+
/**
* Created by xiefan on 16-12-28.
*/
@@ -50,8 +50,7 @@ public class TrieDictionaryForestBenchmark {
public void before() {
int dataSize = 100 * 10000;
TrieDictionaryBuilder<String> b1 = new TrieDictionaryBuilder<>(new StringBytesConverter());
- TrieDictionaryForestBuilder<String> b2 = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0,
- 5);
+ TrieDictionaryForestBuilder<String> b2 = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0, 5);
this.rawData = genStringDataSet(dataSize);
for (String str : this.rawData) {
b1.addValue(str);
@@ -153,8 +152,7 @@ public class TrieDictionaryForestBenchmark {
return System.currentTimeMillis() - startTime;
}
- private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality,
- int testTimes) {
+ private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, int testTimes) {
long startTime = System.currentTimeMillis();
int step = 1;
for (int i = 0; i < testTimes; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index a8c9a67..82380b3 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -186,14 +186,8 @@ public class TrieDictionaryForestTest {
str.add("party");
str.add("parties");
str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+ String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
System.out.println("The length of the long string is " + longStr.length());
str.add(longStr);
@@ -745,8 +739,7 @@ public class TrieDictionaryForestTest {
System.out.println("times:" + i);
}
- System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : "
- + newDictTotalBuildTime / 1000.0 + "s");
+ System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 1000.0 + "s");
}
private void evaluateDataSize(ArrayList<String> list) {
@@ -812,8 +805,7 @@ public class TrieDictionaryForestTest {
benchmark("Benchmark", dict, set, map, strArray, array);
}
- private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set,
- HashMap<String, Integer> map, String[] strArray, byte[][] array) {
+ private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
@@ -937,16 +929,14 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
- baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
for (String s : strs)
b.addValue(s);
return b;
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
- baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
b.setMaxTrieTreeSize(treeSize);
for (String s : strs) {
b.addValue(s);
@@ -955,8 +945,7 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterator<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
- baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
b.setMaxTrieTreeSize(treeSize);
while (strs.hasNext())
b.addValue(strs.next());
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index af4c9cd..13c83ac 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -111,14 +111,8 @@ public class TrieDictionaryTest {
str.add("party");
str.add("parties");
str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+ String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
System.out.println("The length of the long string is " + longStr.length());
str.add(longStr);
@@ -275,8 +269,7 @@ public class TrieDictionaryTest {
benchmark("Benchmark", dict, set, map, strArray, array);
}
- private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set,
- HashMap<String, Integer> map, String[] strArray, byte[][] array) {
+ private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
@@ -412,12 +405,10 @@ public class TrieDictionaryTest {
@Test
public void testSuperLongStringValue() {
- String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
- + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
+ String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- String v1 = longPrefix
- + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
+ String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
String v2 = longPrefix + "xyz";
b.addValue(v1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
index 1671748..47011fe 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
@@ -66,8 +66,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
public void beforeTest() {
staticCreateTestMetadata();
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000");
- BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR
- + "/";
+ BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
}
@@ -85,19 +84,11 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
}
}
- private static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions",
- "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "字典", "字典树", "字母", // non-ascii characters
+ private static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "字典", "字典树", "字母", // non-ascii characters
"", // empty
- "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
- "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
- "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk",
+ "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
+ "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk",
"paint", "tar", "try", // some dup
};
@@ -177,8 +168,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
dict.dump(System.out);
}
- private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList)
- throws IOException {
+ private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList) throws IOException {
Random rnd = new Random(System.currentTimeMillis());
ArrayList<String> strList = new ArrayList<String>();
strList.addAll(list);
@@ -205,8 +195,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
byte[] bytes = converter.convertToBytes(str);
int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)),
- checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
}
@@ -230,8 +219,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
} else {
// check second append str, should be new id
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)),
- checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
}
}
@@ -256,8 +244,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
} else {
// check third append str, should be new id
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)),
- checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
}
}
@@ -278,8 +265,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
}
}
- private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict,
- BytesConverter converter) {
+ private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict, BytesConverter converter) {
try {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dataout = new DataOutputStream(bout);
@@ -526,8 +512,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME);
fs.delete(v2IndexFile, true);
- GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs,
- HadoopUtil.getCurrentConfiguration());
+ GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, HadoopUtil.getCurrentConfiguration());
indexFormatV1.writeIndexFile(versionPath, metadata);
//convert v2 fileName format to v1 fileName format
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index b798b86..5a5c988 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -72,8 +72,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
@Test
public void testMapRange() throws Exception {
- Pair<String, String> results = lookupTable.mapRange("CAL_DT", millis("2012-01-24"), millis("2012-12-30"),
- "QTR_BEG_DT");
+ Pair<String, String> results = lookupTable.mapRange("CAL_DT", millis("2012-01-24"), millis("2012-12-30"), "QTR_BEG_DT");
Assert.assertTrue(results != null);
System.out.println("The first qtr_beg_dt is " + results.getFirst());
@@ -85,8 +84,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
@Test
public void testMapRange2() throws Exception {
- Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"),
- "CAL_DT");
+ Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"), "CAL_DT");
System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getFirst())));
System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getSecond())));
@@ -111,7 +109,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
}
@Test
- public void testGetClassName() {
+ public void testGetClassName(){
String name = TrieDictionaryForest.class.getName();
System.out.println(name);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
index 302af14..acaa7da 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
@@ -45,11 +45,11 @@ public class EngineFactory {
public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
return batchEngine(cubeDesc).getJoinedFlatTableDesc(cubeDesc);
}
-
+
public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
return batchEngine(newSegment).getJoinedFlatTableDesc(newSegment);
}
-
+
/** Build a new cube segment, typically its time range appends to the end of current cube. */
public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
index 3abdbc2..754dbde 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
@@ -24,11 +24,11 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public interface IBatchCubingEngine {
-
+
/** Mark deprecated to indicate for test purpose only */
@Deprecated
public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc);
-
+
public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment);
/** Build a new cube segment, typically its time range appends to the end of current cube. */
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
index b45e3e0..8dcdff6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
@@ -70,8 +70,7 @@ public class JobInstance extends RootPersistentEntity implements Comparable<JobI
public JobStep getRunningStep() {
for (JobStep step : this.getSteps()) {
- if (step.getStatus().equals(JobStepStatusEnum.RUNNING)
- || step.getStatus().equals(JobStepStatusEnum.WAITING)) {
+ if (step.getStatus().equals(JobStepStatusEnum.RUNNING) || step.getStatus().equals(JobStepStatusEnum.WAITING)) {
return step;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 3f82f16..4665465 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -87,21 +87,18 @@ public class JoinedFlatTable {
if (kylinConfig.isAdvancedFlatTableUsed()) {
try {
Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
- Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class,
- JobEngineConfig.class);
+ Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class);
return (String) method.invoke(null, flatDesc);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
- return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc)
- + ";\n";
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n";
}
public static String generateInsertPartialDataStatement(IJoinedFlatTableDesc flatDesc, String statement) {
- return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc)
- + statement + ";\n";
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + statement + ";\n";
}
public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
@@ -123,8 +120,7 @@ public class JoinedFlatTable {
final StringBuilder sql = new StringBuilder();
final TableRef rootTbl = flatDesc.getDataModel().getRootFactTable();
sql.append("dfs -mkdir -p " + outputDir + ";\n");
- sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + rootTbl.getTableIdentity()
- + " " + rootTbl.getAlias() + "\n");
+ sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + rootTbl.getTableIdentity() + " " + rootTbl.getAlias() + "\n");
appendWhereStatement(flatDesc, sql);
return sql.toString();
}
@@ -195,8 +191,7 @@ public class JoinedFlatTable {
if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
whereBuilder.append(hasCondition ? " AND (" : " (");
- whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
- dateStart, dateEnd));
+ whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd));
whereBuilder.append(")\n");
hasCondition = true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
index 3b007f6..8be5d02 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
@@ -37,23 +37,21 @@ public class PatternedLogger extends BufferedLogger {
private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
- private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern
- .compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
+ private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
// hive
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
- private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern
- .compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
- private static final Pattern PATTERN_HIVE_APP_ID_URL_2 = Pattern
- .compile("Executing on YARN cluster with App id (.*?)");
+ private static final Pattern PATTERN_HIVE_APP_ID_URL_2 = Pattern.compile("Executing on YARN cluster with App id (.*?)");
// spark
private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
+
public PatternedLogger(Logger wrappedLogger) {
super(wrappedLogger);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 2b69d8f..9f431b0 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -19,7 +19,6 @@
package org.apache.kylin.job.common;
import java.io.IOException;
-
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -44,11 +43,9 @@ public class ShellExecutable extends AbstractExecutable {
try {
logger.info("executing:" + getCmd());
final PatternedLogger patternedLogger = new PatternedLogger(logger);
- final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(),
- patternedLogger);
+ final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), patternedLogger);
getManager().addJobInfo(getId(), patternedLogger.getInfo());
- return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED,
- result.getSecond());
+ return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED, result.getSecond());
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 6f30d06..d7f6292 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -57,11 +57,5 @@ public final class ExecutableConstants {
public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage Collection on HBase";
public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table";
- public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>"
- + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>"
- + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>"
- + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>"
- + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>"
- + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>"
- + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
+ public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
index 8a9693d..4c6ac97 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
@@ -43,8 +43,7 @@ public enum JobStatusEnum {
}
public boolean isComplete() {
- return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode()
- || code == JobStatusEnum.DISCARDED.getCode();
+ return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode() || code == JobStatusEnum.DISCARDED.getCode();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
index d83b322..08cd138 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
@@ -42,8 +42,7 @@ public enum JobStepStatusEnum {
}
public boolean isComplete() {
- return code == JobStepStatusEnum.FINISHED.getCode() || code == JobStepStatusEnum.ERROR.getCode()
- || code == JobStepStatusEnum.DISCARDED.getCode();
+ return code == JobStepStatusEnum.FINISHED.getCode() || code == JobStepStatusEnum.ERROR.getCode() || code == JobStepStatusEnum.DISCARDED.getCode();
}
public boolean isRunable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index b395ff8..70799d8 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -42,8 +42,7 @@ import com.google.common.collect.Lists;
public class ExecutableDao {
private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<ExecutablePO>(ExecutablePO.class);
- private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>(
- ExecutableOutputPO.class);
+ private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>(ExecutableOutputPO.class);
private static final Logger logger = LoggerFactory.getLogger(ExecutableDao.class);
private static final ConcurrentMap<KylinConfig, ExecutableDao> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableDao>();
@@ -107,8 +106,7 @@ public class ExecutableDao {
public List<ExecutableOutputPO> getJobOutputs() throws PersistentException {
try {
- return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, ExecutableOutputPO.class,
- JOB_OUTPUT_SERIALIZER);
+ return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
throw new PersistentException(e);
@@ -117,8 +115,7 @@ public class ExecutableDao {
public List<ExecutableOutputPO> getJobOutputs(long timeStart, long timeEndExclusive) throws PersistentException {
try {
- return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, timeStart, timeEndExclusive,
- ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
+ return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, timeStart, timeEndExclusive, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
throw new PersistentException(e);
@@ -136,8 +133,7 @@ public class ExecutableDao {
public List<ExecutablePO> getJobs(long timeStart, long timeEndExclusive) throws PersistentException {
try {
- return store.getAllResources(ResourceStore.EXECUTE_RESOURCE_ROOT, timeStart, timeEndExclusive,
- ExecutablePO.class, JOB_SERIALIZER);
+ return store.getAllResources(ResourceStore.EXECUTE_RESOURCE_ROOT, timeStart, timeEndExclusive, ExecutablePO.class, JOB_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
throw new PersistentException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 43789d6..c9ac583 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -59,8 +59,7 @@ public class JobEngineConfig {
File jobConfig = getJobConfig(hadoopJobConfFile);
if (jobConfig == null || !jobConfig.exists()) {
- logger.warn(
- "fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
+ logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml");
if (jobConfig == null || !jobConfig.exists()) {
logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java b/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
index 67ff03f..f19b0ca 100644
--- a/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
+++ b/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
@@ -39,8 +39,7 @@ public class IllegalStateTranferException extends RuntimeException {
super(cause);
}
- public IllegalStateTranferException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
+ public IllegalStateTranferException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 63a36b5..1f1be41 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -60,16 +60,16 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
public AbstractExecutable() {
setId(UUID.randomUUID().toString());
}
-
+
protected void initConfig(KylinConfig config) {
Preconditions.checkState(this.config == null || this.config == config);
this.config = config;
}
-
+
protected KylinConfig getConfig() {
return config;
}
-
+
protected ExecutableManager getManager() {
return ExecutableManager.getInstance(config);
}
@@ -391,7 +391,6 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus())
- .toString();
+ return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()).toString();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 817565b..8bcaaad 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -43,7 +43,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
sub.initConfig(config);
}
}
-
+
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
List<? extends Executable> executables = getTasks();
@@ -58,8 +58,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
// the job is paused
break;
} else if (state == ExecutableState.ERROR) {
- throw new IllegalStateException(
- "invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus());
+ throw new IllegalStateException("invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus());
}
if (subTask.isRunnable()) {
return subTask.execute(context);
@@ -89,7 +88,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai
@Override
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
ExecutableManager mgr = getManager();
-
+
if (isDiscarded()) {
setEndTime(System.currentTimeMillis());
notifyUserStatusChange(executableContext, ExecutableState.DISCARDED);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 4dc22b7..2272582 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -18,10 +18,6 @@
package org.apache.kylin.job.execution;
-import static org.apache.kylin.job.constant.ExecutableConstants.MR_JOB_ID;
-import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID;
-import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL;
-
import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.IllegalFormatException;
@@ -46,6 +42,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import static org.apache.kylin.job.constant.ExecutableConstants.MR_JOB_ID;
+import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID;
+import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL;
+
/**
*/
public class ExecutableManager {
@@ -235,8 +235,7 @@ public class ExecutableManager {
* @param expectedClass
* @return
*/
- public List<AbstractExecutable> getAllAbstractExecutables(long timeStartInMillis, long timeEndInMillis,
- Class<? extends AbstractExecutable> expectedClass) {
+ public List<AbstractExecutable> getAllAbstractExecutables(long timeStartInMillis, long timeEndInMillis, Class<? extends AbstractExecutable> expectedClass) {
try {
List<AbstractExecutable> ret = Lists.newArrayList();
for (ExecutablePO po : executableDao.getJobs(timeStartInMillis, timeEndInMillis)) {
@@ -383,8 +382,7 @@ public class ExecutableManager {
ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus());
if (newStatus != null && oldStatus != newStatus) {
if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) {
- throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:"
- + newStatus + ", job id: " + jobId);
+ throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus + ", job id: " + jobId);
}
jobOutput.setStatus(newStatus.toString());
}
@@ -484,8 +482,7 @@ public class ExecutableManager {
}
}
- private AbstractExecutable parseToAbstract(ExecutablePO executablePO,
- Class<? extends AbstractExecutable> expectedClass) {
+ private AbstractExecutable parseToAbstract(ExecutablePO executablePO, Class<? extends AbstractExecutable> expectedClass) {
if (executablePO == null) {
logger.warn("executablePO is null");
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
index 0d3258e..910bd7e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
@@ -36,14 +36,12 @@ public enum ExecutableState {
private static Multimap<ExecutableState, ExecutableState> VALID_STATE_TRANSFER;
static {
- VALID_STATE_TRANSFER = Multimaps.newSetMultimap(
- Maps.<ExecutableState, Collection<ExecutableState>> newEnumMap(ExecutableState.class),
- new Supplier<Set<ExecutableState>>() {
- @Override
- public Set<ExecutableState> get() {
- return new CopyOnWriteArraySet<ExecutableState>();
- }
- });
+ VALID_STATE_TRANSFER = Multimaps.newSetMultimap(Maps.<ExecutableState, Collection<ExecutableState>> newEnumMap(ExecutableState.class), new Supplier<Set<ExecutableState>>() {
+ @Override
+ public Set<ExecutableState> get() {
+ return new CopyOnWriteArraySet<ExecutableState>();
+ }
+ });
//scheduler
VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.RUNNING);
@@ -66,9 +64,11 @@ public enum ExecutableState {
VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.DISCARDED);
VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.READY);
+
VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.STOPPED);
VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.STOPPED);
+
//rollback
VALID_STATE_TRANSFER.put(ExecutableState.SUCCEED, ExecutableState.READY);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 128fa35..8b6b5aa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -121,9 +121,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
logger.warn(jobDesc + " fail to schedule", ex);
}
}
- logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
- + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
- + " error, " + nDiscarded + " discarded, " + nOthers + " others");
+ logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, " + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError + " error, " + nDiscarded + " discarded, " + nOthers + " others");
} catch (Exception e) {
logger.warn("Job Fetcher caught a exception " + e);
}
@@ -191,7 +189,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
jobLock = lock;
-
+
String serverMode = jobEngineConfig.getConfig().getServerMode();
if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) {
logger.info("server mode: " + serverMode + ", no need to run job scheduler");
@@ -215,15 +213,13 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
//load all executable, set them to a consistent status
fetcherPool = Executors.newScheduledThreadPool(1);
int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
- jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
- new SynchronousQueue<Runnable>());
+ jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>());
context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());
executableManager.resumeAllRunningJobs();
fetcher = new FetcherRunner();
- fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS,
- TimeUnit.SECONDS);
+ fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
hasStarted = true;
}