You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/06/25 11:12:03 UTC
[02/18] kylin git commit: KYLIN-1821 Reformat all of the java files
and enable checkstyle to enforce code formatting
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 9150a48..d1c31de 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -296,7 +296,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
if (hfileSizeMB > 0.0 && kylinConfig.isDevEnv()) {
hfileSizeMB = mbPerRegion / 2;
}
-
+
int compactionThreshold = Integer.valueOf(hbaseConf.get("hbase.hstore.compactionThreshold", "3"));
logger.info("hbase.hstore.compactionThreshold is " + compactionThreshold);
if (hfileSizeMB > 0.0 && hfileSizeMB * compactionThreshold < mbPerRegion) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 19f37ec..e4a9c1e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -131,7 +131,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
}
TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), path);
// The reduce tasks should be one more than partition keys
- job.setNumReduceTasks(partitionCount+1);
+ job.setNumReduceTasks(partitionCount + 1);
}
} else {
logger.info("File '" + path.toString() + " doesn't exist, will not reconfigure hfile Partitions");
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index c62e1c3..fe65598 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -122,7 +122,7 @@ public class CubeHTableUtil {
admin.close();
}
}
-
+
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
@@ -137,7 +137,7 @@ public class CubeHTableUtil {
HTableDescriptor tableDesc = new HTableDescriptor(tableName);
tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
-
+
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false));
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 472b638..7aecd7e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -27,7 +27,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index ddc868d..4fe7748 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -33,7 +33,9 @@
*/
package org.apache.kylin.storage.hbase.steps;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -50,8 +52,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
*/
@@ -127,15 +128,15 @@ public class HBaseCuboidWriter implements ICuboidWriter {
@Override
public final void flush() throws IOException {
- if (!puts.isEmpty()) {
- long t = System.currentTimeMillis();
- if (hTable != null) {
- hTable.put(puts);
- hTable.flushCommits();
- }
- logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
- puts.clear();
+ if (!puts.isEmpty()) {
+ long t = System.currentTimeMillis();
+ if (hTable != null) {
+ hTable.put(puts);
+ hTable.flushCommits();
}
+ logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
+ puts.clear();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index fa62a62..9adaf24 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -22,7 +22,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -30,7 +29,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.inmemcubing.CompoundCuboidWriter;
@@ -39,11 +37,14 @@ import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
import org.apache.kylin.engine.streaming.IStreamingOutput;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
*/
public class HBaseStreamingOutput implements IStreamingOutput {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
index ff828b0..fbfd582 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
@@ -33,11 +33,11 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.storage.hbase.HBaseConnection;
-
-import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* Created by sunyerui on 15/9/17.
*/
@@ -45,7 +45,6 @@ public class HDFSPathGarbageCollectionStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(HDFSPathGarbageCollectionStep.class);
-
public static final String TO_DELETE_PATHS = "toDeletePaths";
private StringBuffer output;
private JobEngineConfig config;
@@ -61,7 +60,7 @@ public class HDFSPathGarbageCollectionStep extends AbstractExecutable {
config = new JobEngineConfig(context.getConfig());
List<String> toDeletePaths = getDeletePaths();
dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HadoopUtil.getCurrentConfiguration()));
-
+
if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) {
dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration()));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index a1377f2..5b2441c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
@@ -63,9 +62,9 @@ public class MergeGCStep extends AbstractExecutable {
} catch (InterruptedException e) {
logger.warn("Thread interrupted");
}
-
+
logger.info("Start doing merge gc work");
-
+
StringBuffer output = new StringBuffer();
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
index ee3b19c..2876e3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
@@ -35,7 +35,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,7 +96,6 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
this.deletePath(job.getConfiguration(), output);
-
float hfileSizeGB = kylinConfig.getHBaseHFileSizeGB();
float regionSplitSize = kylinConfig.getKylinHBaseRegionCut();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
index d58f8e8..c2190fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
@@ -38,7 +38,6 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text, Text, Lo
private Text lastKey;
-
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
index f9dfa49..a4b7956 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
@@ -77,8 +77,7 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable
maxRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.CFG_REGION_NUMBER_MAX));
}
- logger.info("Chosen cut for htable is " + cut + ", max region count=" + maxRegionCount
- + ", min region count=" + minRegionCount + ", hfile size=" + hfileSizeGB);
+ logger.info("Chosen cut for htable is " + cut + ", max region count=" + maxRegionCount + ", min region count=" + minRegionCount + ", hfile size=" + hfileSizeGB);
// add empty key at position 0
gbPoints.add(new Text());
@@ -116,9 +115,7 @@ public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable
System.out.println(hfilePerRegion + " hfile per region");
Path hfilePartitionFile = new Path(output + "/part-r-00000_hfile");
- SequenceFile.Writer hfilePartitionWriter = new SequenceFile.Writer(
- hfilePartitionFile.getFileSystem(context.getConfiguration()),
- context.getConfiguration(), hfilePartitionFile, ImmutableBytesWritable.class, NullWritable.class);
+ SequenceFile.Writer hfilePartitionWriter = new SequenceFile.Writer(hfilePartitionFile.getFileSystem(context.getConfiguration()), context.getConfiguration(), hfilePartitionFile, ImmutableBytesWritable.class, NullWritable.class);
int hfileCountInOneRegion = 0;
for (int i = hfileSizeGB; i < gbPoints.size(); i += hfileSizeGB) {
hfilePartitionWriter.append(new ImmutableBytesWritable(gbPoints.get(i).getBytes()), NullWritable.get());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index e1e8f8c..2aa285c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -121,7 +121,7 @@ public class RowValueDecoder implements Cloneable {
}
// result is in order of <code>CubeDesc.getMeasures()</code>
- public void loadCubeMeasureArray(Object result[]) {
+ public void loadCubeMeasureArray(Object[] result) {
int[] measureIndex = hbaseColumn.getMeasureIndex();
for (int i = 0; i < measureIndex.length; i++) {
result[measureIndex[i]] = values[i];
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 801f8e4..8f2fc80 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -18,6 +18,8 @@
package org.apache.kylin.storage.hbase.steps;
+import java.io.IOException;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,8 +34,6 @@ import org.apache.kylin.engine.mr.steps.KVGTRecordWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
*/
public class SequenceFileCuboidWriter extends KVGTRecordWriter {
@@ -46,7 +46,7 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
try {
initiate();
} catch (IOException e) {
- throw new RuntimeException(e);
+ throw new RuntimeException(e);
}
}
@@ -75,7 +75,7 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
@Override
protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException {
-
+
Text outputValue = new Text();
Text outputKey = new Text();
outputKey.set(key.array(), key.offset(), key.length());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 1ba96e9..9e30a06 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.util.ToolRunner;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index d33975e..3f67e5c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -18,12 +18,25 @@
package org.apache.kylin.storage.hbase.util;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -51,11 +64,6 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
/**
* <p/>
* This tool serves for the purpose of migrating cubes. e.g. upgrade cube from
@@ -150,7 +158,7 @@ public class CubeMigrationCLI {
moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
}
- public static void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException{
+ public static void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException {
CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
checkCLI.execute(cubeName);
}
@@ -225,7 +233,6 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { cubeName, projectName }));
}
-
private static void purgeAndDisable(String cubeName) throws IOException {
operations.add(new Opt(OptType.PURGE_AND_DISABLE, new Object[] { cubeName }));
}
@@ -305,169 +312,176 @@ public class CubeMigrationCLI {
}
}
+ @SuppressWarnings("checkstyle:methodlength")
private static void doOpt(Opt opt) throws IOException, InterruptedException {
logger.info("Executing operation: " + opt.toString());
switch (opt.type) {
- case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
- desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
- logger.info("CHANGE_HTABLE_HOST is completed");
- break;
- }
- case COPY_FILE_IN_META: {
- String item = (String) opt.params[0];
- RawResource res = srcStore.getResource(item);
- dstStore.putResource(item, res.inputStream, res.timestamp);
- res.inputStream.close();
- logger.info("Item " + item + " is copied");
- break;
- }
- case COPY_DICT_OR_SNAPSHOT: {
- String item = (String) opt.params[0];
-
- if (item.toLowerCase().endsWith(".dict")) {
- DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig);
- DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig);
- DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
-
- long ts = dictSrc.getLastModified();
- dictSrc.setLastModified(0);//to avoid resource store write conflict
- Dictionary dictObj = dictSrc.getDictionaryObject().copyToAnotherMeta(srcConfig, dstConfig);
- DictionaryInfo dictSaved = dstDictMgr.trySaveNewDict(dictObj, dictSrc);
- dictSrc.setLastModified(ts);
-
- if (dictSaved == dictSrc) {
- //no dup found, already saved to dest
- logger.info("Item " + item + " is copied");
- } else {
- //dictSrc is rejected because of duplication
- //modify cube's dictionary path
- String cubeName = (String) opt.params[1];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- for (CubeSegment segment : cube.getSegments()) {
- for (Map.Entry<String, String> entry : segment.getDictionaries().entrySet()) {
- if (entry.getValue().equalsIgnoreCase(item)) {
- entry.setValue(dictSaved.getResourcePath());
- }
+ case CHANGE_HTABLE_HOST: {
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
+ desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
+ hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.enableTable(tableName);
+ logger.info("CHANGE_HTABLE_HOST is completed");
+ break;
+ }
+ case COPY_FILE_IN_META: {
+ String item = (String) opt.params[0];
+ RawResource res = srcStore.getResource(item);
+ dstStore.putResource(item, res.inputStream, res.timestamp);
+ res.inputStream.close();
+ logger.info("Item " + item + " is copied");
+ break;
+ }
+ case COPY_DICT_OR_SNAPSHOT: {
+ String item = (String) opt.params[0];
+
+ if (item.toLowerCase().endsWith(".dict")) {
+ DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig);
+ DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig);
+ DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
+
+ long ts = dictSrc.getLastModified();
+ dictSrc.setLastModified(0);//to avoid resource store write conflict
+ Dictionary dictObj = dictSrc.getDictionaryObject().copyToAnotherMeta(srcConfig, dstConfig);
+ DictionaryInfo dictSaved = dstDictMgr.trySaveNewDict(dictObj, dictSrc);
+ dictSrc.setLastModified(ts);
+
+ if (dictSaved == dictSrc) {
+ //no dup found, already saved to dest
+ logger.info("Item " + item + " is copied");
+ } else {
+ //dictSrc is rejected because of duplication
+ //modify cube's dictionary path
+ String cubeName = (String) opt.params[1];
+ String cubeResPath = CubeInstance.concatResourcePath(cubeName);
+ Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
+ for (CubeSegment segment : cube.getSegments()) {
+ for (Map.Entry<String, String> entry : segment.getDictionaries().entrySet()) {
+ if (entry.getValue().equalsIgnoreCase(item)) {
+ entry.setValue(dictSaved.getResourcePath());
}
}
- dstStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Item " + item + " is dup, instead " + dictSaved.getResourcePath() + " is reused");
}
+ dstStore.putResource(cubeResPath, cube, cubeSerializer);
+ logger.info("Item " + item + " is dup, instead " + dictSaved.getResourcePath() + " is reused");
+ }
+
+ } else if (item.toLowerCase().endsWith(".snapshot")) {
+ SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig);
+ SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig);
+ SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
- } else if (item.toLowerCase().endsWith(".snapshot")) {
- SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig);
- SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig);
- SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
-
- long ts = snapSrc.getLastModified();
- snapSrc.setLastModified(0);
- SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc);
- snapSrc.setLastModified(ts);
-
- if (snapSaved == snapSrc) {
- //no dup found, already saved to dest
- logger.info("Item " + item + " is copied");
-
- } else {
- String cubeName = (String) opt.params[1];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- for (CubeSegment segment : cube.getSegments()) {
- for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) {
- if (entry.getValue().equalsIgnoreCase(item)) {
- entry.setValue(snapSaved.getResourcePath());
- }
+ long ts = snapSrc.getLastModified();
+ snapSrc.setLastModified(0);
+ SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc);
+ snapSrc.setLastModified(ts);
+
+ if (snapSaved == snapSrc) {
+ //no dup found, already saved to dest
+ logger.info("Item " + item + " is copied");
+
+ } else {
+ String cubeName = (String) opt.params[1];
+ String cubeResPath = CubeInstance.concatResourcePath(cubeName);
+ Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
+ for (CubeSegment segment : cube.getSegments()) {
+ for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) {
+ if (entry.getValue().equalsIgnoreCase(item)) {
+ entry.setValue(snapSaved.getResourcePath());
}
}
- dstStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");
-
}
+ dstStore.putResource(cubeResPath, cube, cubeSerializer);
+ logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");
- } else {
- logger.error("unknown item found: " + item);
- logger.info("ignore it");
}
- break;
+ } else {
+ logger.error("unknown item found: " + item);
+ logger.info("ignore it");
}
- case RENAME_FOLDER_IN_HDFS: {
- String srcPath = (String) opt.params[0];
- String dstPath = (String) opt.params[1];
- hdfsFS.rename(new Path(srcPath), new Path(dstPath));
- logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
- break;
- }
- case ADD_INTO_PROJECT: {
- String cubeName = (String) opt.params[0];
- String projectName = (String) opt.params[1];
- String projectResPath = ProjectInstance.concatResourcePath(projectName);
- Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
- ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
- project.removeRealization(RealizationType.CUBE, cubeName);
- project.addRealizationEntry(RealizationType.CUBE, cubeName);
- dstStore.putResource(projectResPath, project, projectSerializer);
- logger.info("Project instance for " + projectName + " is corrected");
- break;
- }
- case COPY_ACL: {
- String cubeId = (String) opt.params[0];
- String modelId = (String) opt.params[1];
- String projectName = (String) opt.params[2];
- String projectResPath = ProjectInstance.concatResourcePath(projectName);
- Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
- ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
- String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
- try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
-
- // cube acl
- Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
- if (result.listCells() != null) {
- for (Cell cell : result.listCells()) {
- byte[] family = CellUtil.cloneFamily(cell);
- byte[] column = CellUtil.cloneQualifier(cell);
- byte[] value = CellUtil.cloneValue(cell);
-
- // use the target project uuid as the parent
- if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
- String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
- value = Bytes.toBytes(valueString);
- }
- Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
- destAclHtable.put(put);
+
+ break;
+ }
+ case RENAME_FOLDER_IN_HDFS: {
+ String srcPath = (String) opt.params[0];
+ String dstPath = (String) opt.params[1];
+ hdfsFS.rename(new Path(srcPath), new Path(dstPath));
+ logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
+ break;
+ }
+ case ADD_INTO_PROJECT: {
+ String cubeName = (String) opt.params[0];
+ String projectName = (String) opt.params[1];
+ String projectResPath = ProjectInstance.concatResourcePath(projectName);
+ Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
+ ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
+ project.removeRealization(RealizationType.CUBE, cubeName);
+ project.addRealizationEntry(RealizationType.CUBE, cubeName);
+ dstStore.putResource(projectResPath, project, projectSerializer);
+ logger.info("Project instance for " + projectName + " is corrected");
+ break;
+ }
+ case COPY_ACL: {
+ String cubeId = (String) opt.params[0];
+ String modelId = (String) opt.params[1];
+ String projectName = (String) opt.params[2];
+ String projectResPath = ProjectInstance.concatResourcePath(projectName);
+ Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
+ ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
+ String projUUID = project.getUuid();
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
+ try {
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+
+ // cube acl
+ Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
+ if (result.listCells() != null) {
+ for (Cell cell : result.listCells()) {
+ byte[] family = CellUtil.cloneFamily(cell);
+ byte[] column = CellUtil.cloneQualifier(cell);
+ byte[] value = CellUtil.cloneValue(cell);
+
+ // use the target project uuid as the parent
+ if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
+ String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
+ value = Bytes.toBytes(valueString);
}
+ Put put = new Put(Bytes.toBytes(cubeId));
+ put.add(family, column, value);
+ destAclHtable.put(put);
}
- destAclHtable.flushCommits();
- } finally {
- IOUtils.closeQuietly(srcAclHtable);
- IOUtils.closeQuietly(destAclHtable);
}
- break;
- }
- case PURGE_AND_DISABLE:{
- String cubeName = (String) opt.params[0];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = srcStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- cube.getSegments().clear();
- cube.setStatus(RealizationStatusEnum.DISABLED);
- srcStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Cube " + cubeName + " is purged and disabled in " + srcConfig.getMetadataUrl());
+ destAclHtable.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(srcAclHtable);
+ IOUtils.closeQuietly(destAclHtable);
}
+ break;
+ }
+ case PURGE_AND_DISABLE: {
+ String cubeName = (String) opt.params[0];
+ String cubeResPath = CubeInstance.concatResourcePath(cubeName);
+ Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ CubeInstance cube = srcStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
+ cube.getSegments().clear();
+ cube.setStatus(RealizationStatusEnum.DISABLED);
+ srcStore.putResource(cubeResPath, cube, cubeSerializer);
+ logger.info("Cube " + cubeName + " is purged and disabled in " + srcConfig.getMetadataUrl());
+
+ break;
+ }
+ default: {
+ //do nothing
+ break;
+ }
}
}
@@ -475,58 +489,63 @@ public class CubeMigrationCLI {
logger.info("Undo operation: " + opt.toString());
switch (opt.type) {
- case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
- desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
- break;
- }
- case COPY_FILE_IN_META: {
- // no harm
- logger.info("Undo for COPY_FILE_IN_META is ignored");
- break;
- }
- case COPY_DICT_OR_SNAPSHOT: {
- // no harm
- logger.info("Undo for COPY_DICT_OR_SNAPSHOT is ignored");
- break;
- }
- case RENAME_FOLDER_IN_HDFS: {
- String srcPath = (String) opt.params[1];
- String dstPath = (String) opt.params[0];
-
- if (hdfsFS.exists(new Path(srcPath)) && !hdfsFS.exists(new Path(dstPath))) {
- hdfsFS.rename(new Path(srcPath), new Path(dstPath));
- logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
- }
- break;
- }
- case ADD_INTO_PROJECT: {
- logger.info("Undo for ADD_INTO_PROJECT is ignored");
- break;
- }
- case COPY_ACL: {
- String cubeId = (String) opt.params[0];
- String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
- try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ case CHANGE_HTABLE_HOST: {
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
+ desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
+ hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.enableTable(tableName);
+ break;
+ }
+ case COPY_FILE_IN_META: {
+ // no harm
+ logger.info("Undo for COPY_FILE_IN_META is ignored");
+ break;
+ }
+ case COPY_DICT_OR_SNAPSHOT: {
+ // no harm
+ logger.info("Undo for COPY_DICT_OR_SNAPSHOT is ignored");
+ break;
+ }
+ case RENAME_FOLDER_IN_HDFS: {
+ String srcPath = (String) opt.params[1];
+ String dstPath = (String) opt.params[0];
- destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
- destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
- } finally {
- IOUtils.closeQuietly(destAclHtable);
- }
- break;
+ if (hdfsFS.exists(new Path(srcPath)) && !hdfsFS.exists(new Path(dstPath))) {
+ hdfsFS.rename(new Path(srcPath), new Path(dstPath));
+ logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
}
- case PURGE_AND_DISABLE: {
- logger.info("Undo for PURGE_AND_DISABLE is not supported");
- break;
+ break;
+ }
+ case ADD_INTO_PROJECT: {
+ logger.info("Undo for ADD_INTO_PROJECT is ignored");
+ break;
+ }
+ case COPY_ACL: {
+ String cubeId = (String) opt.params[0];
+ String modelId = (String) opt.params[1];
+ HTableInterface destAclHtable = null;
+ try {
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+
+ destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
+ destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(destAclHtable);
}
+ break;
+ }
+ case PURGE_AND_DISABLE: {
+ logger.info("Undo for PURGE_AND_DISABLE is not supported");
+ break;
+ }
+ default:
+ {
+ //do nothing
+ break;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index f1fc8e1..295750a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -18,13 +18,15 @@
package org.apache.kylin.storage.hbase.util;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -38,9 +40,7 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* <p/>
@@ -59,13 +59,12 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
-
private KylinConfig dstCfg;
private HBaseAdmin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
-
+
private boolean ifFix = false;
public static void main(String[] args) throws ParseException, IOException {
@@ -101,9 +100,9 @@ public class CubeMigrationCheckCLI {
}
KylinConfig kylinConfig;
- if(dstCfgUri==null){
+ if (dstCfgUri == null) {
kylinConfig = KylinConfig.getInstanceFromEnv();
- }else{
+ } else {
kylinConfig = KylinConfig.createInstanceFromUri(dstCfgUri);
}
@@ -111,21 +110,21 @@ public class CubeMigrationCheckCLI {
checkCLI.execute(cubeName);
}
- public void execute() throws IOException{
+ public void execute() throws IOException {
execute(null);
}
- public void execute(String cubeName) throws IOException{
- if(cubeName==null){
+ public void execute(String cubeName) throws IOException {
+ if (cubeName == null) {
checkAll();
- }else {
+ } else {
checkCube(cubeName);
}
fixInconsistent();
printIssueExistingHTables();
}
- public CubeMigrationCheckCLI(KylinConfig kylinConfig, Boolean isFix) throws IOException{
+ public CubeMigrationCheckCLI(KylinConfig kylinConfig, Boolean isFix) throws IOException {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
@@ -140,12 +139,12 @@ public class CubeMigrationCheckCLI {
List<String> segFullNameList = Lists.newArrayList();
CubeInstance cube = CubeManager.getInstance(dstCfg).getCube(cubeName);
- addHTableNamesForCube(cube,segFullNameList);
+ addHTableNamesForCube(cube, segFullNameList);
check(segFullNameList);
}
- public void checkAll(){
+ public void checkAll() {
List<String> segFullNameList = Lists.newArrayList();
CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
@@ -156,58 +155,58 @@ public class CubeMigrationCheckCLI {
check(segFullNameList);
}
- public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList){
+ public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
for (CubeSegment seg : cube.getSegments()) {
String tableName = seg.getStorageLocationIdentifier();
- segFullNameList.add(tableName+","+cube.getName());
+ segFullNameList.add(tableName + "," + cube.getName());
}
}
- public void check(List<String> segFullNameList){
+ public void check(List<String> segFullNameList) {
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
- for(String segFullName:segFullNameList){
+ for (String segFullName : segFullNameList) {
String[] sepNameList = segFullName.split(",");
try {
HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag);
- if(!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)){
+ if (!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
inconsistentHTables.add(segFullName);
}
- }catch (IOException e){
+ } catch (IOException e) {
issueExistHTables.add(segFullName);
continue;
}
}
}
- public void fixInconsistent() throws IOException{
- if(ifFix == true){
- for(String segFullName : inconsistentHTables){
+ public void fixInconsistent() throws IOException {
+ if (ifFix == true) {
+ for (String segFullName : inconsistentHTables) {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
- logger.info("Change the host of htable "+sepNameList[0]+"belonging to cube "+sepNameList[1]+" from "+desc.getValue(IRealizationConstants.HTableTag)+" to "+dstCfg.getMetadataUrlPrefix());
+ logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
hbaseAdmin.disableTable(sepNameList[0]);
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(sepNameList[0], desc);
hbaseAdmin.enableTable(sepNameList[0]);
}
- }else{
+ } else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
for (String hTable : inconsistentHTables) {
String[] sepNameList = hTable.split(",");
- logger.info(sepNameList[0]+" belonging to cube "+sepNameList[1]);
+ logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]);
}
logger.info("----------------------------------------------------");
}
}
- public void printIssueExistingHTables(){
+ public void printIssueExistingHTables() {
logger.info("------ HTables exist issues in hbase : not existing, metadata broken ------");
- for(String segFullName : issueExistHTables){
+ for (String segFullName : issueExistHTables) {
String[] sepNameList = segFullName.split(",");
- logger.error(sepNameList[0]+" belonging to cube "+sepNameList[1]+" has some issues and cannot be read successfully!!!");
+ logger.error(sepNameList[0] + " belonging to cube " + sepNameList[1] + " has some issues and cannot be read successfully!!!");
}
logger.info("----------------------------------------------------");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 8a88b6d..86ba22f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -267,6 +267,7 @@ public class GridTableHBaseBenchmark {
hbase.getTableDescriptor(TableName.valueOf(tableName));
tableExist = true;
} catch (TableNotFoundException e) {
+ //do nothing?
}
if (tableExist) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 4ce0dac..b6958d6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -22,12 +22,10 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import com.google.common.collect.Lists;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.util.ToolRunner;
@@ -37,6 +35,8 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* clean hbase tables by tag
*/
@@ -63,7 +63,7 @@ public class HBaseClean extends AbstractHadoopJob {
parseOptions(options, args);
logger.info("options: '" + getOptionsAsString() + "'");
-
+
tag = getOptionValue(OPTION_TAG);
delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 0a791b7..266f7e7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.metadata.realization.IRealizationConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index deb8da1..f30f2c9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -28,7 +28,6 @@ import java.util.Random;
import java.util.concurrent.Semaphore;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java
index f1ca4de..c435f34 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HiveCmdBuilder.java
@@ -54,41 +54,41 @@ public class HiveCmdBuilder {
StringBuffer buf = new StringBuffer();
switch (clientMode) {
- case CLI:
- buf.append("hive -e \"");
+ case CLI:
+ buf.append("hive -e \"");
+ for (String statement : statements) {
+ buf.append(statement).append("\n");
+ }
+ buf.append("\"");
+ break;
+ case BEELINE:
+ BufferedWriter bw = null;
+ try {
+ File tmpHql = File.createTempFile("beeline_", ".hql");
+ StringBuffer hqlBuf = new StringBuffer();
+ bw = new BufferedWriter(new FileWriter(tmpHql));
for (String statement : statements) {
- buf.append(statement).append("\n");
- }
- buf.append("\"");
- break;
- case BEELINE:
- BufferedWriter bw = null;
- try {
- File tmpHql = File.createTempFile("beeline_", ".hql");
- StringBuffer hqlBuf = new StringBuffer();
- bw = new BufferedWriter(new FileWriter(tmpHql));
- for (String statement : statements) {
- bw.write(statement);
- bw.newLine();
-
- hqlBuf.append(statement).append("\n");
- }
- buf.append("beeline ");
- buf.append(kylinConfig.getHiveBeelineParams());
- buf.append(" -f ");
- buf.append(tmpHql.getAbsolutePath());
- buf.append(";rm -f ");
- buf.append(tmpHql.getAbsolutePath());
-
- logger.info("The statements to execute in beeline: \n" + hqlBuf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- IOUtils.closeQuietly(bw);
+ bw.write(statement);
+ bw.newLine();
+
+ hqlBuf.append(statement).append("\n");
}
- break;
- default:
- throw new RuntimeException("Hive client cannot be recognized: " + clientMode);
+ buf.append("beeline ");
+ buf.append(kylinConfig.getHiveBeelineParams());
+ buf.append(" -f ");
+ buf.append(tmpHql.getAbsolutePath());
+ buf.append(";rm -f ");
+ buf.append(tmpHql.getAbsolutePath());
+
+ logger.info("The statements to execute in beeline: \n" + hqlBuf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(bw);
+ }
+ break;
+ default:
+ throw new RuntimeException("Hive client cannot be recognized: " + clientMode);
}
return buf.toString();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index 2b4a9a7..4db183b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -24,7 +24,6 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index b1a5ba4..01edb1f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index ac35ccf..af64df7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -24,7 +24,12 @@ import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 23357f5..26ee055 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -63,7 +63,7 @@ public class UpdateHTableHostCLI {
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
- public static void main(String args[]) throws Exception {
+ public static void main(String[] args) throws Exception {
if (args.length < 1) {
printUsageAndExit();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
index 86f895a..e516bc2 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.storage.hbase.common;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.util.Arrays;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 19ba688..f8e2644 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -18,7 +18,8 @@
package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.math.BigDecimal;
@@ -89,7 +90,7 @@ public class AggregateRegionObserverTest {
private Cell newCell(byte[] key, HCol col, String decimal, int number) {
Object[] values = number == Integer.MIN_VALUE ? //
- new Object[] { new BigDecimal(decimal) } //
+ new Object[] { new BigDecimal(decimal) } //
: new Object[] { new BigDecimal(decimal), new LongMutable(number) };
ByteBuffer buf = col.measureCodec.encode(values);
@@ -202,7 +203,7 @@ public class AggregateRegionObserverTest {
t.setDatabase("DEFAULT");
TblColRef[] cols = new TblColRef[] { newCol(1, "A", t), newCol(2, "B", t), newCol(3, "C", t), newCol(4, "D", t) };
int[] sizes = new int[] { 1, 1, 1, 1 };
- return new CoprocessorRowType(cols, sizes,0);
+ return new CoprocessorRowType(cols, sizes, 0);
}
private TblColRef newCol(int i, String name, TableDesc t) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
index 7881688..d0a0710 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
@@ -34,7 +34,7 @@ public class RowAggregatorsTest {
@Test
public void testSerialize() {
ObserverAggregators.HCol[] hcols = new ObserverAggregators.HCol[] { //
- newHCol("f", "c1", new String[] { "SUM", "COUNT" }, new String[] { "decimal", "long" }), //
+ newHCol("f", "c1", new String[] { "SUM", "COUNT" }, new String[] { "decimal", "long" }), //
newHCol("f", "c2", new String[] { "SUM", "SUM" }, new String[] { "long", "long" }) };
ObserverAggregators sample = new ObserverAggregators(hcols);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
index 4547896..74b2112 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
@@ -33,9 +33,10 @@ public class TestFuzzyRowFilterV2 {
Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- 0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { 0, 0, 1, 0 })); // mask
+ Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
+ FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
+ 0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
+ new byte[] { 0, 0, 1, 0 })); // mask
Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, 0, 5, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
@@ -51,9 +52,10 @@ public class TestFuzzyRowFilterV2 {
Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { -1, -1, 0, -1 })); // mask
+ Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
+ FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
+ new byte[] { 1, 2, 0, 3 }, // fuzzy row
+ new byte[] { -1, -1, 0, -1 })); // mask
Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
@@ -172,11 +174,11 @@ public class TestFuzzyRowFilterV2 {
new byte[] { 1, 1, 0, 2, 0 }); // expected next
assertNext(true, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2, 0, 1 },
- // TODO: should be {1, (byte) 128, 2} ?
+ // TODO: should be {1, (byte) 128, 2} ?
new byte[] { 1, (byte) 128, 1, (byte) 0xFF, (byte) 0xFF });
assertNext(true, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2, 1 },
- // TODO: should be {5, 1, 0, 2} ?
+ // TODO: should be {5, 1, 0, 2} ?
new byte[] { 5, 1, 0, 1, (byte) 0xFF });
assertNext(true, new byte[] { 0, 1, 0, 0 }, // fuzzy row
@@ -195,7 +197,7 @@ public class TestFuzzyRowFilterV2 {
new byte[] { 5, 1, (byte) 254, 1 }); // expected next
assertNext(true, new byte[] { 1, 1, 0, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 2, 1, 3, 2 },
- // TODO: should be {1, 0} ?
+ // TODO: should be {1, 0} ?
new byte[] { 1, 1, 0, 0 });
assertNext(true, new byte[] { 1, 0, 1 }, // fuzzy row
@@ -205,35 +207,35 @@ public class TestFuzzyRowFilterV2 {
new byte[] { 1, 0, 1, (byte) 0xFF, (byte) 0xFF });
assertNext(true, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, (byte) 245, 1, 3, 0 },
- // TODO: should be {1, 1, (byte) 255, 4} ?
+ // TODO: should be {1, 1, (byte) 255, 4} ?
new byte[] { 1, 1, 0, 3, (byte) 0xFF });
assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, 3, 1, 3, 0 },
- // TODO: should be 1, 2, (byte) 255, 4 ?
+ // TODO: should be 1, 2, (byte) 255, 4 ?
new byte[] { 1, 2, 0, 3, (byte) 0xFF });
assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 2, 1, 1, 1, 0 },
- // TODO: should be {1, 2, (byte) 255, 4} ?
+ // TODO: should be {1, 2, (byte) 255, 4} ?
new byte[] { 1, 2, 0, 3, (byte) 0xFF });
assertNext(true,
- // TODO: should be null?
+ // TODO: should be null?
new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2 }, new byte[] { 1, (byte) 128, 1 });
assertNext(true,
- // TODO: should be null?
+ // TODO: should be null?
new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2 }, new byte[] { 5, 1, 0, 1 });
assertNext(true,
- // TODO: should be null?
+ // TODO: should be null?
new byte[] { 5, 1, 1, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 5, 1, (byte) 0xFF, 1 }, new byte[] { 5, 1, (byte) 0xFF, 0 });
assertNext(true,
- // TODO: should be null?
+ // TODO: should be null?
new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 2, 2 }, new byte[] { 1, 1, 2, 1 });
assertNext(true,
- // TODO: should be null?
+ // TODO: should be null?
new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 2, 3 }, new byte[] { 1, 1, 2, 2 });
Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(true, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
index 04e1f7f..ac9e995 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
@@ -18,7 +18,16 @@
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
-import com.google.common.collect.Lists;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.invertedindex.IIInstance;
@@ -37,11 +46,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-import java.util.*;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import com.google.common.collect.Lists;
/**
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
index 6868533..f994886 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
@@ -43,7 +43,7 @@ public class CreateHTableTest extends LocalFileMetadataTestCase {
conf.set("fs.default.name", "file:///");
conf.set("mapreduce.framework.name", "local");
conf.set("mapreduce.application.framework.path", "");
-
+
this.createTestMetadata();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index dbf39e7..c1fd2e2 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -21,7 +21,6 @@ package org.apache.kylin.storage.hbase.steps;
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.nio.ByteBuffer;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +31,6 @@ import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.MeasureDecoder;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index 6475bad..81ac32f 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.storage.hbase.steps;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index e49640e..3d6ac57 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -54,10 +54,10 @@ public class SandboxMetastoreCLI {
if ("reset".equalsIgnoreCase(args[0])) {
ResourceTool.main(new String[] { "reset" });
- }else if ("download".equalsIgnoreCase(args[0])) {
+ } else if ("download".equalsIgnoreCase(args[0])) {
ResourceTool.main(new String[] { "download", args[1] });
} else if ("fetch".equalsIgnoreCase(args[0])) {
- ResourceTool.main(new String[] { "fetch", args[1], args[2]});
+ ResourceTool.main(new String[] { "fetch", args[1], args[2] });
} else if ("upload".equalsIgnoreCase(args[0])) {
ResourceTool.main(new String[] { "upload", args[1] });
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/tool/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/tool/.settings/org.eclipse.core.resources.prefs b/tool/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..365bbd6
--- /dev/null
+++ b/tool/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding/<project>=UTF-8