You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/09/06 09:39:41 UTC
[35/50] [abbrv] incubator-kylin git commit: KYLIN-926 clean all
runtime files
KYLIN-926 clean all runtime files
KYLIN-926
KYLIN-926
KYLIN-926
KYLIN-926
KYLIN-926 clean parent file
KYLIN-926 cond
KYLIN-926
KYLIN-926 fix /tmp/cardinality and minicluster tgz
KYLIN-926
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7a3941d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7a3941d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7a3941d8
Branch: refs/heads/master
Commit: 7a3941d81c2f7df036c7d9f0a96045055730433f
Parents: 2beba36
Author: honma <ho...@ebay.com>
Authored: Mon Aug 17 16:32:10 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 19 13:14:32 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 15 +-
.../apache/kylin/common/util/HbaseImporter.java | 9 +-
.../kylin/job/constant/BatchConstants.java | 33 ++--
.../kylin/job/hadoop/AbstractHadoopJob.java | 45 +++--
.../cardinality/HiveColumnCardinalityJob.java | 3 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 10 +-
.../kylin/job/hadoop/cube/NDCuboidMapper.java | 4 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +-
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 4 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 1 +
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../org/apache/kylin/job/ExportHBaseData.java | 41 +++--
.../org/apache/kylin/job/ImportHBaseData.java | 174 -------------------
.../job/hadoop/cube/MergeCuboidJobTest.java | 8 +-
.../apache/kylin/query/test/IIQueryTest.java | 2 -
.../apache/kylin/rest/service/CubeService.java | 3 -
16 files changed, 84 insertions(+), 272 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0c7b813..0e4cc07 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -52,15 +52,6 @@ public class KylinConfig {
public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone";
- /**
- * The dir containing scripts for kylin. For example: /usr/lib/kylin/bin
- */
- public static final String KYLIN_SCRIPT_DIR = "kylin.script.dir";
- /**
- * The script file name for generating table metadat from hive. For example:
- * generateTable.sh
- */
- public static final String KYLIN_SCRIPT_GEN_TABLE_META = "kylin.script.genTableMeta";
public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
@@ -124,6 +115,7 @@ public class KylinConfig {
public static final String MAIL_SENDER = "mail.sender";
public static final String KYLIN_HOME = "KYLIN_HOME";
+
public static final String KYLIN_CONF = "KYLIN_CONF";
private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class);
@@ -233,8 +225,7 @@ public class KylinConfig {
/**
* Find config from environment. The Search process: 1. Check the
- * $KYLIN_CONF/kylin.properties 2. Check the /etc/kylin/kylin.properties 3.
- * Check the kylin.properties in classpath
+ * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties
*
* @return
*/
@@ -284,7 +275,7 @@ public class KylinConfig {
if (!root.endsWith("/")) {
root += "/";
}
- return root + getMetadataUrlPrefix();
+ return root + getMetadataUrlPrefix() + "/";
}
public String getKylinJobLogDir() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
index 0465be0..097272f 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
@@ -35,9 +35,6 @@ import org.apache.kylin.common.persistence.HBaseConnection;
import com.google.common.base.Preconditions;
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
public class HbaseImporter {
private static final Log logger = LogFactory.getLog(HbaseImporter.class);
@@ -56,12 +53,12 @@ public class HbaseImporter {
return;
}
- File folder = new File("/tmp/hbase-export/");
+ File folder = File.createTempFile("hbase-import", "tmp");
if (folder.exists()) {
- FileUtils.deleteDirectory(folder);
+ FileUtils.forceDelete(folder);
}
folder.mkdirs();
- folder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(folder);
//TarGZUtil.uncompressTarGZ(exportFile, folder);
FileUtil.unTar(exportFile, folder);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
index 0ac04fb..34c5760 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
@@ -24,29 +24,30 @@ package org.apache.kylin.job.constant;
*/
public interface BatchConstants {
- public static final char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
+ char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
- public static final String CFG_CUBE_NAME = "cube.name";
- public static final String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
+ String CFG_CUBE_NAME = "cube.name";
+ String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
- public static final String CFG_II_NAME = "ii.name";
- public static final String CFG_II_SEGMENT_NAME = "ii.segment.name";
+ String CFG_II_NAME = "ii.name";
+ String CFG_II_SEGMENT_NAME = "ii.segment.name";
- public static final String INPUT_DELIM = "input.delim";
+ String OUTPUT_PATH = "output.path";
- public static final String OUTPUT_PATH = "output.path";
+ String TABLE_NAME = "table.name";
+ String TABLE_COLUMNS = "table.columns";
- public static final String TABLE_NAME = "table.name";
- public static final String TABLE_COLUMNS = "table.columns";
+ String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
- public static final String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
+ String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
- public static final String MAPREDUCE_COUTNER_GROUP_NAME = "Cube Builder";
+ String MAPPER_SAMPLE_NUMBER = "mapper.sample.number";
+ String REGION_NUMBER = "region.number";
+ String CUBE_CAPACITY = "cube.capacity";
- public static final String MAPPER_SAMPLE_NUMBER = "mapper.sample.number";
- public static final String REGION_NUMBER = "region.number";
- public static final String CUBE_CAPACITY = "cube.capacity";
+ String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/";
+ String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/";
- public static final int COUNTER_MAX = 100000;
- public static final int ERROR_RECORD_THRESHOLD = 100;
+ int COUNTER_MAX = 100000;
+ int ERROR_RECORD_THRESHOLD = 100;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 61a8e48..4c60c52 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -36,6 +36,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,7 +49,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -82,11 +82,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName("level").hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create("level");
protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Partition file path.").create("input");
protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName("htable name").hasArg().isRequired(true).withDescription("HTable name").create("htablename");
- protected static final Option OPTION_KEY_COLUMN_PERCENTAGE = OptionBuilder.withArgName("rowkey column percentage").hasArg().isRequired(true).withDescription("Percentage of row key columns").create("columnpercentage");
- protected static final Option OPTION_KEY_SPLIT_NUMBER = OptionBuilder.withArgName("key split number").hasArg().isRequired(true).withDescription("Number of key split range").create("splitnumber");
protected String name;
- protected String description;
protected boolean isAsync = false;
protected OptionsHelper optionsHelper = new OptionsHelper();
@@ -129,16 +126,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return retVal;
}
- protected static void runJob(Tool job, String[] args) {
- try {
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- } catch (Exception e) {
- e.printStackTrace(System.err);
- System.exit(5);
- }
- }
-
private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
protected void setJobClasspath(Job job) {
@@ -222,7 +209,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
- tmp.delete(); // we need a directory, so delete the file first
+ FileUtils.forceDelete(tmp);
File metaDir = new File(tmp, "meta");
metaDir.mkdirs();
@@ -249,29 +236,41 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
dumpResources(kylinConfig, metaDir, dumpList);
// hadoop distributed cache
- String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
-
+
}
protected void cleanupTempConfFile(Configuration conf) {
String tempMetaFileString = conf.get("tmpfiles");
+ logger.info("tempMetaFileString is : " + tempMetaFileString);
if (tempMetaFileString != null) {
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- tempMetaFile.getParentFile().delete();
+ if (tempMetaFileString.startsWith("file://")) {
+ tempMetaFileString = tempMetaFileString.substring("file://".length());
+ File tempMetaFile = new File(tempMetaFileString);
+ if (tempMetaFile.exists()) {
+ try {
+ FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+ } catch (IOException e) {
+ logger.warn("error when deleting " + tempMetaFile, e);
+ }
+ } else {
+ logger.info("" + tempMetaFileString + " does not exist");
+ }
+ } else {
+ logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
}
}
}
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
- tmp.delete(); // we need a directory, so delete the file first
+ FileUtils.forceDelete(tmp);
File metaDir = new File(tmp, "meta");
metaDir.mkdirs();
- metaDir.getParentFile().deleteOnExit();
// write kylin.properties
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
dumpResources(kylinConfig, metaDir, dumpList);
// hadoop distributed cache
- String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
index e47e05a..ed0abe7 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
/**
@@ -44,7 +45,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
@SuppressWarnings("static-access")
protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table");
- public static final String OUTPUT_PATH = "/tmp/cardinality";
+ public static final String OUTPUT_PATH = BatchConstants.CFG_KYLIN_HDFS_TEMP_DIR + "cardinality";
public HiveColumnCardinalityJob() {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 6e2b467..e13e011 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,29 +18,28 @@
package org.apache.kylin.job.hadoop.cube;
+import java.io.IOException;
+
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* @author yangli9
*/
@@ -88,8 +87,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
printUsage(options);
throw e;
} finally {
- if (job != null)
+ if (job != null) {
cleanupTempConfFile(job.getConfiguration());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
index 7a69c91..f846232 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
@@ -115,7 +115,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Skipped records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L);
skipCounter++;
if (skipCounter % BatchConstants.COUNTER_MAX == 0) {
logger.info("Skipped " + skipCounter + " records!");
@@ -123,7 +123,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
return;
}
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Processed records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L);
handleCounter++;
if (handleCounter % BatchConstants.COUNTER_MAX == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index b6e4233..e5c891b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -339,7 +339,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
} catch (Throwable t) {
logger.error("", t);
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Error records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Error records").increment(1L);
return;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index e9e30c2..1350077 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -53,10 +53,6 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * @author George Song (ysong1)
- */
-
public class CreateHTableJob extends AbstractHadoopJob {
protected static final Logger logger = LoggerFactory.getLogger(CreateHTableJob.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index d39a27e..3916c40 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -237,6 +237,7 @@ public class BuildIIWithEngineTest {
private void exportHBaseData() throws IOException {
ExportHBaseData export = new ExportHBaseData();
export.exportTables();
+ export.tearDown();
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index d1c9745..41399c7 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -24,6 +24,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -43,7 +44,6 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.apache.tools.ant.filters.StringInputStream;
-import org.codehaus.plexus.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
index 9e302b9..557dd39 100644
--- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
+++ b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
@@ -21,6 +21,7 @@ package org.apache.kylin.job;
import java.io.File;
import java.io.IOException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -31,7 +32,7 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HBaseMiniclusterHelper;
import org.apache.kylin.common.util.SSHClient;
-import org.codehaus.plexus.util.FileUtils;
+import org.apache.kylin.job.constant.BatchConstants;
public class ExportHBaseData {
@@ -39,10 +40,13 @@ public class ExportHBaseData {
HTableDescriptor[] allTables;
Configuration config;
HBaseAdmin hbase;
- CliCommandExecutor cli = null;
- String exportFolder;
- String backupArchive = null;
+ CliCommandExecutor cli;
+ String exportHdfsFolder;
+ String exportLocalFolderParent;
+ String exportLocalFolder;
+ String backupArchive;
String tableNameBase;
+ long currentTIME;
public ExportHBaseData() {
try {
@@ -53,9 +57,6 @@ public class ExportHBaseData {
}
private void setup() throws IOException {
- long currentTIME = System.currentTimeMillis();
- exportFolder = "/tmp/hbase-export/" + currentTIME + "/";
- backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz";
KylinConfig.destoryInstance();
System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
@@ -63,6 +64,12 @@ public class ExportHBaseData {
kylinConfig = KylinConfig.getInstanceFromEnv();
cli = kylinConfig.getCliCommandExecutor();
+ currentTIME = System.currentTimeMillis();
+ exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/";
+ exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/";
+ exportLocalFolder = exportLocalFolderParent + currentTIME + "/";
+ backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz";
+
String metadataUrl = kylinConfig.getMetadataUrl();
// split TABLE@HBASE_URL
int cut = metadataUrl.indexOf('@');
@@ -84,16 +91,16 @@ public class ExportHBaseData {
// cleanup hdfs
try {
- if (cli != null && exportFolder != null) {
- cli.execute("hadoop fs -rm -r " + exportFolder);
+ if (cli != null && exportHdfsFolder != null) {
+ cli.execute("hadoop fs -rm -r " + exportHdfsFolder);
}
} catch (IOException e) {
e.printStackTrace();
}
// cleanup sandbox disk
try {
- if (cli != null && exportFolder != null) {
- cli.execute("rm -r " + exportFolder);
+ if (cli != null && exportLocalFolder != null) {
+ cli.execute("rm -r " + exportLocalFolder);
}
} catch (IOException e) {
e.printStackTrace();
@@ -107,24 +114,22 @@ public class ExportHBaseData {
} catch (IOException e) {
e.printStackTrace();
}
-
- KylinConfig.destoryInstance();
-
}
public void exportTables() throws IOException {
- cli.execute("mkdir -p " + exportFolder);
+ cli.execute("mkdir -p " + exportLocalFolderParent);
for (HTableDescriptor table : allTables) {
String tName = table.getNameAsString();
if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
continue;
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportFolder + tName);
+ cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName);
}
- cli.execute("hadoop fs -copyToLocal " + exportFolder + " " + exportFolder);
- cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportFolder + " .");
+ cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent);
+ cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME);
+
downloadToLocal();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
deleted file mode 100644
index 60983b6..0000000
--- a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.persistence.HBaseResourceStore;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HBaseMiniclusterHelper;
-import org.apache.kylin.common.util.SSHClient;
-import org.apache.kylin.common.util.TarGZUtil;
-
-public class ImportHBaseData {
-
- KylinConfig kylinConfig;
- HTableDescriptor[] allTables;
- Configuration config;
- HBaseAdmin hbase;
- CliCommandExecutor cli = null;
- String importFolder = "/tmp/hbase-export/";
- String backupArchive = null;
- String tableNameBase;
-
- public void setup() throws IOException {
-
- KylinConfig.destoryInstance();
- System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cli = kylinConfig.getCliCommandExecutor();
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = metadataUrl.substring(0, cut);
- String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- HConnection conn = HBaseConnection.get(hbaseUrl);
- try {
- hbase = new HBaseAdmin(conn);
- config = hbase.getConfiguration();
- //allTables = hbase.listTables();
- } catch (IOException e) {
- e.printStackTrace();
- throw e;
- }
-
- uploadTarballToRemote();
- }
-
- public void tearDown() {
- // cleanup sandbox disk
- try {
- if (cli != null && importFolder != null) {
- cli.execute("rm -r " + importFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- KylinConfig.destoryInstance();
-
- }
-
- public void importTables() throws IOException {
- // create the metadata htables
- new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
-
- List<String> tablelocations = getTablesBackupLocations(importFolder);
- for (String tableLocation : tablelocations) {
- String table = tableLocation.substring(tableLocation.lastIndexOf("/") + 1);
-
- if (!(table.equalsIgnoreCase(tableNameBase) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) {
- continue;
- }
-
- if (table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX)) {
- // create the cube table; otherwise the import will fail.
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2");
- }
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Import " + table + " file://" + tableLocation);
- }
-
- }
-
- public void uploadTarballToRemote() throws IOException {
-
- cli.execute("mkdir -p /tmp/hbase-export/");
- @SuppressWarnings("unused")
- SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword());
- try {
- // ssh.scpFileToRemote("../examples/test_case_data/minicluster/hbase-export.tar.gz", importFolder);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- cli.execute("tar -xzf /tmp/hbase-export/hbase-export.tar.gz --directory=" + importFolder);
- }
-
- private List<String> getTablesBackupLocations(String exportBase) throws IOException {
- File exportFile = new File("../examples/test_case_data/minicluster/hbase-export.tar.gz");
-
- if (!exportFile.exists()) {
- return null;
- }
-
- File folder = new File("/tmp/hbase-export/");
-
- if (folder.exists()) {
- folder.delete();
- }
-
- folder.mkdirs();
- folder.deleteOnExit();
-
- TarGZUtil.uncompressTarGZ(exportFile, folder);
-
- String[] child = folder.list();
-
- assert child.length == 1;
-
- String backupTime = child[0];
-
- File backupFolder = new File(folder, backupTime);
-
- String[] tableNames = backupFolder.list();
-
- List<String> locations = new ArrayList<String>(15);
-
- for (String t : tableNames) {
- locations.add(exportBase + backupTime + "/" + t);
- }
-
- return locations;
- }
-
- public static void main(String[] args) {
- ImportHBaseData export = new ImportHBaseData();
- try {
- export.setup();
- export.importTables();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- //export.tearDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
index f9d4f45..72f515a 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
@@ -60,16 +60,16 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
String jobname = "merge_cuboid";
File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- baseFolder.delete();
+ FileUtils.forceDelete(baseFolder);
baseFolder.mkdir();
FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- baseFolder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(baseFolder);
File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d");
- eightFolder.delete();
+ FileUtils.forceDelete(eightFolder);
eightFolder.mkdir();
FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder);
- eightFolder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(eightFolder);
FileUtil.fullyDelete(new File(output));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index 198fb2f..80ff4bb 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query.test;
-import java.io.File;
import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -59,5 +58,4 @@ public class IIQueryTest extends KylinQueryTest {
execAndCompQuery("src/test/resources/query/sql_ii", null, true);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index bd81c45..c3e6694 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -313,7 +313,6 @@ public class CubeService extends BasicService {
* @param cube
* @return
* @throws IOException
- * @throws CubeIntegrityException
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@@ -339,7 +338,6 @@ public class CubeService extends BasicService {
* Update a cube status from ready to disabled.
*
* @return
- * @throws CubeIntegrityException
* @throws IOException
* @throws JobException
*/
@@ -528,7 +526,6 @@ public class CubeService extends BasicService {
*
* @throws IOException
* @throws JobException
- * @throws CubeIntegrityException
*/
private void releaseAllSegments(CubeInstance cube) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);