You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/09/02 09:33:04 UTC
incubator-kylin git commit: KYLIN-972 MR_V1 build well and pass query
test
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 d501dc4e3 -> a79db0562
KYLIN-972 MR_V1 build well and pass query test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a79db056
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a79db056
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a79db056
Branch: refs/heads/0.8
Commit: a79db0562d8fcf299ea1ecd19ec701258fd7a25d
Parents: d501dc4
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Sep 2 15:31:34 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Sep 2 15:32:38 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 18 +++++---
.../kylin/common/persistence/ResourceTool.java | 2 +-
.../org/apache/kylin/source/SourceFactory.java | 2 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 4 +-
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 2 +-
.../apache/kylin/source/hive/HiveMRInput.java | 5 ++-
.../apache/kylin/source/hive/HiveSource.java | 44 +++++++++++++++++++
.../kylin/source/hive/HiveTableSource.java | 45 --------------------
8 files changed, 64 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 4191f8e..06deb55 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -30,6 +30,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.Sets;
+
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.restclient.RestClient;
@@ -42,10 +43,14 @@ import java.util.Map;
import java.util.Properties;
/**
- * @author yangli9
*/
+@SuppressWarnings("serial")
public class KylinConfig implements Serializable {
+ /*
+ * NOTE: These key constants should be private or even better be removed.
+ * All external access should go through public methods.
+ */
public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
@@ -287,8 +292,6 @@ public class KylinConfig implements Serializable {
}
return root + getMetadataUrlPrefix() + "/";
}
-
-
public String getKylinJobLogDir() {
return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs");
@@ -497,11 +500,15 @@ public class KylinConfig implements Serializable {
percent = Math.min(percent, 100);
return percent;
}
-
+
public String getHbaseDefaultCompressionCodec() {
return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC);
}
+ public boolean isHiveKeepFlatTable() {
+ return Boolean.parseBoolean(this.getOptional("kylin.hive.keep.flat.table", "false"));
+ }
+
private String getOptional(String prop) {
final String property = System.getProperty(prop);
return property != null ? property : properties.getProperty(prop);
@@ -512,7 +519,7 @@ public class KylinConfig implements Serializable {
if (!StringUtils.isBlank(property)) {
return property.split("\\s*,\\s*");
} else {
- return new String[]{};
+ return new String[] {};
}
}
@@ -625,7 +632,6 @@ public class KylinConfig implements Serializable {
}
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 0b766fb..7e89c21 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -50,7 +50,7 @@ public class ResourceTool {
reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
break;
case "list":
- list(KylinConfig.getInstanceFromEnv(), args[2]);
+ list(KylinConfig.getInstanceFromEnv(), args[1]);
break;
case "download":
copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
index 2fbf847..a771f5e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
@@ -31,7 +31,7 @@ public class SourceFactory {
private static ImplementationSwitch sources;
static {
Map<Integer, String> impls = new HashMap<>();
- impls.put(ID_HIVE, "org.apache.kylin.source.hive.HiveTableSource");
+ impls.put(ID_HIVE, "org.apache.kylin.source.hive.HiveSource");
sources = new ImplementationSwitch(impls);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 0e22243..c2d0f92 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -189,9 +189,9 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
protected void handleErrorRecord(BytesSplitter bytesSplitter, Exception ex) throws IOException {
- System.err.println("Insane record: " + bytesSplitter);
- ex.printStackTrace(System.err);
+ logger.error("Insane record: " + bytesSplitter, ex);
+ // TODO expose errorRecordCounter as hadoop counter
errorRecordCounter++;
if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
if (ex instanceof IOException)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
index a1b95c7..e2a49df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
@@ -60,7 +60,7 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN, O
private byte[][] convertUTF8Bytes(String[] row) throws UnsupportedEncodingException {
byte[][] result = new byte[row.length][];
for (int i = 0; i < row.length; i++) {
- result[i] = row[i].getBytes("UTF-8");
+ result[i] = row[i] == null ? HIVE_NULL : row[i].getBytes("UTF-8");
}
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index ec9b432..b4edd2e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -152,15 +152,16 @@ public class HiveMRInput implements IMRInput {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ KylinConfig config = context.getConfig();
StringBuffer output = new StringBuffer();
final String hiveTable = this.getIntermediateTableIdentity();
- if (StringUtils.isNotEmpty(hiveTable)) {
+ if (config.isHiveKeepFlatTable() == false && StringUtils.isNotEmpty(hiveTable)) {
final String dropSQL = "DROP TABLE IF EXISTS " + hiveTable + ";";
final String dropHiveCMD = "hive -e \"" + dropSQL + "\"";
ShellCmdOutput shellCmdOutput = new ShellCmdOutput();
try {
- context.getConfig().getCliCommandExecutor().execute(dropHiveCMD, shellCmdOutput);
+ config.getCliCommandExecutor().execute(dropHiveCMD, shellCmdOutput);
output.append("Hive table " + hiveTable + " is dropped. \n");
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
new file mode 100644
index 0000000..b7dbff0
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.hive;
+
+import org.apache.kylin.engine.mr.IMRInput;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.source.ISource;
+import org.apache.kylin.source.ReadableTable;
+
+//used by reflection
+public class HiveSource implements ISource {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ if (engineInterface == IMRInput.class) {
+ return (I) new HiveMRInput();
+ } else {
+ throw new RuntimeException("Cannot adapt to " + engineInterface);
+ }
+ }
+
+ @Override
+ public ReadableTable createReadableTable(TableDesc tableDesc) {
+ return new HiveTable(tableDesc);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a79db056/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
deleted file mode 100644
index 2dbe071..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ISource;
-import org.apache.kylin.source.ReadableTable;
-
-@SuppressWarnings("unused")
-//used by reflection
-public class HiveTableSource implements ISource {
-
- @SuppressWarnings("unchecked")
- @Override
- public <I> I adaptToBuildEngine(Class<I> engineInterface) {
- if (engineInterface == IMRInput.class) {
- return (I) new HiveMRInput();
- } else {
- throw new RuntimeException("Cannot adapt to " + engineInterface);
- }
- }
-
- @Override
- public ReadableTable createReadableTable(TableDesc tableDesc) {
- return new HiveTable(tableDesc);
- }
-
-}