You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/25 06:50:06 UTC
kylin git commit: KYLIN-2195 re-format KylinConfigBase
Repository: kylin
Updated Branches:
refs/heads/master 8530ebd69 -> 89875fae7
KYLIN-2195 re-format KylinConfigBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/89875fae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/89875fae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/89875fae
Branch: refs/heads/master
Commit: 89875fae78556d4d5f5099accb70d6572faeb3d4
Parents: 8530ebd
Author: Li Yang <li...@apache.org>
Authored: Fri Nov 25 14:49:58 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 14:49:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 778 ++++++++++---------
1 file changed, 411 insertions(+), 367 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/89875fae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6131013..8ea03be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -142,16 +142,47 @@ abstract public class KylinConfigBase implements Serializable {
this.properties = BCC.check(properties);
}
+ private Map<Integer, String> convertKeyToInteger(Map<String, String> map) {
+ Map<Integer, String> result = Maps.newLinkedHashMap();
+ for (Entry<String, String> entry : map.entrySet()) {
+ result.put(Integer.valueOf(entry.getKey()), entry.getValue());
+ }
+ return result;
+ }
+
+ public String toString() {
+ return getMetadataUrl();
+ }
+
+ // ============================================================================
+ // ENV
// ============================================================================
public boolean isDevEnv() {
return "DEV".equals(getOptional("kylin.env", "DEV"));
}
+
+ public String getDeployEnv() {
+ return getOptional("kylin.env", "DEV");
+ }
+
+ public String getHdfsWorkingDirectory() {
+ String root = getRequired("kylin.env.hdfs-working-dir");
+ if (!root.endsWith("/")) {
+ root += "/";
+ }
+ return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
+ }
+
+ // ============================================================================
+ // METADATA
+ // ============================================================================
public String getMetadataUrl() {
return getOptional("kylin.metadata.url");
}
+ // for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
}
@@ -169,178 +200,120 @@ abstract public class KylinConfigBase implements Serializable {
}
}
- public String getServerMode() {
- return this.getOptional("kylin.server.mode", "all");
- }
-
- public String getStorageUrl() {
- return getOptional("kylin.storage.url");
- }
-
- public void setStorageUrl(String storageUrl) {
- setProperty("kylin.storage.url", storageUrl);
+ public String[] getRealizationProviders() {
+ return getOptionalStringArray("kylin.metadata.realization-providers", //
+ new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
}
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHiveUrl() {
- return getOptional("kylin.source.hive.connection-url", "");
+ public String[] getCubeDimensionCustomEncodingFactories() {
+ return getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new String[0]);
}
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHiveUser() {
- return getOptional("kylin.source.hive.connection-user", "");
+ public Map<String, String> getCubeCustomMeasureTypes() {
+ return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
+
+ // ============================================================================
+ // DICTIONARY & SNAPSHOT
+ // ============================================================================
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHivePassword() {
- return getOptional("kylin.source.hive.connection-password", "");
+ public int getTrieDictionaryForestMaxTrieSizeMB() {
+ return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
}
- public String getHdfsWorkingDirectory() {
- String root = getRequired("kylin.env.hdfs-working-dir");
- if (!root.endsWith("/")) {
- root += "/";
- }
- return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
+ public int getCachedDictMaxEntrySize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.max-cache-entry", "3000"));
}
- public String[] getRealizationProviders() {
- return getOptionalStringArray("kylin.metadata.realization-providers", //
- new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
+ public boolean isGrowingDictEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false"));
}
- public CliCommandExecutor getCliCommandExecutor() throws IOException {
- CliCommandExecutor exec = new CliCommandExecutor();
- if (getRunAsRemoteCommand()) {
- exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliPort(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
- }
- return exec;
+ public int getAppendDictEntrySize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
}
- public String getHBaseClusterFs() {
- return getOptional("kylin.storage.hbase.cluster-fs", "");
+ // for test
+ public void setAppendDictEntrySize(int entrySize) {
+ setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
}
- public String getHBaseClusterHDFSConfigFile() {
- return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", "");
+ public int getAppendDictCacheSize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
}
- public String getKylinJobLogDir() {
- return getOptional("kylin.job.log-dir", "/tmp/kylin/logs");
+ // for test
+ public void setAppendDictCacheSize(int cacheSize) {
+ setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
}
- public String getKylinJobJarPath() {
- final String jobJar = getOptional("kylin.engine.mr.job-jar");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
- }
- return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
+ public int getCachedSnapshotMaxEntrySize() {
+ return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
}
- public void overrideMRJobJarPath(String path) {
- logger.info("override " + "kylin.engine.mr.job-jar" + " to " + path);
- System.setProperty("kylin.engine.mr.job-jar", path);
+ public int getTableSnapshotMaxMB() {
+ return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300"));
}
- public String getKylinJobMRLibDir() {
- return getOptional("kylin.engine.mr.lib-dir", "");
- }
+ // ============================================================================
+ // CUBE
+ // ============================================================================
- public Map<String, String> getMRConfigOverride() {
- return getPropertiesByPrefix("kylin.engine.mr.config-override.");
+ public double getJobCuboidSizeRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
- public Map<String, String> getHiveConfigOverride() {
- return getPropertiesByPrefix("kylin.source.hive.config-override.");
+ public double getJobCuboidSizeMemHungryRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05"));
}
- public String getKylinSparkJobJarPath() {
- final String jobJar = getOptional("kylin.engine.mr.job-jar.spark");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
- }
- return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
+ public String getCubeAlgorithm() {
+ return getOptional("kylin.cube.algorithm", "auto");
}
- public void overrideSparkJobJarPath(String path) {
- logger.info("override " + "kylin.engine.mr.job-jar.spark" + " to " + path);
- System.setProperty("kylin.engine.mr.job-jar.spark", path);
+ public double getCubeAlgorithmAutoThreshold() {
+ return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7"));
}
- private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
- private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
- private static final Pattern SPARK_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-engine-spark-(.+)\\.jar");
-
- public String getCoprocessorLocalJar() {
- final String coprocessorJar = getOptional("kylin.storage.hbase.coprocessor-local-jar");
- if (StringUtils.isNotEmpty(coprocessorJar)) {
- return coprocessorJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
- }
- return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
+ public int getCubeAlgorithmAutoMapperLimit() {
+ return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
}
- public void overrideCoprocessorLocalJar(String path) {
- logger.info("override " + "kylin.storage.hbase.coprocessor-local-jar" + " to " + path);
- System.setProperty("kylin.storage.hbase.coprocessor-local-jar", path);
+ @Deprecated
+ public int getCubeAggrGroupMaxSize() {
+ return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
}
- private static String getFileName(String homePath, Pattern pattern) {
- File home = new File(homePath);
- SortedSet<String> files = Sets.newTreeSet();
- if (home.exists() && home.isDirectory()) {
- for (File file : home.listFiles()) {
- final Matcher matcher = pattern.matcher(file.getName());
- if (matcher.matches()) {
- files.add(file.getAbsolutePath());
- }
- }
- }
- if (files.isEmpty()) {
- throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
- } else {
- return files.last();
- }
+ public int getCubeAggrGroupMaxCombination() {
+ return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096"));
}
- public double getDefaultHadoopJobReducerInputMB() {
- return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
+ public boolean getCubeAggrGroupIsMandatoryOnlyValid() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false"));
}
- public double getDefaultHadoopJobReducerCountRatio() {
- return Double.parseDouble(getOptional("kylin.engine.mr.reduce-count-ratio", "1.0"));
+ public int getMaxBuildingSegments() {
+ return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
- public int getHadoopJobMinReducerNumber() {
- return Integer.parseInt(getOptional("kylin.engine.mr.min-reducer-number", "1"));
+ public void setMaxBuildingSegments(int maxBuildingSegments) {
+ setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments));
}
+
+ // ============================================================================
+ // JOB
+ // ============================================================================
- public int getHadoopJobMaxReducerNumber() {
- return Integer.parseInt(getOptional("kylin.engine.mr.max-reducer-number", "500"));
+ public CliCommandExecutor getCliCommandExecutor() throws IOException {
+ CliCommandExecutor exec = new CliCommandExecutor();
+ if (getRunAsRemoteCommand()) {
+ exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliPort(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
+ }
+ return exec;
}
- public int getHadoopJobMapperInputRows() {
- return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000"));
+ public String getKylinJobLogDir() {
+ return getOptional("kylin.job.log-dir", "/tmp/kylin/logs");
}
public boolean getRunAsRemoteCommand() {
@@ -387,177 +360,233 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.job.allow-empty-segment", "true"));
}
- //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
- public int getUHCReducerCount() {
- return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+ public int getMaxConcurrentJobLimit() {
+ return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10"));
}
- public String getOverrideHiveTableLocation(String table) {
- return getOptional("kylin.source.hive.table-location." + table.toUpperCase());
+ public String[] getAdminDls() {
+ return getOptionalStringArray("kylin.job.notification-admin-emails", null);
}
- public String getYarnStatusCheckUrl() {
- return getOptional("kylin.engine.mr.yarn-check-status-url", null);
+ public int getCubingInMemSamplingPercent() {
+ int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100"));
+ percent = Math.max(percent, 1);
+ percent = Math.min(percent, 100);
+ return percent;
}
- public int getYarnStatusCheckIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60"));
+ public String getHiveDependencyFilterList() {
+ return this.getOptional("kylin.job.dependency-filter-list", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
}
- public int getMaxConcurrentJobLimit() {
- return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10"));
+ public boolean isMailEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
}
- public String getTimeZone() {
- return getOptional("kylin.web.timezone", "PST");
+ public void setMailEnabled(boolean enable) {
+ setProperty("kylin.job.notification-enabled", "" + enable);
}
- public String[] getRestServers() {
- return getOptionalStringArray("kylin.server.cluster-servers", new String[0]);
+ public String getMailHost() {
+ return getOptional("kylin.job.notification-mail-host", "");
}
- public String getClusterName() {
- return this.getOptional("kylin.server.cluster-name", getMetadataUrlPrefix());
+ public String getMailUsername() {
+ return getOptional("kylin.job.notification-mail-username", "");
}
- public int getWorkersPerServer() {
- //for sequence sql use
- return Integer.parseInt(getOptional("kylin.server.sequence-sql.workers-per-server", "1"));
+ public String getMailPassword() {
+ return getOptional("kylin.job.notification-mail-password", "");
}
- public String[] getAdminDls() {
- return getOptionalStringArray("kylin.job.notification-admin-emails", null);
+ public String getMailSender() {
+ return getOptional("kylin.job.notification-mail-sender", "");
}
- public double getJobCuboidSizeRatio() {
- return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
+ public int getJobRetry() {
+ return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
}
- public double getJobCuboidSizeMemHungryRatio() {
- return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05"));
+ public int getCubeStatsHLLPrecision() {
+ return Integer.parseInt(getOptional("kylin.job.sampling-hll-precision", "14"));
}
- public String getCubeAlgorithm() {
- return getOptional("kylin.cube.algorithm", "auto");
+ public String getJobControllerLock() {
+ return getOptional("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock");
}
- public double getCubeAlgorithmAutoThreshold() {
- return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7"));
+ public Map<Integer, String> getSchedulers() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.job.scheduler.provider."));
+ r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
+ r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler");
+ return r;
}
- public int getCubeAlgorithmAutoMapperLimit() {
- return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
+ public Integer getSchedulerType() {
+ return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
}
- @Deprecated
- public int getCubeAggrGroupMaxSize() {
- return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
- }
+ // ============================================================================
+ // SOURCE.HIVE
+ // ============================================================================
- public int getCubeAggrGroupMaxCombination() {
- return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096"));
+ public Map<Integer, String> getSourceEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider."));
+ // ref constants in ISourceAware
+ r.put(0, "org.apache.kylin.source.hive.HiveSource");
+ r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
+ return r;
}
- public boolean getCubeAggrGroupIsMandatoryOnlyValid() {
- return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false"));
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHiveUrl() {
+ return getOptional("kylin.source.hive.connection-url", "");
}
- public String[] getCubeDimensionCustomEncodingFactories() {
- return getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new String[0]);
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHiveUser() {
+ return getOptional("kylin.source.hive.connection-user", "");
}
- public Map<String, String> getCubeCustomMeasureTypes() {
- return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHivePassword() {
+ return getOptional("kylin.source.hive.connection-password", "");
}
- public int getTableSnapshotMaxMB() {
- return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300"));
+ public Map<String, String> getHiveConfigOverride() {
+ return getPropertiesByPrefix("kylin.source.hive.config-override.");
}
- public int getTrieDictionaryForestMaxTrieSizeMB() {
- return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
+ public String getOverrideHiveTableLocation(String table) {
+ return getOptional("kylin.source.hive.table-location." + table.toUpperCase());
}
- public int getHBaseRegionCountMin() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
+ public boolean isHiveKeepFlatTable() {
+ return Boolean.parseBoolean(this.getOptional("kylin.source.hive.keep-flat-table", "false"));
}
- public int getHBaseRegionCountMax() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));
+ public String getHiveDatabaseForIntermediateTable() {
+ return this.getOptional("kylin.source.hive.database-for-flat-table", "default");
}
- // for test only
- public void setHBaseHFileSizeGB(float size) {
- setProperty("kylin.storage.hbase.hfile-size-gb", String.valueOf(size));
+ public boolean isHiveRedistributeEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.source.hive.redistribute-flat-table", "true"));
}
- public float getHBaseHFileSizeGB() {
- return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0"));
+ public String getHiveClientMode() {
+ return getOptional("kylin.source.hive.client", "cli");
}
- //check KYLIN-1684, in most cases keep the default value
- public boolean isSkippingEmptySegments() {
- return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
+ public String getHiveBeelineParams() {
+ return getOptional("kylin.source.hive.beeline-params", "");
}
- public int getStoragePushDownLimitMax() {
- return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
+ @Deprecated
+ public String getCreateFlatHiveTableMethod() {
+ return getOptional("kylin.source.hive.create-flat-table-method", "1");
}
- public int getScanThreshold() {
- return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
+ // ============================================================================
+ // STORAGE.HBASE
+ // ============================================================================
+
+ public Map<Integer, String> getStorageEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider."));
+ // ref constants in IStorageAware
+ r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
+ r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage");
+ r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
+ return r;
}
- public int getDerivedInThreshold() {
- return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
+ public int getDefaultStorageEngine() {
+ return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
- public int getBadQueryStackTraceDepth() {
- return Integer.parseInt(getOptional("kylin.query.badquery-stacktrace-depth", "10"));
+ public String getStorageUrl() {
+ return getOptional("kylin.storage.url");
}
- public int getBadQueryHistoryNum() {
- return Integer.parseInt(getOptional("kylin.query.badquery-history-number", "10"));
+ // for test only
+ public void setStorageUrl(String storageUrl) {
+ setProperty("kylin.storage.url", storageUrl);
}
- public int getBadQueryDefaultAlertingSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90"));
+ public String getHBaseClusterFs() {
+ return getOptional("kylin.storage.hbase.cluster-fs", "");
}
- public int getBadQueryDefaultDetectIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
+ public String getHBaseClusterHDFSConfigFile() {
+ return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", "");
}
- public boolean getBadQueryPersistentEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.query.badquery-persistent-enabled", "true"));
+ private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
+ private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
+ private static final Pattern SPARK_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-engine-spark-(.+)\\.jar");
+
+ public String getCoprocessorLocalJar() {
+ final String coprocessorJar = getOptional("kylin.storage.hbase.coprocessor-local-jar");
+ if (StringUtils.isNotEmpty(coprocessorJar)) {
+ return coprocessorJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
+ }
+ return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
}
- public String[] getQueryTransformers() {
- return getOptionalStringArray("kylin.query.transformers", new String[0]);
+ public void overrideCoprocessorLocalJar(String path) {
+ logger.info("override " + "kylin.storage.hbase.coprocessor-local-jar" + " to " + path);
+ System.setProperty("kylin.storage.hbase.coprocessor-local-jar", path);
}
- public int getCachedDictMaxEntrySize() {
- return Integer.parseInt(getOptional("kylin.dictionary.max-cache-entry", "3000"));
+ private static String getFileName(String homePath, Pattern pattern) {
+ File home = new File(homePath);
+ SortedSet<String> files = Sets.newTreeSet();
+ if (home.exists() && home.isDirectory()) {
+ for (File file : home.listFiles()) {
+ final Matcher matcher = pattern.matcher(file.getName());
+ if (matcher.matches()) {
+ files.add(file.getAbsolutePath());
+ }
+ }
+ }
+ if (files.isEmpty()) {
+ throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
+ } else {
+ return files.last();
+ }
}
- public int getCachedSnapshotMaxEntrySize() {
- return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
+ public int getHBaseRegionCountMin() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
}
- public boolean getQueryRunLocalCoprocessor() {
- return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", "false"));
+ public int getHBaseRegionCountMax() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));
}
- public long getQueryDurationCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache-threshold-duration", String.valueOf(2000)));
+ // for test only
+ public void setHBaseHFileSizeGB(float size) {
+ setProperty("kylin.storage.hbase.hfile-size-gb", String.valueOf(size));
}
- public long getQueryScanCountCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024)));
+ public float getHBaseHFileSizeGB() {
+ return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0"));
}
- public long getQueryMemBudget() {
- return Long.parseLong(this.getOptional("kylin.query.memory-budget-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", "false"));
}
public double getQueryCoprocessorMemGB() {
@@ -568,18 +597,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds", "0"));
}
- public boolean isQuerySecureEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true"));
- }
-
- public boolean isQueryCacheEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", "true"));
- }
-
- public boolean isQueryIgnoreUnknownFunction() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.ignore-unknown-function", "false"));
- }
-
public int getQueryScanFuzzyKeyMax() {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-fuzzykey-scan", "200"));
}
@@ -588,23 +605,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.valueOf(this.getOptional("kylin.storage.hbase.max-visit-scanrange", "1000000"));
}
- public String getQueryAccessController() {
- return getOptional("kylin.query.access-controller", null);
- }
-
- public long getSequenceExpireTime() {
- return Long.valueOf(this.getOptional("kylin.server.sequence-sql.expire-time", "86400000"));//default a day
- }
-
- public boolean getQueryMetricsEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false"));
- }
-
- public int[] getQueryMetricsPercentilesIntervals() {
- String[] dft = { "60", "300", "3600" };
- return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
- }
-
public String getDefaultIGTStorage() {
return getOptional("kylin.storage.hbase.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
}
@@ -613,15 +613,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.scan-cache-rows", "1024"));
}
- public boolean isGrowingDictEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false"));
- }
-
- /**
- * HBase region cut size, in GB
- *
- * @return
- */
public float getKylinHBaseRegionCut() {
return Float.valueOf(getOptional("kylin.storage.hbase.region-cut-gb", "5.0"));
}
@@ -630,13 +621,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-scan-result-bytes", "" + (5 * 1024 * 1024))); // 5 MB
}
- public int getCubingInMemSamplingPercent() {
- int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100"));
- percent = Math.max(percent, 1);
- percent = Math.min(percent, 100);
- return percent;
- }
-
public String getHbaseDefaultCompressionCodec() {
return getOptional("kylin.storage.hbase.compression-codec", "none");
}
@@ -653,152 +637,204 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.valueOf(getOptional("kylin.storage.hbase.small-family-block-size-bytes", "65536"));
}
- public boolean isHiveKeepFlatTable() {
- return Boolean.parseBoolean(this.getOptional("kylin.source.hive.keep-flat-table", "false"));
+ public String getKylinOwner() {
+ return this.getOptional("kylin.storage.hbase.owner-tag", "");
}
- public String getHiveDatabaseForIntermediateTable() {
- return this.getOptional("kylin.source.hive.database-for-flat-table", "default");
+ public boolean getCompressionResult() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", "true"));
}
- public boolean isHiveRedistributeEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.source.hive.redistribute-flat-table", "true"));
+ public int getHBaseMaxConnectionThreads() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048"));
}
- public String getHiveDependencyFilterList() {
- return this.getOptional("kylin.job.dependency-filter-list", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+ public int getHBaseCoreConnectionThreads() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048"));
}
- public String getKylinOwner() {
- return this.getOptional("kylin.storage.hbase.owner-tag", "");
+ public long getHBaseConnectionThreadPoolAliveSeconds() {
+ return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60"));
}
- public String getSparkHome() {
- return getRequired("kylin.engine.spark.spark-home");
+ // ============================================================================
+ // ENGINE.MR
+ // ============================================================================
+
+ public Map<Integer, String> getJobEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."));
+ // ref constants in IEngineAware
+ r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
+ r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
+ return r;
}
- public String getSparkMaster() {
- return getRequired("kylin.engine.spark.spark-master");
+ public int getDefaultCubeEngine() {
+ return Integer.parseInt(getOptional("kylin.engine.default", "2"));
}
- public boolean isMailEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
+ public String getKylinJobJarPath() {
+ final String jobJar = getOptional("kylin.engine.mr.job-jar");
+ if (StringUtils.isNotEmpty(jobJar)) {
+ return jobJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ return "";
+ }
+ return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
}
- public void setMailEnabled(boolean enable) {
- setProperty("kylin.job.notification-enabled", "" + enable);
+ public void overrideMRJobJarPath(String path) {
+ logger.info("override " + "kylin.engine.mr.job-jar" + " to " + path);
+ System.setProperty("kylin.engine.mr.job-jar", path);
}
- public String getMailHost() {
- return getOptional("kylin.job.notification-mail-host", "");
+ public String getKylinJobMRLibDir() {
+ return getOptional("kylin.engine.mr.lib-dir", "");
}
- public String getMailUsername() {
- return getOptional("kylin.job.notification-mail-username", "");
+ public Map<String, String> getMRConfigOverride() {
+ return getPropertiesByPrefix("kylin.engine.mr.config-override.");
}
- public String getMailPassword() {
- return getOptional("kylin.job.notification-mail-password", "");
+ public double getDefaultHadoopJobReducerInputMB() {
+ return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
}
- public String getMailSender() {
- return getOptional("kylin.job.notification-mail-sender", "");
+ public double getDefaultHadoopJobReducerCountRatio() {
+ return Double.parseDouble(getOptional("kylin.engine.mr.reduce-count-ratio", "1.0"));
}
- public boolean isWebCrossDomainEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
+ public int getHadoopJobMinReducerNumber() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.min-reducer-number", "1"));
}
- public int getJobRetry() {
- return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
+ public int getHadoopJobMaxReducerNumber() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.max-reducer-number", "500"));
}
- public String toString() {
- return getMetadataUrl();
+ public int getHadoopJobMapperInputRows() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000"));
}
- public String getHiveClientMode() {
- return getOptional("kylin.source.hive.client", "cli");
+ //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
+ public int getUHCReducerCount() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
}
- public String getHiveBeelineParams() {
- return getOptional("kylin.source.hive.beeline-params", "");
+ public String getYarnStatusCheckUrl() {
+ return getOptional("kylin.engine.mr.yarn-check-status-url", null);
}
- public String getDeployEnv() {
- return getOptional("kylin.env", "DEV");
+ public int getYarnStatusCheckIntervalSeconds() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60"));
}
- public String getInitTasks() {
- return getOptional("kylin.server.init-tasks");
+ // ============================================================================
+ // ENGINE.SPARK
+ // ============================================================================
+
+ public String getKylinSparkJobJarPath() {
+ final String jobJar = getOptional("kylin.engine.spark.job-jar");
+ if (StringUtils.isNotEmpty(jobJar)) {
+ return jobJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ return "";
+ }
+ return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
}
- public int getDimCountDistinctMaxCardinality() {
- return Integer.parseInt(getOptional("kylin.query.max-dimension-count-distinct", "5000000"));
+ public void overrideSparkJobJarPath(String path) {
+ logger.info("override " + "kylin.engine.spark.job-jar" + " to " + path);
+ System.setProperty("kylin.engine.spark.job-jar", path);
}
- public int getCubeStatsHLLPrecision() {
- return Integer.parseInt(getOptional("kylin.job.sampling-hll-precision", "14"));
+ public String getSparkHome() {
+ return getRequired("kylin.engine.spark.spark-home");
}
- public String getJobControllerLock() {
- return getOptional("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock");
+ public String getSparkMaster() {
+ return getRequired("kylin.engine.spark.spark-master");
}
- public Map<Integer, String> getJobEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."));
- // ref constants in IEngineAware
- r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
- r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
- return r;
+ // ============================================================================
+ // QUERY
+ // ============================================================================
+
+ //check KYLIN-1684, in most cases keep the default value
+ public boolean isSkippingEmptySegments() {
+ return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
- public Map<Integer, String> getSourceEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider."));
- // ref constants in ISourceAware
- r.put(0, "org.apache.kylin.source.hive.HiveSource");
- r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
- return r;
+ public int getStoragePushDownLimitMax() {
+ return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
- public Map<Integer, String> getStorageEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider."));
- // ref constants in IStorageAware
- r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
- r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage");
- r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
- return r;
+ public int getScanThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
- public int getDefaultStorageEngine() {
- return Integer.parseInt(getOptional("kylin.storage.default", "2"));
+ public int getDerivedInThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
}
- public int getDefaultCubeEngine() {
- return Integer.parseInt(getOptional("kylin.engine.default", "2"));
+ public int getBadQueryStackTraceDepth() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-stacktrace-depth", "10"));
}
- public Map<Integer, String> getSchedulers() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.job.scheduler.provider."));
- r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
- r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler");
- return r;
+ public int getBadQueryHistoryNum() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-history-number", "10"));
}
- public Integer getSchedulerType() {
- return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
+ public int getBadQueryDefaultAlertingSeconds() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90"));
}
- private Map<Integer, String> convertKeyToInteger(Map<String, String> map) {
- Map<Integer, String> result = Maps.newLinkedHashMap();
- for (Entry<String, String> entry : map.entrySet()) {
- result.put(Integer.valueOf(entry.getKey()), entry.getValue());
- }
- return result;
+ public int getBadQueryDefaultDetectIntervalSeconds() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
}
- public boolean getCompressionResult() {
- return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", "true"));
+ public boolean getBadQueryPersistentEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.badquery-persistent-enabled", "true"));
+ }
+
+ public String[] getQueryTransformers() {
+ return getOptionalStringArray("kylin.query.transformers", new String[0]);
+ }
+
+ public long getQueryDurationCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache-threshold-duration", String.valueOf(2000)));
+ }
+
+ public long getQueryScanCountCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024)));
+ }
+
+ public long getQueryMemBudget() {
+ return Long.parseLong(this.getOptional("kylin.query.memory-budget-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
+ }
+
+ public boolean isQuerySecureEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true"));
+ }
+
+ public boolean isQueryCacheEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", "true"));
+ }
+
+ public boolean isQueryIgnoreUnknownFunction() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.ignore-unknown-function", "false"));
+ }
+
+ public String getQueryAccessController() {
+ return getOptional("kylin.query.access-controller", null);
+ }
+
+ public int getDimCountDistinctMaxCardinality() {
+ return Integer.parseInt(getOptional("kylin.query.max-dimension-count-distinct", "5000000"));
}
public Map<String, String> getUDFs() {
@@ -806,46 +842,54 @@ abstract public class KylinConfigBase implements Serializable {
return udfMap;
}
- public int getHBaseMaxConnectionThreads() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048"));
+ // ============================================================================
+ // SERVER
+ // ============================================================================
+
+ public String getServerMode() {
+ return this.getOptional("kylin.server.mode", "all");
}
- public int getHBaseCoreConnectionThreads() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048"));
+ public String[] getRestServers() {
+ return getOptionalStringArray("kylin.server.cluster-servers", new String[0]);
}
- public long getHBaseConnectionThreadPoolAliveSeconds() {
- return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60"));
+ public String getClusterName() {
+ return this.getOptional("kylin.server.cluster-name", getMetadataUrlPrefix());
}
- public int getAppendDictEntrySize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
+ public String getInitTasks() {
+ return getOptional("kylin.server.init-tasks");
}
- // for test
- public void setAppendDictEntrySize(int entrySize) {
- setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
+ public int getWorkersPerServer() {
+ //for sequence sql use
+ return Integer.parseInt(getOptional("kylin.server.sequence-sql.workers-per-server", "1"));
}
- public int getAppendDictCacheSize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
+ public long getSequenceExpireTime() {
+ return Long.valueOf(this.getOptional("kylin.server.sequence-sql.expire-time", "86400000"));//default a day
}
- // for test
- public void setAppendDictCacheSize(int cacheSize) {
- setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
+ public boolean getQueryMetricsEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false"));
}
- @Deprecated
- public String getCreateFlatHiveTableMethod() {
- return getOptional("kylin.source.hive.create-flat-table-method", "1");
+ public int[] getQueryMetricsPercentilesIntervals() {
+ String[] dft = { "60", "300", "3600" };
+ return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
}
- public int getMaxBuildingSegments() {
- return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
+ // ============================================================================
+ // WEB
+ // ============================================================================
+
+ public String getTimeZone() {
+ return getOptional("kylin.web.timezone", "PST");
}
- public void setMaxBuildingSegments(int maxBuildingSegments) {
- setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments));
+ public boolean isWebCrossDomainEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+
}