You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/11 01:26:47 UTC
[01/50] [abbrv] incubator-kylin git commit: KYLIN-849 Couldn't query
metrics on lookup table PK
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7 6480a9a7a -> 83d05800d
KYLIN-849 Couldn't query metrics on lookup table PK
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6ab09034
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6ab09034
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6ab09034
Branch: refs/heads/0.7
Commit: 6ab09034d6edc50d879fd41afab3cf340883d82c
Parents: a487c6e
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 23 19:10:56 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 23 19:11:17 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 5 +++-
.../kylin/query/relnode/OLAPAggregateRel.java | 8 ++----
.../AdjustForWeeklyMatchedRealization.java | 2 +-
query/src/test/resources/query/sql/query80.sql | 27 ++++++++++++++++++++
4 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ab09034/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 27cf4ec..bd95d6e 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -135,7 +135,10 @@ public class CubeCapabilityChecker {
if (functionDesc.isCountDistinct()) // calcite can not handle distinct count
matched = false;
- TblColRef col = functionDesc.selectTblColRef(metricColumns, cubeDesc.getFactTable());
+ TblColRef col = null;
+ if (functionDesc.getParameter().getColRefs().size() > 0)
+ col = functionDesc.getParameter().getColRefs().get(0);
+
if (col == null || !cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
matched = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ab09034/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 7cdb0b4..10d4150 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -18,12 +18,7 @@
package org.apache.kylin.query.relnode;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import net.hydromatic.optiq.AggregateFunction;
import net.hydromatic.optiq.FunctionParameter;
@@ -212,6 +207,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
parameter = new ParameterDesc();
parameter.setValue(column.getName());
parameter.setType("column");
+ parameter.setColRefs(Arrays.asList(column));
}
}
FunctionDesc aggFunc = new FunctionDesc();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ab09034/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
index df58944..465d162 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
@@ -79,7 +79,7 @@ public class AdjustForWeeklyMatchedRealization extends RoutingRule {
FunctionDesc functionDesc = it.next();
if (!availableAggregations.contains(functionDesc)) {
// try to convert the metric to dimension to see if it works
- TblColRef col = functionDesc.selectTblColRef(olapContext.metricsColumns, factTableName);
+ TblColRef col = functionDesc.getParameter().getColRefs().get(0);
functionDesc.setDimensionAsMetric(true);
olapContext.rewriteFields.remove(functionDesc.getRewriteFieldName());
if (col != null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ab09034/query/src/test/resources/query/sql/query80.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query80.sql b/query/src/test/resources/query/sql/query80.sql
new file mode 100644
index 0000000..990e6dc
--- /dev/null
+++ b/query/src/test/resources/query/sql/query80.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ min(test_cal_dt.cal_dt) as mmin
+ ,max(test_cal_dt.cal_dt) as mmax
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
\ No newline at end of file
[06/50] [abbrv] incubator-kylin git commit: hot fix: delete query
cache at wiping all cache
Posted by li...@apache.org.
hot fix: delete query cache at wiping all cache
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/269d9225
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/269d9225
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/269d9225
Branch: refs/heads/0.7
Commit: 269d92253907bb714f7d33a8daf3df4e08c4584f
Parents: e0733c9
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 21:20:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:20:53 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/CacheService.java | 119 ++++++++++---------
1 file changed, 60 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/269d9225/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index e104897..164ca12 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -41,44 +41,45 @@ public class CacheService extends BasicService {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().loadCubeCache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().reloadCubeDesc(cacheKey);
- break;
- case PROJECT:
- getProjectManager().reloadProject(cacheKey);
- break;
- case INVERTED_INDEX:
- getIIManager().loadIICache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDesc(cacheKey);
- break;
- case TABLE:
- getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case DATA_MODEL:
- getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case ALL:
- getMetadataManager().reload();
- CubeDescManager.clearCache();
- CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
- break;
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().loadCubeCache(cacheKey);
+ cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().reloadCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ getProjectManager().reloadProject(cacheKey);
+ break;
+ case INVERTED_INDEX:
+ getIIManager().loadIICache(cacheKey);
+ cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().reloadIIDesc(cacheKey);
+ break;
+ case TABLE:
+ getMetadataManager().reloadTableCache(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case DATA_MODEL:
+ getMetadataManager().reloadDataModelDesc(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case ALL:
+ getMetadataManager().reload();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ IIDescManager.clearCache();
+ IIManager.clearCache();
+ ProjectManager.clearCache();
+ BasicService.resetOLAPDataSources();
+ cleanDataCache();
+ break;
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
@@ -98,27 +99,27 @@ public class CacheService extends BasicService {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().removeCubeCacheLocal(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().removeLocalCubeDesc(cacheKey);
- break;
- case PROJECT:
- ProjectManager.clearCache();
- break;
- case INVERTED_INDEX:
- getIIManager().removeIILocalCache(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
- case TABLE:
- throw new UnsupportedOperationException(log);
- case DATA_MODEL:
- throw new UnsupportedOperationException(log);
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().removeCubeCacheLocal(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().removeLocalCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ ProjectManager.clearCache();
+ break;
+ case INVERTED_INDEX:
+ getIIManager().removeIILocalCache(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().removeIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ throw new UnsupportedOperationException(log);
+ case DATA_MODEL:
+ throw new UnsupportedOperationException(log);
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
[03/50] [abbrv] incubator-kylin git commit: KYLIN-844 control
observer behavior
Posted by li...@apache.org.
KYLIN-844 control observer behavior
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ad4a28af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ad4a28af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ad4a28af
Branch: refs/heads/0.7
Commit: ad4a28af5e07fb75ec5d9a5eda11cd7409b033b1
Parents: 198be9e
Author: honma <ho...@ebay.com>
Authored: Fri Jun 19 18:29:59 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 12:02:25 2015 +0800
----------------------------------------------------------------------
.../observer/AggregateRegionObserver.java | 17 +++++---
.../observer/AggregationScanner.java | 37 +++++++++-------
.../coprocessor/observer/ObserverBehavior.java | 9 ++++
.../coprocessor/observer/ObserverEnabler.java | 33 +++++++--------
.../observer/AggregateRegionObserverTest.java | 44 ++++++++++----------
5 files changed, 80 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index d893621..aea2cea 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -18,11 +18,6 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Scan;
@@ -32,6 +27,11 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+
+import java.io.IOException;
/**
* @author yangli9
@@ -46,6 +46,8 @@ public class AggregateRegionObserver extends BaseRegionObserver {
static final String PROJECTOR = "_Projector";
static final String AGGREGATORS = "_Aggregators";
static final String FILTER = "_Filter";
+ static final String BEHAVIOR = "_Behavior";
+
@Override
public final RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
@@ -83,6 +85,9 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
+ byte[] behavior = scan.getAttribute(BEHAVIOR);
+ ObserverBehavior observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
@@ -90,7 +95,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner,observerBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
index 6bdb53a..85f3f6e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
@@ -18,18 +18,17 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
/**
* @author yangli9
@@ -38,11 +37,14 @@ import org.apache.kylin.metadata.measure.MeasureAggregator;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
+ private ObserverBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, ObserverBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
+ this.behavior = behavior;
+
ObserverAggregationCache aggCache;
Stats stats = new Stats();
@@ -72,14 +74,19 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (filter != null && filter.evaluate(tuple) == false)
- continue;
- CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(results);
- MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
- aggregators.aggregate(bufs, results);
+ if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER.ordinal()) {
+ if (filter != null && filter.evaluate(tuple) == false)
+ continue;
- aggCache.checkMemoryUsage();
+ if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(results);
+ MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
+ aggregators.aggregate(bufs, results);
+
+ aggCache.checkMemoryUsage();
+ }
+ }
}
return aggCache;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
new file mode 100644
index 0000000..62c5bac
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
@@ -0,0 +1,9 @@
+package org.apache.kylin.storage.hbase.coprocessor.observer;
+
+/**
+ */
+public enum ObserverBehavior {
+ SCAN, //only scan data, used for profiling tuple scan speed
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed
+ SCAN_FILTER_AGGR, //default normal behavior
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index eff0eee..ce9ba9d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -18,32 +18,31 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.storage.StorageContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowValueDecoder;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.RegionScannerAdapter;
import org.apache.kylin.storage.hbase.ResultScannerAdapter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
/**
* @author yangli9
@@ -70,10 +69,11 @@ public class ObserverEnabler {
if (DEBUG_LOCAL_COPROCESSOR) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
+ scan.setAttribute(AggregateRegionObserver.BEHAVIOR, ObserverBehavior.SCAN_FILTER_AGGR.toString().getBytes());
scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));
@@ -90,7 +90,6 @@ public class ObserverEnabler {
private static boolean isCoprocessorBeneficial(CubeInstance cube, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
-
String forceFlag = System.getProperty(FORCE_COPROCESSOR);
if (forceFlag != null) {
return Boolean.parseBoolean(forceFlag);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad4a28af/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
index 6c789d3..9bed0ca 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
@@ -18,36 +18,36 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverAggregators.HCol;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverAggregators.HCol;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* @author yangli9
@@ -122,7 +122,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -171,7 +171,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner,ObserverBehavior.SCAN_FILTER_AGGR);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[13/50] [abbrv] incubator-kylin git commit: KYLIN-839 use dictionary
in building snapshot
Posted by li...@apache.org.
KYLIN-839 use dictionary in building snapshot
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b97d0e55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b97d0e55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b97d0e55
Branch: refs/heads/0.7
Commit: b97d0e550739a9ebc22249eca37e025ec2479d36
Parents: c47813f
Author: shaofengshi <sh...@apache.org>
Authored: Sun Jun 28 09:56:32 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Jun 28 09:56:32 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/lookup/SnapshotTable.java | 61 +++++++++++++++++---
1 file changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b97d0e55/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index faa4fe6..35cbb84 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -32,6 +32,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.StringBytesConverter;
+import org.apache.kylin.dict.TrieDictionary;
+import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.metadata.model.TableDesc;
/**
@@ -44,6 +48,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
private TableSignature signature;
@JsonProperty("column_delimeter")
private String columnDelimeter;
+ @JsonProperty("useDictionary")
+ private Boolean useDictionary;
private ArrayList<String[]> rows;
@@ -54,6 +60,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
SnapshotTable(ReadableTable table) throws IOException {
this.signature = table.getSignature();
this.columnDelimeter = table.getColumnDelimeter();
+ this.useDictionary = Boolean.TRUE;
}
public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
@@ -154,25 +161,61 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
if (rows.size() > 0) {
int n = rows.get(0).length;
out.writeInt(n);
- for (int i = 0; i < rows.size(); i++) {
- String[] row = rows.get(i);
- for (int j = 0; j < n; j++) {
- out.writeUTF(row[j]);
+
+ if (this.useDictionary == Boolean.TRUE) {
+ Dictionary<String> dict = buildDictionary();
+ dict.write(out);
+ for (int i = 0; i < rows.size(); i++) {
+ String[] row = rows.get(i);
+ for (int j = 0; j < n; j++) {
+ out.writeInt(dict.getIdFromValue(row[j]));
+ }
+ }
+
+ } else {
+ for (int i = 0; i < rows.size(); i++) {
+ String[] row = rows.get(i);
+ for (int j = 0; j < n; j++) {
+ out.writeUTF(row[j]);
+ }
}
}
}
}
+ Dictionary<String> buildDictionary() {
+ TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+ for (String[] row : rows) {
+ for (String cell : row) {
+ b.addValue(cell);
+ }
+ }
+ TrieDictionary<String> dict = b.build(0);
+ return dict;
+ }
+
void readData(DataInput in) throws IOException {
int rowNum = in.readInt();
rows = new ArrayList<String[]>(rowNum);
if (rowNum > 0) {
int n = in.readInt();
- for (int i = 0; i < rowNum; i++) {
- String[] row = new String[n];
- rows.add(row);
- for (int j = 0; j < n; j++) {
- row[j] = in.readUTF();
+ if (this.useDictionary == Boolean.TRUE) {
+ Dictionary<String> dict = new TrieDictionary<String>();
+ dict.readFields(in);
+ for (int i = 0; i < rowNum; i++) {
+ String[] row = new String[n];
+ rows.add(row);
+ for (int j = 0; j < n; j++) {
+ row[j] = dict.getValueFromId(in.readInt());
+ }
+ }
+ } else {
+ for (int i = 0; i < rowNum; i++) {
+ String[] row = new String[n];
+ rows.add(row);
+ for (int j = 0; j < n; j++) {
+ row[j] = in.readUTF();
+ }
}
}
}
[27/50] [abbrv] incubator-kylin git commit: minor bug fix,
ensure FunctionDesc.returnDataType is updated in sync with returnType
Posted by li...@apache.org.
minor bug fix, ensure FunctionDesc.returnDataType is updated in sync with returnType
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cc1ab7b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cc1ab7b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cc1ab7b9
Branch: refs/heads/0.7
Commit: cc1ab7b9dad6cf63f54a9c3c29ca89c6c528a22b
Parents: 5cc130d
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Jul 6 11:12:06 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 6 11:12:06 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 17 +++++-----
.../apache/kylin/cube/model/v1/CubeDesc.java | 17 +++++-----
.../kylin/invertedindex/model/IIDesc.java | 23 +++++++------
.../kylin/metadata/model/FunctionDesc.java | 18 ++++++-----
.../endpoint/EndpointTupleIterator.java | 34 +++++++++-----------
5 files changed, 56 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc1ab7b9/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index cae9c01..771c989 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -36,13 +36,6 @@ import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.util.Base64;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -53,7 +46,6 @@ import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -61,6 +53,13 @@ import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
/**
* Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
* change this template use File | Settings | File Templates.
@@ -625,7 +624,7 @@ public class CubeDesc extends RootPersistentEntity {
FunctionDesc f = m.getFunction();
f.setExpression(f.getExpression().toUpperCase());
- f.setReturnDataType(DataType.getInstance(f.getReturnType()));
+ f.initReturnDataType();
ParameterDesc p = f.getParameter();
p.normalizeColumnValue();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc1ab7b9/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
index 1564899..0103523 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
@@ -35,13 +35,6 @@ import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.net.util.Base64;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -55,7 +48,6 @@ import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -63,6 +55,13 @@ import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
/**
* Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
* change this template use File | Settings | File Templates.
@@ -716,7 +715,7 @@ public class CubeDesc extends RootPersistentEntity {
FunctionDesc f = m.getFunction();
f.setExpression(f.getExpression().toUpperCase());
- f.setReturnDataType(DataType.getInstance(f.getReturnType()));
+ f.initReturnDataType();
ParameterDesc p = f.getParameter();
p.normalizeColumnValue();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc1ab7b9/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index 75cea57..6db6082 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -26,8 +26,21 @@ import java.util.HashSet;
import java.util.List;
import org.apache.commons.net.util.Base64;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -37,14 +50,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.*;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc1ab7b9/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index eda31a1..3a9fde6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -130,14 +130,6 @@ public class FunctionDesc {
this.parameter = parameter;
}
- public DataType getReturnDataType() {
- return returnDataType;
- }
-
- public void setReturnDataType(DataType returnDataType) {
- this.returnDataType = returnDataType;
- }
-
public String getSQLType() {
if (isCountDistinct())
return "any";
@@ -151,8 +143,18 @@ public class FunctionDesc {
return returnType;
}
+ public DataType getReturnDataType() {
+ return returnDataType;
+ }
+
public void setReturnType(String returnType) {
this.returnType = returnType;
+ this.initReturnDataType();
+ }
+
+ // Jackson does not provide object post-processing currently
+ public void initReturnDataType() {
+ this.returnDataType = DataType.getInstance(returnType);
}
public TblColRef selectTblColRef(Collection<TblColRef> metricColumns, String factTableName) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc1ab7b9/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 8b4d8cf..1d2a355 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -19,9 +19,11 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
import java.io.IOException;
-import java.util.*;
-
-import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.HConnection;
@@ -29,7 +31,17 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.index.TableRecord;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
@@ -40,20 +52,9 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import com.google.protobuf.ByteString;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-
/**
* Created by Hongbin Ma(Binmahone) on 12/2/14.
@@ -144,7 +145,6 @@ public class EndpointTupleIterator implements ITupleIterator {
for (FunctionDesc functionDesc : measures) {
if (functionDesc.isCount()) {
functionDesc.setReturnType("bigint");
- functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
} else {
boolean updated = false;
for (TblColRef column : columns) {
@@ -153,10 +153,8 @@ public class EndpointTupleIterator implements ITupleIterator {
//TODO: default precision might need be configurable
String iiDefaultHLLC = "hllc10";
functionDesc.setReturnType(iiDefaultHLLC);
- functionDesc.setReturnDataType(DataType.getInstance(iiDefaultHLLC));
} else {
functionDesc.setReturnType(column.getColumn().getType().toString());
- functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
}
functionDesc.getParameter().setColRefs(ImmutableList.of(column));
updated = true;
[40/50] [abbrv] incubator-kylin git commit: KYLIN-792,
alert style beautify
Posted by li...@apache.org.
KYLIN-792,alert style beautify
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a2bab5f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a2bab5f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a2bab5f2
Branch: refs/heads/0.7
Commit: a2bab5f2edb7f9ec9518a2e7fb4f9bce45baa31b
Parents: f364ffc
Author: jiazhong <ji...@ebay.com>
Authored: Mon Jul 6 15:03:00 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Jul 6 15:16:02 2015 +0800
----------------------------------------------------------------------
webapp/app/less/component.less | 59 ++++++++++++++++++++++++++++++++++++-
1 file changed, 58 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2bab5f2/webapp/app/less/component.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/component.less b/webapp/app/less/component.less
index ec69022..105005d 100644
--- a/webapp/app/less/component.less
+++ b/webapp/app/less/component.less
@@ -855,4 +855,61 @@ span.input-icon {
.cube-dimension .dl-horizontal{
margin-top: 0px !important;
margin-bottom: 0px !important;
-}
\ No newline at end of file
+}
+
+
+//style choosen for bootstrap3
+.chosen-container-single .chosen-single {
+ height: 30px;
+ border-radius: 3px;
+ border: 1px solid #CCCCCC;
+}
+.chosen-container-single .chosen-single span {
+ padding-top: 2px;
+}
+.chosen-container-single .chosen-single div b {
+ margin-top: 2px;
+}
+.chosen-container-active .chosen-single,
+.chosen-container-active.chosen-with-drop .chosen-single {
+ border-color: #ccc;
+ border-color: rgba(82, 168, 236, .8);
+ outline: 0;
+ outline: thin dotted \9;
+ -moz-box-shadow: 0 0 8px rgba(82, 168, 236, .6);
+ box-shadow: 0 0 8px rgba(82, 168, 236, .6)
+}
+
+
+//override callout info in AdinLTE.css
+.callout {
+ margin: 0 0 20px 0 !important;
+ padding: 15px 30px 15px 15px !important;
+ border-left: 5px solid #eee !important;
+ color: black !important;
+}
+
+.callout.callout-info {
+ background-color: #f0f7fd !important;
+ border-color: #d0e3f0 !important;
+ color: black !important;
+}
+
+.callout.callout-danger {
+ background-color: #fcf2f2 !important;
+ border-color: #dFb5b4 !important;
+ color: black !important;
+}
+
+.callout.callout-warning {
+ background-color: #fefbed !important;
+ border-color: #f1e7bc !important;
+ color: black !important;
+}
+
+ul.messenger-theme-ice .messenger-message{
+ color:#666667 !important;
+}
+.messenger-message.alert{
+ background-color: #f6f6f6 !important;
+}
[44/50] [abbrv] incubator-kylin git commit: KYLIN-867 update
HybridManagerTest
Posted by li...@apache.org.
KYLIN-867 update HybridManagerTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d08e6f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d08e6f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d08e6f5
Branch: refs/heads/0.7
Commit: 8d08e6f542f8b431b2f68ca9957c4f99808d6106
Parents: ab3b07d
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jul 6 11:20:25 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 6 16:19:17 2015 +0800
----------------------------------------------------------------------
.../localmeta/hybrid/test_kylin_hybrid_ready.json | 17 +++++++++++++++++
.../kylin/storage/hybrid/HybridManagerTest.java | 10 ++++++----
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d08e6f5/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
new file mode 100644
index 0000000..ea60ae4
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
@@ -0,0 +1,17 @@
+{
+ "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
+ "name": "test_kylin_hybrid_ready",
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready_2_segments"
+ },
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready"
+ }
+ ],
+ "cost": 40,
+ "last_modified": 1420016227424,
+ "create_time": null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d08e6f5/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 8a1bcf0..00b68cf 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -3,7 +3,6 @@ package org.apache.kylin.storage.hybrid;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.metadata.realization.IRealization;
import org.junit.After;
import org.junit.Assert;
@@ -26,12 +25,15 @@ public class HybridManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
- HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_ready");
System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));
- IRealization history = hybridInstance.getRealizations()[0];
+ IRealization[] realizations = hybridInstance.getRealizations();
+ Assert.assertEquals(realizations.length, 2);
- Assert.assertTrue(history instanceof CubeInstance);
+ IRealization lastReal = hybridInstance.getLatestRealization();
+ Assert.assertTrue(lastReal instanceof CubeInstance);
+ Assert.assertEquals(lastReal.getName(), "test_kylin_cube_with_slr_ready_2_segments");
}
[20/50] [abbrv] incubator-kylin git commit: KYLIN-824 Cube Build
fails if lookup table doesn't have any files under HDFS location
Posted by li...@apache.org.
KYLIN-824 Cube Build fails if lookup table doesn't have any files under HDFS location
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/09751393
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/09751393
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/09751393
Branch: refs/heads/0.7
Commit: 09751393f9abe6eec8e087b45d6b422c12b5f105
Parents: e7fcc20
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 30 10:22:11 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jul 1 14:22:53 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HiveClient.java | 21 +++++++++++--
.../org/apache/kylin/dict/lookup/FileTable.java | 18 ++++++++++--
.../org/apache/kylin/dict/lookup/HiveTable.java | 31 +++++++++++++++-----
3 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
index cfeea31..86a403d 100644
--- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
+++ b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -44,6 +45,7 @@ public class HiveClient {
protected HiveConf hiveConf = null;
protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
+ protected String type;
public HiveClient() {
hiveConf = new HiveConf(HiveClient.class);
@@ -86,8 +88,8 @@ public class HiveClient {
/**
*
* @param hql
- * @throws CommandNeedRetryException
- * @throws IOException
+ * @throws org.apache.hadoop.hive.ql.CommandNeedRetryException
+ * @throws java.io.IOException
*/
public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
CommandProcessorResponse response = getDriver().run(hql);
@@ -110,6 +112,7 @@ public class HiveClient {
return metaStoreClient;
}
+
public Table getHiveTable(String database, String tableName) throws Exception {
return getMetaStoreClient().getTable(database, tableName);
}
@@ -155,4 +158,18 @@ public class HiveClient {
return result;
}
+ /**
+ *
+ * @param database
+ * @param tableName
+ * @throws Exception
+ */
+
+ public boolean isNativeTable(String database, String tableName) throws Exception{
+
+ return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName));
+
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
index e6e3de9..a659ca3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
@@ -35,15 +35,24 @@ public class FileTable implements ReadableTable {
String path;
String delim;
int nColumns;
+ boolean nativeTable;
public FileTable(String path, int nColumns) {
- this(path, DELIM_AUTO, nColumns);
+ this(path, DELIM_AUTO, nColumns, true);
}
- public FileTable(String path, String delim, int nColumns) {
+ public FileTable(String path, String delim, int nColumns, boolean nativeTable) {
this.path = path;
this.delim = delim;
this.nColumns = nColumns;
+ this.nativeTable = nativeTable;
+ }
+
+ public FileTable(String path, int nColumns, boolean nativeTable) {
+ this.path = path;
+ this.delim = DELIM_AUTO;
+ this.nColumns = nColumns;
+ this.nativeTable = nativeTable;
}
@Override
@@ -60,7 +69,10 @@ public class FileTable implements ReadableTable {
public TableSignature getSignature() throws IOException {
FileSystem fs = HadoopUtil.getFileSystem(path);
FileStatus status = fs.getFileStatus(new Path(path));
- return new TableSignature(path, status.getLen(), status.getModificationTime());
+ if (nativeTable) {
+ return new TableSignature(path, status.getLen(), status.getModificationTime());
+ }
+ return new TableSignature(path, status.getLen(), System.currentTimeMillis());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
index ae224ac..aa22814 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
@@ -48,6 +48,8 @@ public class HiveTable implements ReadableTable {
private int nColumns;
private String hdfsLocation;
private FileTable fileTable;
+ private HiveClient hiveClient;
+ private boolean nativeTable;
public HiveTable(MetadataManager metaMgr, String table) {
TableDesc tableDesc = metaMgr.getTableDesc(table);
@@ -72,20 +74,26 @@ public class HiveTable implements ReadableTable {
}
private FileTable getFileTable() throws IOException {
- if (fileTable == null) {
- fileTable = new FileTable(getHDFSLocation(true), nColumns);
+ try {
+ if (fileTable == null) {
+ nativeTable = getHiveClient().isNativeTable(database, hiveTable);
+ fileTable = new FileTable(getHDFSLocation(), nColumns, nativeTable);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
}
return fileTable;
}
- public String getHDFSLocation(boolean needFilePath) throws IOException {
+ public String getHDFSLocation() throws IOException {
if (hdfsLocation == null) {
- hdfsLocation = computeHDFSLocation(needFilePath);
+ hdfsLocation = computeHDFSLocation();
}
return hdfsLocation;
}
- private String computeHDFSLocation(boolean needFilePath) throws IOException {
+ private String computeHDFSLocation() throws IOException {
String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable);
if (override != null) {
@@ -95,14 +103,13 @@ public class HiveTable implements ReadableTable {
String hdfsDir = null;
try {
- HiveClient hiveClient = new HiveClient();
- hdfsDir = hiveClient.getHiveTableLocation(database, hiveTable);
+ hdfsDir = getHiveClient().getHiveTableLocation(database, hiveTable);
} catch (Exception e) {
e.printStackTrace();
throw new IOException(e);
}
- if (needFilePath) {
+ if (nativeTable) {
FileSystem fs = HadoopUtil.getFileSystem(hdfsDir);
FileStatus file = findOnlyFile(hdfsDir, fs);
return file.getPath().toString();
@@ -129,4 +136,12 @@ public class HiveTable implements ReadableTable {
return "hive: database=[" + database + "], table=[" + hiveTable + "]";
}
+ public HiveClient getHiveClient() {
+
+ if (hiveClient == null) {
+ hiveClient = new HiveClient();
+ }
+ return hiveClient;
+ }
+
}
[36/50] [abbrv] incubator-kylin git commit: KYLIN-780 JDBC upgrade,
test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
deleted file mode 100644
index 17e8ed4..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
+++ /dev/null
@@ -1,821 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.lang.reflect.Field;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.ColumnMetaData.Rep;
-import org.apache.calcite.avatica.Meta;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.sql.SqlJdbcFunctionCall;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.kylin.jdbc.stub.DataSet;
-import org.apache.kylin.jdbc.stub.KylinColumnMetaData;
-import org.apache.kylin.jdbc.stub.RemoteClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.kylin.jdbc.util.SQLTypeMap;
-
-/**
- * Implementation of avatica interface
- *
- * @author xduo
- */
-public class KylinMetaImpl implements Meta {
-
- private static final Logger logger = LoggerFactory.getLogger(KylinMetaImpl.class);
-
- private final KylinConnectionImpl conn;
-
- private final KylinJdbc41Factory factory;
-
- /**
- * @param conn
- */
- public KylinMetaImpl(KylinConnectionImpl conn, KylinJdbc41Factory factory) {
- super();
- this.conn = conn;
- this.factory = factory;
- }
-
- private ResultSet mockEmptyResultSet() {
- AvaticaResultSet resultSet = null;
- try {
- List<ColumnMetaData> columnMetas = new ArrayList<ColumnMetaData>();
- List<Object[]> data = new ArrayList<Object[]>();
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, new KylinEnumerator<Object[]>(data), ColumnMetaData.struct(columnMetas)), this.conn.getTimeZone());
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- }
-
- public String getSqlKeywords() {
- return SqlParser.create("").getMetadata().getJdbcKeywords();
- }
-
- public String getNumericFunctions() {
- return SqlJdbcFunctionCall.getNumericFunctions();
- }
-
- public String getStringFunctions() {
- return SqlJdbcFunctionCall.getStringFunctions();
- }
-
- public String getSystemFunctions() {
- return SqlJdbcFunctionCall.getSystemFunctions();
- }
-
- public String getTimeDateFunctions() {
- return SqlJdbcFunctionCall.getTimeDateFunctions();
- }
-
- public ResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
- logger.debug("Get tables with conn " + conn);
- MetaProject metaProject = conn.getMetaProject();
-
- if (null != metaProject) {
- final DataSet<MetaTable> tables = metaProject.getMetaTables(catalog, schemaPattern, tableNamePattern);
- final NamedFieldGetter<MetaTable> tableGetter = new NamedFieldGetter<MetaTable>(MetaTable.class, tables.getMeta(), "TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS", "TYPE_CAT", "TYPE_SCHEM", "TYPE_NAME", "SELF_REFERENCING_COL_NAME", "REF_GENERATION");
-
- AvaticaResultSet resultSet = null;
- try {
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, null, tableGetter.structType) {
- @Override
- public Cursor createCursor() {
- return tableGetter.cursor(tables.getEnumerator());
- }
- }, this.conn.getTimeZone());
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- } else {
- return mockEmptyResultSet();
- }
- }
-
- public ResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
- logger.debug("Get columns with conn " + conn);
- MetaProject metaProject = conn.getMetaProject();
-
- if (null != metaProject) {
- final DataSet<MetaColumn> columns = metaProject.getMetaColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern);
- final NamedFieldGetter<MetaColumn> columnGetter = new NamedFieldGetter<MetaColumn>(MetaColumn.class, columns.getMeta(), "TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "DATA_TYPE", "TYPE_NAME", "COLUMN_SIZE", "BUFFER_LENGTH", "DECIMAL_DIGITS", "NUM_PREC_RADIX", "NULLABLE", "REMARKS", "COLUMN_DEF", "SQL_DATA_TYPE", "SQL_DATETIME_SUB", "CHAR_OCTET_LENGTH", "ORDINAL_POSITION", "IS_NULLABLE", "SCOPE_CATALOG", "SCOPE_TABLE", "SOURCE_DATA_TYPE", "IS_AUTOINCREMENT", "IS_GENERATEDCOLUMN");
-
- AvaticaResultSet resultSet = null;
- try {
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, null, columnGetter.structType) {
- @Override
- public Cursor createCursor() {
- return columnGetter.cursor(columns.getEnumerator());
- }
- }, this.conn.getTimeZone());
-
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- } else {
- return mockEmptyResultSet();
- }
- }
-
- public ResultSet getSchemas(String catalog, Pat schemaPattern) {
- logger.debug("Get schemas with conn " + conn);
- MetaProject metaProject = conn.getMetaProject();
-
- if (null != metaProject) {
- final DataSet<MetaSchema> schemas = metaProject.getMetaSchemas(catalog, schemaPattern);
- final NamedFieldGetter<MetaSchema> schemaGetter = new NamedFieldGetter<MetaSchema>(MetaSchema.class, schemas.getMeta(), "TABLE_SCHEM", "TABLE_CATALOG");
-
- AvaticaResultSet resultSet = null;
- try {
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, null, schemaGetter.structType) {
- @Override
- public Cursor createCursor() {
- return schemaGetter.cursor(schemas.getEnumerator());
- }
- }, this.conn.getTimeZone());
-
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- } else {
- return mockEmptyResultSet();
- }
- }
-
- public ResultSet getCatalogs() {
- MetaProject metaProject = conn.getMetaProject();
-
- if (null != metaProject) {
- final DataSet<MetaCatalog> catalogs = metaProject.getMetaCatalogs();
- final NamedFieldGetter<MetaCatalog> catalogGetter = new NamedFieldGetter<MetaCatalog>(MetaCatalog.class, catalogs.getMeta(), "TABLE_CATALOG");
-
- AvaticaResultSet resultSet = null;
- try {
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, null, catalogGetter.structType) {
- @Override
- public Cursor createCursor() {
- return catalogGetter.cursor(catalogs.getEnumerator());
- }
- }, this.conn.getTimeZone());
-
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- } else {
- return mockEmptyResultSet();
- }
- }
-
- public ResultSet getTableTypes() {
- List<ColumnMetaData> tableTypeMeta = new ArrayList<ColumnMetaData>();
- tableTypeMeta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), false));
- List<Object[]> data = new ArrayList<Object[]>();
- Object[] row = new Object[1];
- row[0] = "TABLE";
- data.add(row);
-
- AvaticaResultSet resultSet = null;
- try {
- resultSet = this.conn.getFactory().newResultSet(this.conn.createStatement(), new KylinPrepare.PrepareResult(null, null, new KylinEnumerator<Object[]>(data), ColumnMetaData.struct(tableTypeMeta)), this.conn.getTimeZone());
- KylinConnectionImpl.TROJAN.execute(resultSet);
- } catch (SQLException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
-
- return resultSet;
- }
-
- public ResultSet getProcedures(String catalog, Pat schemaPattern, Pat procedureNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getProcedureColumns(String catalog, Pat schemaPattern, Pat procedureNamePattern, Pat columnNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getColumnPrivileges(String catalog, String schema, String table, Pat columnNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getTablePrivileges(String catalog, Pat schemaPattern, Pat tableNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getVersionColumns(String catalog, String schema, String table) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getPrimaryKeys(String catalog, String schema, String table) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getImportedKeys(String catalog, String schema, String table) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getExportedKeys(String catalog, String schema, String table) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, String foreignCatalog, String foreignSchema, String foreignTable) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getTypeInfo() {
- return mockEmptyResultSet();
- }
-
- public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getUDTs(String catalog, Pat schemaPattern, Pat typeNamePattern, int[] types) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getSuperTypes(String catalog, Pat schemaPattern, Pat typeNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getSuperTables(String catalog, Pat schemaPattern, Pat tableNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getAttributes(String catalog, Pat schemaPattern, Pat typeNamePattern, Pat attributeNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getClientInfoProperties() {
- return mockEmptyResultSet();
- }
-
- public ResultSet getFunctions(String catalog, Pat schemaPattern, Pat functionNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getFunctionColumns(String catalog, Pat schemaPattern, Pat functionNamePattern, Pat columnNamePattern) {
- return mockEmptyResultSet();
- }
-
- public ResultSet getPseudoColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
- return mockEmptyResultSet();
- }
-
- public Cursor createCursor(AvaticaResultSet resultSet) {
-
- if (!(resultSet instanceof KylinResultSet))
- throw new IllegalStateException("resultSet is not KylinResultSet");
-
- KylinPrepare.PrepareResult result = ((KylinResultSet) resultSet).getPrepareResult();
-
- return result.createCursor();
- }
-
- /*
- * Client could request metadata after prepare
- *
- * (non-Javadoc)
- * @see org.apache.calcite.avatica.Meta#prepare(org.apache.calcite.avatica.AvaticaStatement, java.lang.String)
- */
- public AvaticaPrepareResult prepare(AvaticaStatement statement, String sql) {
- RemoteClient client = factory.newRemoteClient(conn);
- DataSet<Object[]> result = null;
-
- try {
- result = (DataSet<Object[]>) client.query(statement, sql);
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new RuntimeException("Failed to query kylin server with exception " + e.getLocalizedMessage());
- }
-
- return new KylinPrepare.PrepareResult(sql, null, (Enumerator<Object[]>) result.getEnumerator(), ColumnMetaData.struct(result.getMeta()));
- }
-
- /**
- * Tree node used by project tree-like structure
- *
- * @author xduo
- */
- interface Node {
- /**
- * Get the node name
- *
- * @return
- */
- public String getName();
-
- /**
- * Get direct children of the node.
- *
- * @return
- */
- public List<? extends Node> getChildren();
-
- /**
- * Search the subtree of the node with patterns. One pattern, one level.
- *
- * @param patterns
- * @return
- */
- public List<? extends Node> searchByPatterns(Pat... patterns);
- }
-
- /**
- * Abstract of the tree-like structure
- *
- * @author xduo
- */
- public static abstract class AbstractNode implements Node {
-
- public List<? extends Node> searchByPatterns(Pat... patterns) {
- if (patterns.length == 1) {
- return findChildren(patterns[0]);
- } else {
- List<Node> children = new ArrayList<Node>();
-
- for (Node child : this.findChildren(patterns[0])) {
- children.addAll(child.searchByPatterns(Arrays.copyOfRange(patterns, 1, patterns.length)));
- }
-
- return children;
- }
- }
-
- private List<? extends Node> findChildren(Pat pattern) {
- if (null == pattern.s || pattern.s.equals("%")) {
- return this.getChildren();
- }
-
- List<Node> list = new ArrayList<Node>();
-
- for (Node c : this.getChildren()) {
- if (likeToRegex(pattern).matcher(c.getName()).matches()) {
- list.add(c);
- }
- }
-
- return list;
- }
-
- ;
-
- /**
- * Converts a LIKE-style pattern (where '%' represents a wild-card,
- * escaped using '\') to a Java regex.
- */
- private Pattern likeToRegex(Pat pattern) {
- StringBuilder buf = new StringBuilder("^");
- char[] charArray = pattern.s.toCharArray();
- int slash = -2;
- for (int i = 0; i < charArray.length; i++) {
- char c = charArray[i];
- if (slash == i - 1) {
- buf.append('[').append(c).append(']');
- } else {
- switch (c) {
- case '\\':
- slash = i;
- break;
- case '%':
- buf.append(".*");
- break;
- case '[':
- buf.append("\\[");
- break;
- case ']':
- buf.append("\\]");
- break;
- default:
- buf.append('[').append(c).append(']');
- }
- }
- }
- buf.append("$");
-
- return Pattern.compile(buf.toString());
- }
- }
-
- public static class MetaProject extends AbstractNode {
- public final String project;
- public final List<MetaCatalog> catalogs;
-
- public MetaProject(String project, List<MetaCatalog> catalogs) {
- super();
- this.project = project;
- this.catalogs = catalogs;
- }
-
- public DataSet<MetaCatalog> getMetaCatalogs() {
- return new DataSet<MetaCatalog>(MetaCatalog.meta, new KylinEnumerator<MetaCatalog>(catalogs));
- }
-
- /**
- * facade method to search schemas in current project.
- *
- * @param catalog
- * @param schemaPattern
- * @return
- */
- @SuppressWarnings("unchecked")
- public DataSet<MetaSchema> getMetaSchemas(String catalog, Pat schemaPattern) {
- List<? extends Node> metaSchemas = this.searchByPatterns(Pat.of(catalog), schemaPattern);
-
- return new DataSet<MetaSchema>(MetaSchema.meta, new KylinEnumerator<MetaSchema>((Collection<MetaSchema>) metaSchemas));
- }
-
- /**
- * facade method to search tables in current project
- *
- * @param catalog
- * @param schemaPattern
- * @param tableNamePattern
- * @return
- */
- @SuppressWarnings("unchecked")
- public DataSet<MetaTable> getMetaTables(String catalog, Pat schemaPattern, Pat tableNamePattern) {
- logger.debug("getMetaTables with catalog:" + catalog + ", schema:" + schemaPattern.s + ", table:" + tableNamePattern.s);
- List<? extends Node> tables = this.searchByPatterns(Pat.of(catalog), schemaPattern, tableNamePattern);
-
- return new DataSet<MetaTable>(MetaTable.meta, new KylinEnumerator<MetaTable>((Collection<MetaTable>) tables));
- }
-
- /**
- * facade method to search columns in current project
- *
- * @param catalog
- * @param schemaPattern
- * @param tableNamePattern
- * @param columnNamePattern
- * @return
- */
- @SuppressWarnings("unchecked")
- public DataSet<MetaColumn> getMetaColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
- logger.debug("getMetaColumns with catalog:" + catalog + ", schema:" + schemaPattern.s + ", table:" + tableNamePattern.s + ", column:" + columnNamePattern.s);
- List<? extends Node> columns = this.searchByPatterns(Pat.of(catalog), schemaPattern, tableNamePattern, columnNamePattern);
-
- return new DataSet<MetaColumn>(MetaColumn.meta, new KylinEnumerator<MetaColumn>((Collection<MetaColumn>) columns));
- }
-
- @Override
- public String getName() {
- return project;
- }
-
- @Override
- public List<? extends Node> getChildren() {
- return this.catalogs;
- }
- }
-
- /**
- * Metadata describing a catalog.
- */
- public static class MetaCatalog extends AbstractNode {
- public static final List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- public final String tableCatalog;
- public final List<MetaSchema> schemas;
-
- static {
- meta.add(KylinColumnMetaData.dummy(0, "TABLE_CAT", "TABLE_CAT", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- }
-
- public MetaCatalog(String tableCatalog, List<MetaSchema> schemas) {
- this.tableCatalog = tableCatalog;
- this.schemas = schemas;
- }
-
- public String getName() {
- return tableCatalog;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((tableCatalog == null) ? 0 : tableCatalog.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- MetaCatalog other = (MetaCatalog) obj;
- if (tableCatalog == null) {
- if (other.tableCatalog != null)
- return false;
- } else if (!tableCatalog.equals(other.tableCatalog))
- return false;
- return true;
- }
-
- @Override
- public List<? extends Node> getChildren() {
- return schemas;
- }
-
- }
-
- /**
- * Metadata describing a schema.
- */
- public static class MetaSchema extends AbstractNode {
- public static final List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- public final String tableCatalog;
- public final String tableSchem;
- public final List<MetaTable> tables;
-
- static {
- for (ColumnMetaData cmd : SQLTypeMap.schemaMetaTypeMapping.values()) {
- meta.add(cmd);
- }
- }
-
- public MetaSchema(String tableCatalog, String tableSchem, List<MetaTable> tables) {
- this.tableCatalog = tableCatalog;
- this.tableSchem = tableSchem;
- this.tables = tables;
- }
-
- public String getName() {
- return tableSchem;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((tableCatalog == null) ? 0 : tableCatalog.hashCode());
- result = prime * result + ((tableSchem == null) ? 0 : tableSchem.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- MetaSchema other = (MetaSchema) obj;
- if (tableCatalog == null) {
- if (other.tableCatalog != null)
- return false;
- } else if (!tableCatalog.equals(other.tableCatalog))
- return false;
- if (tableSchem == null) {
- if (other.tableSchem != null)
- return false;
- } else if (!tableSchem.equals(other.tableSchem))
- return false;
- return true;
- }
-
- @Override
- public List<MetaTable> getChildren() {
- return this.tables;
- }
- }
-
- /**
- * Metadata describing a table type.
- */
- public static class MetaTableType {
- public final String tableType;
-
- public MetaTableType(String tableType) {
- this.tableType = tableType;
- }
- }
-
- /**
- * Metadata describing a table.
- */
- public static class MetaTable extends AbstractNode {
- public static final List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- public final String tableCat;
- public final String tableSchem;
- public final String tableName;
- public final String tableType;
- public final String remarks;
- public final String typeCat;
- public final String typeSchem;
- public final String typeName;
- public final String selfReferencingColName;
- public final String refGeneration;
- public final List<MetaColumn> columns;
-
- static {
- for (ColumnMetaData cmd : SQLTypeMap.tableMetaTypeMapping.values()) {
- meta.add(cmd);
- }
- }
-
- public MetaTable(String tableCat, String tableSchem, String tableName, String tableType, String remarks, String typeCat, String typeSchem, String typeName, String selfReferencingColName, String refGeneration, List<MetaColumn> columns) {
- this.tableCat = tableCat;
- this.tableSchem = tableSchem;
- this.tableName = tableName;
- this.tableType = tableType;
- this.remarks = remarks;
- this.typeCat = typeCat;
- this.typeSchem = typeSchem;
- this.typeName = typeName;
- this.selfReferencingColName = selfReferencingColName;
- this.refGeneration = refGeneration;
- this.columns = columns;
- }
-
- public String getName() {
- return tableName;
- }
-
- @Override
- public List<? extends Node> getChildren() {
- return this.columns;
- }
- }
-
- /**
- * Metadata describing a column.
- */
- public static class MetaColumn implements Node {
- public static final List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- public final String tableCat;
- public final String tableSchem;
- public final String tableName;
- public final String columnName;
- public final int dataType;
- public final String typeName;
- public final int columnSize;
- public final int bufferLength;
- public final int decimalDigits;
- public final int numPrecRadix;
- public final int nullable;
- public final String remarks;
- public final String columnDef;
- public final int sqlDataType;
- public final int sqlDatetimeSub;
- public final int charOctetLength;
- public final int ordinalPosition;
- public final String isNullable;
- public final String scopeCatalog;
- public final String scopeTable;
- public final int sourceDataType;
- public final String isAutoincrement;
- public final String isGeneratedcolumn;
-
- static {
- for (ColumnMetaData cmd : SQLTypeMap.columnMetaTypeMapping.values()) {
- meta.add(cmd);
- }
- }
-
- public MetaColumn(String tableCat, String tableSchem, String tableName, String columnName, int dataType, String typeName, int columnSize, int bufferLength, int decimalDigits, int numPrecRadix, int nullable, String remarks, String columnDef, int sqlDataType, int sqlDatetimeSub, int charOctetLength, int ordinalPosition, String isNullable, String scopeCatalog, String scopeTable, int sourceDataType, String isAutoincrement, String isGeneratedcolumn) {
- super();
- this.tableCat = tableCat;
- this.tableSchem = tableSchem;
- this.tableName = tableName;
- this.columnName = columnName;
- this.dataType = dataType;
- this.typeName = typeName;
- this.columnSize = columnSize;
- this.bufferLength = bufferLength;
- this.decimalDigits = decimalDigits;
- this.numPrecRadix = numPrecRadix;
- this.nullable = nullable;
- this.remarks = remarks;
- this.columnDef = columnDef;
- this.sqlDataType = sqlDataType;
- this.sqlDatetimeSub = sqlDatetimeSub;
- this.charOctetLength = charOctetLength;
- this.ordinalPosition = ordinalPosition;
- this.isNullable = isNullable;
- this.scopeCatalog = scopeCatalog;
- this.scopeTable = scopeTable;
- this.sourceDataType = sourceDataType;
- this.isAutoincrement = isAutoincrement;
- this.isGeneratedcolumn = isGeneratedcolumn;
- }
-
- public String getName() {
- return columnName;
- }
-
- @Override
- public List<? extends Node> getChildren() {
- return Collections.emptyList();
- }
-
- @Override
- public List<? extends Node> searchByPatterns(Pat... patterns) {
- return Collections.emptyList();
- }
- }
-
- /**
- * Accesses fields by name.
- */
- private static class NamedFieldGetter<T> {
- private final List<Field> fields = new ArrayList<Field>();
- private final ColumnMetaData.StructType structType;
-
- public NamedFieldGetter(Class<T> clazz, List<ColumnMetaData> columns, String... names) {
- init(clazz, names, fields);
- structType = ColumnMetaData.struct(columns);
- }
-
- private void init(Class<T> clazz, String[] names, List<Field> fields) {
- for (String name : names) {
- final String fieldName = Util.toCamelCase(name);
- final Field field;
- try {
- field = clazz.getField(fieldName);
- } catch (NoSuchFieldException e) {
- throw new RuntimeException(e);
- }
- fields.add(field);
- }
- }
-
- Object get(Object o, int columnIndex) {
- try {
- return fields.get(columnIndex).get(o);
- } catch (IllegalArgumentException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- public Cursor cursor(Enumerator<T> enumerator) {
- // noinspection unchecked
- return new EnumeratorCursor<T>(enumerator) {
- protected Getter createGetter(final int ordinal) {
- return new Getter() {
- public Object getObject() {
- return get(current(), ordinal);
- }
-
- public boolean wasNull() {
- return getObject() == null;
- }
- };
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
deleted file mode 100644
index 686e948..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.AvaticaPrepareResult;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.Cursor;
-
-/**
- * Interface of kylin prepare statement implementation
- *
- * @author xduo
- *
- */
-public interface KylinPrepare {
-
- PrepareResult prepare(String sql);
-
- /**
- * The result of preparing a query. It gives the Avatica driver framework
- * the information it needs to create a prepared statement, or to execute a
- * statement directly, without an explicit prepare step.
- */
- public static class PrepareResult implements AvaticaPrepareResult {
- public final String sql; // for debug
- public final ColumnMetaData.StructType structType;
- public final Enumerator<Object[]> enumerator;
- public final List<AvaticaParameter> parameterList;
-
- public PrepareResult(String sql, List<AvaticaParameter> parameterList, Enumerator<Object[]> enumerator, ColumnMetaData.StructType structType) {
- super();
- this.sql = sql;
- this.parameterList = parameterList;
- this.enumerator = enumerator;
- this.structType = structType;
- }
-
- public Cursor createCursor() {
- return new EnumeratorCursor<Object[]>(enumerator) {
- @Override
- protected Getter createGetter(int ordinal) {
- return new ArrayEnumeratorGetter(ordinal);
- }
-
- /**
- * Row field accessor via index
- */
- class ArrayEnumeratorGetter extends AbstractGetter {
- protected final int field;
-
- public ArrayEnumeratorGetter(int field) {
- this.field = field;
- }
-
- public Object getObject() {
- Object o = current()[field];
- wasNull[0] = o == null;
- return o;
- }
- }
- };
- }
-
- public List<ColumnMetaData> getColumnList() {
- return structType.columns;
- }
-
- public List<AvaticaParameter> getParameterList() {
- return parameterList;
- }
-
- public Map<String, Object> getInternalParameters() {
- return null;
- }
-
- public String getSql() {
- return sql;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareImpl.java
deleted file mode 100644
index 5aeeb42..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareImpl.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import net.hydromatic.avatica.AvaticaParameter;
-import net.hydromatic.avatica.ColumnMetaData;
-
-/**
- * @author xduo
- *
- */
-public class KylinPrepareImpl implements KylinPrepare {
-
- @Override
- public PrepareResult prepare(String sql) {
- List<AvaticaParameter> aps = new ArrayList<AvaticaParameter>();
-
- int startIndex = 0;
- while (sql.indexOf("?", startIndex) >= 0) {
- AvaticaParameter ap = new AvaticaParameter(false, 0, 0, 0, null, null, null);
- aps.add(ap);
- startIndex = sql.indexOf("?", startIndex) + 1;
- }
-
- return new KylinPrepare.PrepareResult(sql, aps, null, ColumnMetaData.struct(new ArrayList<ColumnMetaData>()));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareStatementImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareStatementImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareStatementImpl.java
deleted file mode 100644
index a2818d8..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepareStatementImpl.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.io.InputStream;
-import java.io.Reader;
-import java.sql.NClob;
-import java.sql.ResultSet;
-import java.sql.RowId;
-import java.sql.SQLException;
-import java.sql.SQLXML;
-import java.util.Collections;
-import java.util.List;
-
-import net.hydromatic.avatica.AvaticaConnection;
-import net.hydromatic.avatica.AvaticaPrepareResult;
-import net.hydromatic.avatica.AvaticaPreparedStatement;
-import net.hydromatic.avatica.AvaticaResultSet;
-
-/**
- * Kylin prepare statement. <br>
- * Supported operations:
- * <ul>
- * <li>setString</li>
- * <li>setInt</li>
- * <li>setShort</li>
- * <li>setLong</li>
- * <li>setFloat</li>
- * <li>setDouble</li>
- * <li>setBoolean</li>
- * <li>setByte</li>
- * <li>setDate</li>
- * <li>setTime</li>
- * <li>setTimestamp</li>
- * </ul>
- *
- * @author xduo
- *
- */
-public abstract class KylinPrepareStatementImpl extends AvaticaPreparedStatement {
-
- /**
- * Before real query,
- */
- protected AvaticaPrepareResult prequeryResult;
-
- protected KylinPrepareStatementImpl(AvaticaConnection connection, AvaticaPrepareResult prepareResult, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- super(connection, prepareResult, resultSetType, resultSetConcurrency, resultSetHoldability);
-
- this.prequeryResult = prepareResult;
- }
-
- @Override
- public ResultSet executeQuery() throws SQLException {
- AvaticaPrepareResult queriedResult = ((KylinConnectionImpl) this.connection).getMeta().prepare(this, this.prequeryResult.getSql());
-
- return executeQueryInternal(queriedResult);
- }
-
- @Override
- protected void close_() {
- if (!closed) {
- closed = true;
- final KylinConnectionImpl connection_ = (KylinConnectionImpl) connection;
- connection_.statements.remove(this);
- if (openResultSet != null) {
- AvaticaResultSet c = openResultSet;
- openResultSet = null;
- c.close();
- }
- // If onStatementClose throws, this method will throw an
- // exception (later
- // converted to SQLException), but this statement still gets
- // closed.
- connection_.getDriver().handler.onStatementClose(this);
- }
- }
-
- public List<Object> getParameterValues() {
- return (List<Object>) Collections.unmodifiableList(super.getParameterValues());
- }
-
- public void setRowId(int parameterIndex, RowId x) throws SQLException {
- getParameter(parameterIndex).setRowId(x);
- }
-
- public void setNString(int parameterIndex, String value) throws SQLException {
- getParameter(parameterIndex).setNString(value);
- }
-
- public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
- getParameter(parameterIndex).setNCharacterStream(value, length);
- }
-
- public void setNClob(int parameterIndex, NClob value) throws SQLException {
- getParameter(parameterIndex).setNClob(value);
- }
-
- public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setClob(reader, length);
- }
-
- public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
- getParameter(parameterIndex).setBlob(inputStream, length);
- }
-
- public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setNClob(reader, length);
- }
-
- public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
- getParameter(parameterIndex).setSQLXML(xmlObject);
- }
-
- public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getParameter(parameterIndex).setAsciiStream(x, length);
- }
-
- public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getParameter(parameterIndex).setBinaryStream(x, length);
- }
-
- public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setCharacterStream(reader, length);
- }
-
- public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
- getParameter(parameterIndex).setAsciiStream(x);
- }
-
- public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
- getParameter(parameterIndex).setBinaryStream(x);
- }
-
- public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setCharacterStream(reader);
- }
-
- public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
- getParameter(parameterIndex).setNCharacterStream(value);
- }
-
- public void setClob(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setClob(reader);
- }
-
- public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
- getParameter(parameterIndex).setBlob(inputStream);
- }
-
- public void setNClob(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setNClob(reader);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
new file mode 100644
index 0000000..21cb162
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
@@ -0,0 +1,100 @@
+package org.apache.kylin.jdbc;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.sql.NClob;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLXML;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaPreparedStatement;
+import org.apache.calcite.avatica.Meta.Signature;
+import org.apache.calcite.avatica.Meta.StatementHandle;
+
+public class KylinPreparedStatement extends AvaticaPreparedStatement {
+
+ protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ protected List<Object> getParameterValues() {
+ return Arrays.asList(slots);
+ }
+
+ // ============================================================================
+
+ public void setRowId(int parameterIndex, RowId x) throws SQLException {
+ getParameter(parameterIndex).setRowId(slots, parameterIndex, x);
+ }
+
+ public void setNString(int parameterIndex, String value) throws SQLException {
+ getParameter(parameterIndex).setNString(slots, parameterIndex, value);
+ }
+
+ public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
+ getParameter(parameterIndex).setNCharacterStream(slots, parameterIndex, value, length);
+ }
+
+ public void setNClob(int parameterIndex, NClob value) throws SQLException {
+ getParameter(parameterIndex).setNClob(slots, parameterIndex, value);
+ }
+
+ public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setClob(slots, parameterIndex, reader, length);
+ }
+
+ public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
+ getParameter(parameterIndex).setBlob(slots, parameterIndex, inputStream, length);
+ }
+
+ public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setNClob(slots, parameterIndex, reader, length);
+ }
+
+ public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
+ getParameter(parameterIndex).setSQLXML(slots, parameterIndex, xmlObject);
+ }
+
+ public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
+ getParameter(parameterIndex).setAsciiStream(slots, parameterIndex, x, length);
+ }
+
+ public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
+ getParameter(parameterIndex).setBinaryStream(slots, parameterIndex, x, length);
+ }
+
+ public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
+ getParameter(parameterIndex).setCharacterStream(slots, parameterIndex, reader, length);
+ }
+
+ public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
+ getParameter(parameterIndex).setAsciiStream(slots, parameterIndex, x);
+ }
+
+ public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
+ getParameter(parameterIndex).setBinaryStream(slots, parameterIndex, x);
+ }
+
+ public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setCharacterStream(slots, parameterIndex, reader);
+ }
+
+ public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
+ getParameter(parameterIndex).setNCharacterStream(slots, parameterIndex, value);
+ }
+
+ public void setClob(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setClob(slots, parameterIndex, reader);
+ }
+
+ public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
+ getParameter(parameterIndex).setBlob(slots, parameterIndex, inputStream);
+ }
+
+ public void setNClob(int parameterIndex, Reader reader) throws SQLException {
+ getParameter(parameterIndex).setNClob(slots, parameterIndex, reader);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 7cb9b25..1229d10 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -1,45 +1,72 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.sql.ResultSetMetaData;
-import java.util.TimeZone;
-
-import org.apache.calcite.avatica.AvaticaPrepareResult;
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaStatement;
-
-import org.apache.kylin.jdbc.KylinPrepare.PrepareResult;
-
-/**
- * Kylin query result set
- *
- * @author xduo
- *
- */
-public class KylinResultSet extends AvaticaResultSet {
-
- public KylinResultSet(AvaticaStatement statement, AvaticaPrepareResult prepareResult, ResultSetMetaData resultSetMetaData, TimeZone timeZone) {
- super(statement, prepareResult, resultSetMetaData, timeZone);
- }
-
- public KylinPrepare.PrepareResult getPrepareResult() {
- return (PrepareResult) prepareResult;
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.io.IOException;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta.Signature;
+import org.apache.calcite.avatica.MetaImpl;
+import org.apache.kylin.jdbc.IRemoteClient.QueryResult;
+
+public class KylinResultSet extends AvaticaResultSet {
+
+ public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Iterable<Object> iterable) {
+ super(statement, signature, resultSetMetaData, timeZone, iterable);
+ }
+
+ @Override
+ protected AvaticaResultSet execute() throws SQLException {
+
+ // skip execution if result is already there (case of meta data lookup)
+ if (this.iterable != null) {
+ return super.execute();
+ }
+
+ String sql = signature.sql;
+ List<AvaticaParameter> params = signature.parameters;
+ List<Object> paramValues = null;
+ if (params != null && params.size() > 0) {
+ paramValues = ((KylinPreparedStatement) statement).getParameterValues();
+ }
+
+ IRemoteClient client = ((KylinConnection) statement.connection).getRemoteClient();
+ QueryResult result;
+ try {
+ result = client.executeQuery(sql, params, paramValues);
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+
+ columnMetaDataList.clear();
+ columnMetaDataList.addAll(result.columnMeta);
+
+ cursor = MetaImpl.createCursor(signature.cursorFactory, result.iterable);
+ return super.execute2(cursor, columnMetaDataList);
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
new file mode 100644
index 0000000..6596389
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta.StatementHandle;
+
+public class KylinStatement extends AvaticaStatement {
+
+ protected KylinStatement(AvaticaConnection connection, StatementHandle h, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
+ super(connection, h, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
deleted file mode 100644
index 0d7605d..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaStatement;
-
-/**
- * Kylin statement implementation
- *
- * @author xduo
- *
- */
-public abstract class KylinStatementImpl extends AvaticaStatement {
-
- protected KylinStatementImpl(AvaticaConnection connection, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
- super(connection, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- protected void close_() {
- if (!closed) {
- closed = true;
- final KylinConnectionImpl connection_ = (KylinConnectionImpl) connection;
- connection_.statements.remove(this);
- if (openResultSet != null) {
- AvaticaResultSet c = openResultSet;
- openResultSet = null;
- c.close();
- }
- // If onStatementClose throws, this method will throw an exception
- // (later
- // converted to SQLException), but this statement still gets closed.
- connection_.getDriver().handler.onStatementClose(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
new file mode 100644
index 0000000..1f37bfd
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc.json;
+
+import java.util.List;
+
+public class PreparedQueryRequest extends QueryRequest {
+ private List<StatementParameter> params;
+
+ public List<StatementParameter> getParams() {
+ return params;
+ }
+
+ public void setParams(List<StatementParameter> params) {
+ this.params = params;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
new file mode 100644
index 0000000..4878aba
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc.json;
+
+public class QueryRequest {
+ private String sql;
+ private String project;
+
+ public String getSql() {
+ return sql;
+ }
+
+ public void setSql(String sql) {
+ this.sql = sql;
+ }
+
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
new file mode 100644
index 0000000..f34e48c
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc.json;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ */
+public class SQLResponseStub implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ // private static final Logger logger =
+ // LoggerFactory.getLogger(SQLResponse.class);
+
+ // the data type for each column
+ private List<ColumnMetaStub> columnMetas;
+
+ // the results rows, each row contains several columns
+ private List<String[]> results;
+
+ private String cube;
+
+ // if not select query, only return affected row count
+ private int affectedRowCount;
+
+ // if isException, the detailed exception message
+ private String exceptionMessage;
+
+ private boolean isException;
+
+ private long duration;
+
+ private boolean isPartial = false;
+
+ private long totalScanCount;
+
+ private boolean hitCache = false;
+
+ public SQLResponseStub() {
+ }
+
+ public List<ColumnMetaStub> getColumnMetas() {
+ return columnMetas;
+ }
+
+ public void setColumnMetas(List<ColumnMetaStub> columnMetas) {
+ this.columnMetas = columnMetas;
+ }
+
+ public List<String[]> getResults() {
+ return results;
+ }
+
+ public void setResults(List<String[]> results) {
+ this.results = results;
+ }
+
+ public String getCube() {
+ return cube;
+ }
+
+ public void setCube(String cube) {
+ this.cube = cube;
+ }
+
+ public int getAffectedRowCount() {
+ return affectedRowCount;
+ }
+
+ public void setAffectedRowCount(int affectedRowCount) {
+ this.affectedRowCount = affectedRowCount;
+ }
+
+ public boolean getIsException() {
+ return isException;
+ }
+
+ public void setIsException(boolean isException) {
+ this.isException = isException;
+ }
+
+ public String getExceptionMessage() {
+ return exceptionMessage;
+ }
+
+ public void setExceptionMessage(String exceptionMessage) {
+ this.exceptionMessage = exceptionMessage;
+ }
+
+ public long getDuration() {
+ return duration;
+ }
+
+ public void setDuration(long duration) {
+ this.duration = duration;
+ }
+
+ public boolean isPartial() {
+ return isPartial;
+ }
+
+ public void setPartial(boolean isPartial) {
+ this.isPartial = isPartial;
+ }
+
+ public long getTotalScanCount() {
+ return totalScanCount;
+ }
+
+ public void setTotalScanCount(long totalScanCount) {
+ this.totalScanCount = totalScanCount;
+ }
+
+ public boolean isHitCache() {
+ return hitCache;
+ }
+
+ public void setHitCache(boolean hitCache) {
+ this.hitCache = hitCache;
+ }
+
+ public static long getSerialversionuid() {
+ return serialVersionUID;
+ }
+
+ public static class ColumnMetaStub {
+
+ private boolean isAutoIncrement;
+ private boolean isCaseSensitive;
+ private boolean isSearchable;
+ private boolean isCurrency;
+ private int isNullable;// 0:nonull, 1:nullable, 2: nullableunknown
+ private boolean isSigned;
+ private int displaySize;
+ private String label;// AS keyword
+ private String name;
+ private String schemaName;
+ private String catelogName;
+ private String tableName;
+ private int precision;
+ private int scale;
+ private int columnType;// as defined in java.sql.Types
+ private String columnTypeName;
+ private boolean isReadOnly;
+ private boolean isWritable;
+ private boolean isDefinitelyWritable;
+
+ public ColumnMetaStub() {
+ }
+
+ public boolean isAutoIncrement() {
+ return isAutoIncrement;
+ }
+
+ public void setAutoIncrement(boolean isAutoIncrement) {
+ this.isAutoIncrement = isAutoIncrement;
+ }
+
+ public boolean isCaseSensitive() {
+ return isCaseSensitive;
+ }
+
+ public void setCaseSensitive(boolean isCaseSensitive) {
+ this.isCaseSensitive = isCaseSensitive;
+ }
+
+ public boolean isSearchable() {
+ return isSearchable;
+ }
+
+ public void setSearchable(boolean isSearchable) {
+ this.isSearchable = isSearchable;
+ }
+
+ public boolean isCurrency() {
+ return isCurrency;
+ }
+
+ public void setCurrency(boolean isCurrency) {
+ this.isCurrency = isCurrency;
+ }
+
+ public int getIsNullable() {
+ return isNullable;
+ }
+
+ public void setIsNullable(int isNullable) {
+ this.isNullable = isNullable;
+ }
+
+ public boolean isSigned() {
+ return isSigned;
+ }
+
+ public void setSigned(boolean isSigned) {
+ this.isSigned = isSigned;
+ }
+
+ public int getDisplaySize() {
+ return displaySize;
+ }
+
+ public void setDisplaySize(int displaySize) {
+ this.displaySize = displaySize;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+ public String getCatelogName() {
+ return catelogName;
+ }
+
+ public void setCatelogName(String catelogName) {
+ this.catelogName = catelogName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public void setPrecision(int precision) {
+ this.precision = precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public void setScale(int scale) {
+ this.scale = scale;
+ }
+
+ public int getColumnType() {
+ return columnType;
+ }
+
+ public void setColumnType(int columnType) {
+ this.columnType = columnType;
+ }
+
+ public String getColumnTypeName() {
+ return columnTypeName;
+ }
+
+ public void setColumnTypeName(String columnTypeName) {
+ this.columnTypeName = columnTypeName;
+ }
+
+ public boolean isReadOnly() {
+ return isReadOnly;
+ }
+
+ public void setReadOnly(boolean isReadOnly) {
+ this.isReadOnly = isReadOnly;
+ }
+
+ public boolean isWritable() {
+ return isWritable;
+ }
+
+ public void setWritable(boolean isWritable) {
+ this.isWritable = isWritable;
+ }
+
+ public boolean isDefinitelyWritable() {
+ return isDefinitelyWritable;
+ }
+
+ public void setDefinitelyWritable(boolean isDefinitelyWritable) {
+ this.isDefinitelyWritable = isDefinitelyWritable;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
new file mode 100644
index 0000000..75cf422
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc.json;
+
+public class StatementParameter {
+
+ private String className;
+ private String value;
+
+ public StatementParameter(String className, String value) {
+ this.className = className;
+ this.value = value;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setClazz(String className) {
+ this.className = className;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
new file mode 100644
index 0000000..3d4b115
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc.json;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+public class TableMetaStub {
+
+ private static final long serialVersionUID = 1L;
+ private String TABLE_CAT;
+ private String TABLE_SCHEM;
+ private String TABLE_NAME;
+ private String TABLE_TYPE;
+ private String REMARKS;
+ private String TYPE_CAT;
+ private String TYPE_SCHEM;
+ private String TYPE_NAME;
+ private String SELF_REFERENCING_COL_NAME;
+ private String REF_GENERATION;
+ @JsonProperty("columns")
+ private List<ColumnMetaStub> columns = new ArrayList<ColumnMetaStub>();
+
+ public String getTABLE_CAT() {
+ return TABLE_CAT;
+ }
+
+ public void setTABLE_CAT(String tABLE_CAT) {
+ TABLE_CAT = tABLE_CAT;
+ }
+
+ public String getTABLE_SCHEM() {
+ return TABLE_SCHEM;
+ }
+
+ public void setTABLE_SCHEM(String tABLE_SCHEM) {
+ TABLE_SCHEM = tABLE_SCHEM;
+ }
+
+ public String getTABLE_NAME() {
+ return TABLE_NAME;
+ }
+
+ public void setTABLE_NAME(String tABLE_NAME) {
+ TABLE_NAME = tABLE_NAME;
+ }
+
+ public String getTABLE_TYPE() {
+ return TABLE_TYPE;
+ }
+
+ public void setTABLE_TYPE(String tABLE_TYPE) {
+ TABLE_TYPE = tABLE_TYPE;
+ }
+
+ public String getREMARKS() {
+ return REMARKS;
+ }
+
+ public void setREMARKS(String rEMARKS) {
+ REMARKS = rEMARKS;
+ }
+
+ public String getTYPE_CAT() {
+ return TYPE_CAT;
+ }
+
+ public void setTYPE_CAT(String tYPE_CAT) {
+ TYPE_CAT = tYPE_CAT;
+ }
+
+ public String getTYPE_SCHEM() {
+ return TYPE_SCHEM;
+ }
+
+ public void setTYPE_SCHEM(String tYPE_SCHEM) {
+ TYPE_SCHEM = tYPE_SCHEM;
+ }
+
+ public String getTYPE_NAME() {
+ return TYPE_NAME;
+ }
+
+ public void setTYPE_NAME(String tYPE_NAME) {
+ TYPE_NAME = tYPE_NAME;
+ }
+
+ public String getSELF_REFERENCING_COL_NAME() {
+ return SELF_REFERENCING_COL_NAME;
+ }
+
+ public void setSELF_REFERENCING_COL_NAME(String sELF_REFERENCING_COL_NAME) {
+ SELF_REFERENCING_COL_NAME = sELF_REFERENCING_COL_NAME;
+ }
+
+ public String getREF_GENERATION() {
+ return REF_GENERATION;
+ }
+
+ public void setREF_GENERATION(String rEF_GENERATION) {
+ REF_GENERATION = rEF_GENERATION;
+ }
+
+ public List<ColumnMetaStub> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<ColumnMetaStub> columns) {
+ this.columns = columns;
+ }
+
+ public static long getSerialversionuid() {
+ return serialVersionUID;
+ }
+
+ public static class ColumnMetaStub {
+ private String TABLE_CAT;
+ private String TABLE_SCHEM;
+ private String TABLE_NAME;
+ private String COLUMN_NAME;
+ private int DATA_TYPE;
+ private String TYPE_NAME;
+ private int COLUMN_SIZE;
+ private int BUFFER_LENGTH;
+ private int DECIMAL_DIGITS;
+ private int NUM_PREC_RADIX;
+ private int NULLABLE;
+ private String REMARKS;
+ private String COLUMN_DEF;
+ private int SQL_DATA_TYPE;
+ private int SQL_DATETIME_SUB;
+ private int CHAR_OCTET_LENGTH;
+ private int ORDINAL_POSITION;
+ private String IS_NULLABLE;
+ private String SCOPE_CATLOG;
+ private String SCOPE_SCHEMA;
+ private String SCOPE_TABLE;
+ private short SOURCE_DATA_TYPE;
+ private String IS_AUTOINCREMENT;
+
+ public String getTABLE_CAT() {
+ return TABLE_CAT;
+ }
+
+ public void setTABLE_CAT(String tABLE_CAT) {
+ TABLE_CAT = tABLE_CAT;
+ }
+
+ public String getTABLE_SCHEM() {
+ return TABLE_SCHEM;
+ }
+
+ public void setTABLE_SCHEM(String tABLE_SCHEM) {
+ TABLE_SCHEM = tABLE_SCHEM;
+ }
+
+ public String getTABLE_NAME() {
+ return TABLE_NAME;
+ }
+
+ public void setTABLE_NAME(String tABLE_NAME) {
+ TABLE_NAME = tABLE_NAME;
+ }
+
+ public String getCOLUMN_NAME() {
+ return COLUMN_NAME;
+ }
+
+ public void setCOLUMN_NAME(String cOLUMN_NAME) {
+ COLUMN_NAME = cOLUMN_NAME;
+ }
+
+ public int getDATA_TYPE() {
+ return DATA_TYPE;
+ }
+
+ public void setDATA_TYPE(int dATA_TYPE) {
+ DATA_TYPE = dATA_TYPE;
+ }
+
+ public String getTYPE_NAME() {
+ return TYPE_NAME;
+ }
+
+ public void setTYPE_NAME(String tYPE_NAME) {
+ TYPE_NAME = tYPE_NAME;
+ }
+
+ public int getCOLUMN_SIZE() {
+ return COLUMN_SIZE;
+ }
+
+ public void setCOLUMN_SIZE(int cOLUMN_SIZE) {
+ COLUMN_SIZE = cOLUMN_SIZE;
+ }
+
+ public int getBUFFER_LENGTH() {
+ return BUFFER_LENGTH;
+ }
+
+ public void setBUFFER_LENGTH(int bUFFER_LENGTH) {
+ BUFFER_LENGTH = bUFFER_LENGTH;
+ }
+
+ public int getDECIMAL_DIGITS() {
+ return DECIMAL_DIGITS;
+ }
+
+ public void setDECIMAL_DIGITS(int dECIMAL_DIGITS) {
+ DECIMAL_DIGITS = dECIMAL_DIGITS;
+ }
+
+ public int getNUM_PREC_RADIX() {
+ return NUM_PREC_RADIX;
+ }
+
+ public void setNUM_PREC_RADIX(int nUM_PREC_RADIX) {
+ NUM_PREC_RADIX = nUM_PREC_RADIX;
+ }
+
+ public int getNULLABLE() {
+ return NULLABLE;
+ }
+
+ public void setNULLABLE(int nULLABLE) {
+ NULLABLE = nULLABLE;
+ }
+
+ public String getREMARKS() {
+ return REMARKS;
+ }
+
+ public void setREMARKS(String rEMARKS) {
+ REMARKS = rEMARKS;
+ }
+
+ public String getCOLUMN_DEF() {
+ return COLUMN_DEF;
+ }
+
+ public void setCOLUMN_DEF(String cOLUMN_DEF) {
+ COLUMN_DEF = cOLUMN_DEF;
+ }
+
+ public int getSQL_DATA_TYPE() {
+ return SQL_DATA_TYPE;
+ }
+
+ public void setSQL_DATA_TYPE(int sQL_DATA_TYPE) {
+ SQL_DATA_TYPE = sQL_DATA_TYPE;
+ }
+
+ public int getSQL_DATETIME_SUB() {
+ return SQL_DATETIME_SUB;
+ }
+
+ public void setSQL_DATETIME_SUB(int sQL_DATETIME_SUB) {
+ SQL_DATETIME_SUB = sQL_DATETIME_SUB;
+ }
+
+ public int getCHAR_OCTET_LENGTH() {
+ return CHAR_OCTET_LENGTH;
+ }
+
+ public void setCHAR_OCTET_LENGTH(int cHAR_OCTET_LENGTH) {
+ CHAR_OCTET_LENGTH = cHAR_OCTET_LENGTH;
+ }
+
+ public int getORDINAL_POSITION() {
+ return ORDINAL_POSITION;
+ }
+
+ public void setORDINAL_POSITION(int oRDINAL_POSITION) {
+ ORDINAL_POSITION = oRDINAL_POSITION;
+ }
+
+ public String getIS_NULLABLE() {
+ return IS_NULLABLE;
+ }
+
+ public void setIS_NULLABLE(String iS_NULLABLE) {
+ IS_NULLABLE = iS_NULLABLE;
+ }
+
+ public String getSCOPE_CATLOG() {
+ return SCOPE_CATLOG;
+ }
+
+ public void setSCOPE_CATLOG(String sCOPE_CATLOG) {
+ SCOPE_CATLOG = sCOPE_CATLOG;
+ }
+
+ public String getSCOPE_SCHEMA() {
+ return SCOPE_SCHEMA;
+ }
+
+ public void setSCOPE_SCHEMA(String sCOPE_SCHEMA) {
+ SCOPE_SCHEMA = sCOPE_SCHEMA;
+ }
+
+ public String getSCOPE_TABLE() {
+ return SCOPE_TABLE;
+ }
+
+ public void setSCOPE_TABLE(String sCOPE_TABLE) {
+ SCOPE_TABLE = sCOPE_TABLE;
+ }
+
+ public short getSOURCE_DATA_TYPE() {
+ return SOURCE_DATA_TYPE;
+ }
+
+ public void setSOURCE_DATA_TYPE(short sOURCE_DATA_TYPE) {
+ SOURCE_DATA_TYPE = sOURCE_DATA_TYPE;
+ }
+
+ public String getIS_AUTOINCREMENT() {
+ return IS_AUTOINCREMENT;
+ }
+
+ public void setIS_AUTOINCREMENT(String iS_AUTOINCREMENT) {
+ this.IS_AUTOINCREMENT = iS_AUTOINCREMENT;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/ConnectionException.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/ConnectionException.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/ConnectionException.java
deleted file mode 100644
index eeeafc6..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/ConnectionException.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-/**
- * @author xduo
- *
- */
-public class ConnectionException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- public ConnectionException() {
- super();
- }
-
- /**
- * @param message
- * @param cause
- */
- public ConnectionException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * @param message
- */
- public ConnectionException(String message) {
- super(message);
- }
-
- /**
- * @param cause
- */
- public ConnectionException(Throwable cause) {
- super(cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
deleted file mode 100644
index 8b0347a..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.util.List;
-
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.linq4j.Enumerator;
-
-/**
- * Data set wrapper.
- *
- * @author xduo
- *
- */
-public class DataSet<E> {
-
- private final List<ColumnMetaData> meta;
-
- private final Enumerator<E> enumerator;
-
- /**
- * @param meta
- * @param enumerator
- */
- public DataSet(List<ColumnMetaData> meta, Enumerator<E> enumerator) {
- this.meta = meta;
- this.enumerator = enumerator;
- }
-
- public List<ColumnMetaData> getMeta() {
- return meta;
- }
-
- public Enumerator<E> getEnumerator() {
- return enumerator;
- }
-
-}
[24/50] [abbrv] incubator-kylin git commit: KYLIN-792 ,
add performance module
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/PerformService.java b/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
new file mode 100644
index 0000000..35b1207
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/service/PerformService.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import au.com.bytecode.opencsv.CSVReader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.*;
+
+/**
+ * @author jiazhong
+ */
+@Component("performService")
+public class PerformService extends BasicService {
+
+ private static final Logger logger = LoggerFactory.getLogger(PerformService.class);
+
+ /*
+ * @return all query user
+ */
+ public List<String[]> getTotalQueryUser() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/total_query_user.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ logger.info("Total Query User:"+res.get(0)[0]);
+ return res;
+ }
+
+ /*
+ * @return last 30 daily query num
+ */
+ public List<String[]> dailyQueryCount() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/last_30_daily_query_count.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ * @return average query count every day
+ */
+ public List<String[]> avgDayQuery() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/avg_day_query.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ logger.info("Avg Day Query:"+res.get(0)[0]);
+ return res;
+ }
+
+ /*
+ *@return average latency every day
+ */
+ public List<String[]> last30DayPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/last_30_day_90_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ *@return average latency for every cube
+ */
+ public List<String[]> eachDayPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/each_day_90_95_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+ /*
+ *@return average latency for every cube
+ */
+ public List<String[]> projectPercentile() throws IOException {
+ String filePath = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()+"/performance/metadata/project_90_95_percentile_latency.csv";
+ List<String[]> res = readHdfsFile(filePath);
+ return res;
+ }
+
+
+ private List<String[]> readHdfsFile(String filePath) throws IOException {
+ List<String[]> allRows = null;
+ CSVReader reader = null;
+ FileSystem fs = null;
+ Configuration conf = new Configuration();
+
+ try {
+ fs = FileSystem.newInstance(conf);
+ FSDataInputStream inputStream = fs.open(new Path(filePath));
+ reader = new CSVReader(new InputStreamReader (inputStream),'|');
+
+ //Read all rows at once
+ allRows = reader.readAll();
+
+
+ } catch (IOException e) {
+ logger.info("failed to read hdfs file:",e);
+ }
+ finally {
+ fs.close();
+ }
+ return allRows;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index d372240..22eea4c 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -34,6 +34,7 @@
<scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')" />
<scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()" />
+ <scr:intercept-url pattern="/api/performance*/**" access="isAuthenticated()" />
<scr:intercept-url pattern="/api/admin/config" access="permitAll" />
<scr:intercept-url pattern="/api/projects" access="permitAll" />
<scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')" />
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index d12f81a..703c815 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -14,10 +14,10 @@ limitations under the License. See accompanying LICENSE file.
-->
<web-app xmlns="http://java.sun.com/xml/ns/javaee"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
- version="2.5">
+ version="2.5">
<display-name>Kylin RESt Service</display-name>
@@ -37,10 +37,10 @@ limitations under the License. See accompanying LICENSE file.
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>
- classpath:applicationContext.xml
- classpath:kylinSecurity.xml
- classpath*:kylin-*-plugin.xml
- </param-value>
+ classpath:applicationContext.xml
+ classpath:kylinSecurity.xml
+ classpath*:kylin-*-plugin.xml
+ </param-value>
</context-param>
<listener>
@@ -50,46 +50,58 @@ limitations under the License. See accompanying LICENSE file.
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
-
+
+ <listener>
+ <listener-class>com.ryantenney.metrics.spring.servlets.MetricsServletsContextListener</listener-class>
+ </listener>
<listener>
- <listener-class>com.ryantenney.metrics.spring.servlets.MetricsServletsContextListener</listener-class>
+ <listener-class>org.apache.kylin.rest.metrics.KylinInstrumentedFilterContextListener</listener-class>
</listener>
+
<listener>
- <listener-class>org.apache.kylin.rest.metrics.KylinInstrumentedFilterContextListener</listener-class>
+ <listener-class>
+ org.springframework.security.web.session.HttpSessionEventPublisher
+ </listener-class>
</listener>
- <listener>
- <listener-class>
- org.springframework.security.web.session.HttpSessionEventPublisher
- </listener-class>
- </listener>
-
-<filter>
- <filter-name>CORS</filter-name>
- <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
- <init-param>
- <param-name>cors.supportedHeaders</param-name>
- <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept</param-value>
- </init-param>
- <init-param>
- <param-name>cors.supportedMethods</param-name>
- <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value>
- </init-param>
- <init-param>
- <param-name>cors.supportsCredentials </param-name>
- <param-value>true</param-value>
- </init-param>
-</filter>
-
-<filter-mapping>
- <filter-name>CORS</filter-name>
- <url-pattern>/*</url-pattern>
-</filter-mapping>
-
- <!--
- Apply Spring Security Filter to all Requests
- -->
- <filter>
+ <filter>
+ <filter-name>CORS</filter-name>
+ <filter-class>com.thetransactioncompany.cors.CORSFilter</filter-class>
+ <init-param>
+ <param-name>cors.supportedHeaders</param-name>
+ <param-value>Authorization,Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified,
+ Cache-Control, Expires, Content-Type, X-E4M-With, Accept
+ </param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.supportedMethods</param-name>
+ <param-value>GET, POST, PUT, DELETE, OPTIONS</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cors.supportsCredentials</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>CORS</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter>
+ <filter-name>loggingFilter</filter-name>
+ <filter-class>org.apache.kylin.rest.filter.KylinApiFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>loggingFilter</filter-name>
+ <url-pattern>/api/*</url-pattern>
+ </filter-mapping>
+
+ <!--
+ Apply Spring Security Filter to all Requests
+ -->
+ <filter>
<filter-name>springSecurityFilterChain</filter-name>
<filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
</filter>
@@ -98,15 +110,15 @@ limitations under the License. See accompanying LICENSE file.
<url-pattern>/*</url-pattern>
</filter-mapping>
- <filter>
- <filter-name>instrumentedFilter</filter-name>
- <filter-class>com.codahale.metrics.servlet.InstrumentedFilter</filter-class>
- </filter>
- <filter-mapping>
- <filter-name>instrumentedFilter</filter-name>
- <url-pattern>/*</url-pattern>
- </filter-mapping>
-
+ <filter>
+ <filter-name>instrumentedFilter</filter-name>
+ <filter-class>com.codahale.metrics.servlet.InstrumentedFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>instrumentedFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
<servlet>
<servlet-name>kylin</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
@@ -125,7 +137,7 @@ limitations under the License. See accompanying LICENSE file.
<servlet-name>metricsservlet</servlet-name>
<url-pattern>/metrics/*</url-pattern>
</servlet-mapping>
-
+
<servlet>
<servlet-name>metricsadminservlet</servlet-name>
<servlet-class>com.codahale.metrics.servlets.AdminServlet</servlet-class>
@@ -135,9 +147,9 @@ limitations under the License. See accompanying LICENSE file.
<servlet-name>metricsadminservlet</servlet-name>
<url-pattern>/metricsadmin/*</url-pattern>
</servlet-mapping>
-
- <!--
- <servlet>
+
+ <!--
+ <servlet>
<servlet-name>matricsadmin</servlet-name>
<servlet-class>com.codahale.metrics.servlets.MetricsServlet</servlet-class>
<load-on-startup>1</load-on-startup>
[33/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
new file mode 100644
index 0000000..3b1a2e0
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -0,0 +1,1261 @@
+/*
+ * OVERRIDE POINT:
+ * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);`
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.runtime;
+
+import org.apache.calcite.DataContext;
+import org.apache.calcite.avatica.util.ByteString;
+import org.apache.calcite.avatica.util.DateTimeUtils;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
+import org.apache.calcite.linq4j.function.Deterministic;
+import org.apache.calcite.linq4j.function.Function1;
+import org.apache.calcite.linq4j.function.NonDeterministic;
+import org.apache.calcite.linq4j.tree.Primitive;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.MathContext;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
+
+/**
+ * Helper methods to implement SQL functions in generated code.
+ *
+ * <p>Not present: and, or, not (builtin operators are better, because they
+ * use lazy evaluation. Implementations do not check for null values; the
+ * calling code must do that.</p>
+ *
+ * <p>Many of the functions do not check for null values. This is intentional.
+ * If null arguments are possible, the code-generation framework checks for
+ * nulls before calling the functions.</p>
+ */
+@SuppressWarnings("UnnecessaryUnboxing")
+@Deterministic
+public class SqlFunctions {
+ private static final DecimalFormat DOUBLE_FORMAT =
+ new DecimalFormat("0.0E0");
+
+ private static final TimeZone LOCAL_TZ = TimeZone.getDefault();
+
+ private static final Function1<List<Object>, Enumerable<Object>>
+ LIST_AS_ENUMERABLE =
+ new Function1<List<Object>, Enumerable<Object>>() {
+ public Enumerable<Object> apply(List<Object> list) {
+ return Linq4j.asEnumerable(list);
+ }
+ };
+
+ /** Holds, for each thread, a map from sequence name to sequence current
+ * value.
+ *
+ * <p>This is a straw man of an implementation whose main goal is to prove
+ * that sequences can be parsed, validated and planned. A real application
+ * will want persistent values for sequences, shared among threads. */
+ private static final ThreadLocal<Map<String, AtomicLong>> THREAD_SEQUENCES =
+ new ThreadLocal<Map<String, AtomicLong>>() {
+ @Override protected Map<String, AtomicLong> initialValue() {
+ return new HashMap<String, AtomicLong>();
+ }
+ };
+
+ private SqlFunctions() {
+ }
+
+ /** SQL SUBSTRING(string FROM ... FOR ...) function. */
+ public static String substring(String s, int from, int for_) {
+ return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
+ }
+
+ /** SQL SUBSTRING(string FROM ...) function. */
+ public static String substring(String s, int from) {
+ return s.substring(from - 1);
+ }
+
+ /** SQL UPPER(string) function. */
+ public static String upper(String s) {
+ return s.toUpperCase();
+ }
+
+ /** SQL LOWER(string) function. */
+ public static String lower(String s) {
+ return s.toLowerCase();
+ }
+
+ /** SQL INITCAP(string) function. */
+ public static String initcap(String s) {
+ // Assumes Alpha as [A-Za-z0-9]
+ // white space is treated as everything else.
+ final int len = s.length();
+ boolean start = true;
+ final StringBuilder newS = new StringBuilder();
+
+ for (int i = 0; i < len; i++) {
+ char curCh = s.charAt(i);
+ final int c = (int) curCh;
+ if (start) { // curCh is whitespace or first character of word.
+ if (c > 47 && c < 58) { // 0-9
+ start = false;
+ } else if (c > 64 && c < 91) { // A-Z
+ start = false;
+ } else if (c > 96 && c < 123) { // a-z
+ start = false;
+ curCh = (char) (c - 32); // Uppercase this character
+ }
+ // else {} whitespace
+ } else { // Inside of a word or white space after end of word.
+ if (c > 47 && c < 58) { // 0-9
+ // noop
+ } else if (c > 64 && c < 91) { // A-Z
+ curCh = (char) (c + 32); // Lowercase this character
+ } else if (c > 96 && c < 123) { // a-z
+ // noop
+ } else { // whitespace
+ start = true;
+ }
+ }
+ newS.append(curCh);
+ } // for each character in s
+ return newS.toString();
+ }
+
+ /** SQL CHARACTER_LENGTH(string) function. */
+ public static int charLength(String s) {
+ return s.length();
+ }
+
+ /** SQL {@code string || string} operator. */
+ public static String concat(String s0, String s1) {
+ return s0 + s1;
+ }
+
+ /** SQL {@code binary || binary} operator. */
+ public static ByteString concat(ByteString s0, ByteString s1) {
+ return s0.concat(s1);
+ }
+
+ /** SQL {@code RTRIM} function applied to string. */
+ public static String rtrim(String s) {
+ return trim_(s, false, true, ' ');
+ }
+
+ /** SQL {@code LTRIM} function. */
+ public static String ltrim(String s) {
+ return trim_(s, true, false, ' ');
+ }
+
+ /** SQL {@code TRIM(... seek FROM s)} function. */
+ public static String trim(boolean leading, boolean trailing, String seek,
+ String s) {
+ return trim_(s, leading, trailing, seek.charAt(0));
+ }
+
+ /** SQL {@code TRIM} function. */
+ private static String trim_(String s, boolean left, boolean right, char c) {
+ int j = s.length();
+ if (right) {
+ for (;;) {
+ if (j == 0) {
+ return "";
+ }
+ if (s.charAt(j - 1) != c) {
+ break;
+ }
+ --j;
+ }
+ }
+ int i = 0;
+ if (left) {
+ for (;;) {
+ if (i == j) {
+ return "";
+ }
+ if (s.charAt(i) != c) {
+ break;
+ }
+ ++i;
+ }
+ }
+ return s.substring(i, j);
+ }
+
+ /** SQL {@code TRIM} function applied to binary string. */
+ public static ByteString trim(ByteString s) {
+ return trim_(s, true, true);
+ }
+
+ /** Helper for CAST. */
+ public static ByteString rtrim(ByteString s) {
+ return trim_(s, false, true);
+ }
+
+ /** SQL {@code TRIM} function applied to binary string. */
+ private static ByteString trim_(ByteString s, boolean left, boolean right) {
+ int j = s.length();
+ if (right) {
+ for (;;) {
+ if (j == 0) {
+ return ByteString.EMPTY;
+ }
+ if (s.byteAt(j - 1) != 0) {
+ break;
+ }
+ --j;
+ }
+ }
+ int i = 0;
+ if (left) {
+ for (;;) {
+ if (i == j) {
+ return ByteString.EMPTY;
+ }
+ if (s.byteAt(i) != 0) {
+ break;
+ }
+ ++i;
+ }
+ }
+ return s.substring(i, j);
+ }
+
+ /** SQL {@code OVERLAY} function. */
+ public static String overlay(String s, String r, int start) {
+ if (s == null || r == null) {
+ return null;
+ }
+ return s.substring(0, start - 1)
+ + r
+ + s.substring(start - 1 + r.length());
+ }
+
+ /** SQL {@code OVERLAY} function. */
+ public static String overlay(String s, String r, int start, int length) {
+ if (s == null || r == null) {
+ return null;
+ }
+ return s.substring(0, start - 1)
+ + r
+ + s.substring(start - 1 + length);
+ }
+
+ /** SQL {@code OVERLAY} function applied to binary strings. */
+ public static ByteString overlay(ByteString s, ByteString r, int start) {
+ if (s == null || r == null) {
+ return null;
+ }
+ return s.substring(0, start - 1)
+ .concat(r)
+ .concat(s.substring(start - 1 + r.length()));
+ }
+
+ /** SQL {@code OVERLAY} function applied to binary strings. */
+ public static ByteString overlay(ByteString s, ByteString r, int start,
+ int length) {
+ if (s == null || r == null) {
+ return null;
+ }
+ return s.substring(0, start - 1)
+ .concat(r)
+ .concat(s.substring(start - 1 + length));
+ }
+
+ /** SQL {@code LIKE} function. */
+ public static boolean like(String s, String pattern) {
+ final String regex = Like.sqlToRegexLike(pattern, null);
+ return Pattern.matches(regex, s);
+ }
+
+ /** SQL {@code LIKE} function with escape. */
+ public static boolean like(String s, String pattern, String escape) {
+ final String regex = Like.sqlToRegexLike(pattern, escape);
+ return Pattern.matches(regex, s);
+ }
+
+ /** SQL {@code SIMILAR} function. */
+ public static boolean similar(String s, String pattern) {
+ final String regex = Like.sqlToRegexSimilar(pattern, null);
+ return Pattern.matches(regex, s);
+ }
+
+ /** SQL {@code SIMILAR} function with escape. */
+ public static boolean similar(String s, String pattern, String escape) {
+ final String regex = Like.sqlToRegexSimilar(pattern, escape);
+ return Pattern.matches(regex, s);
+ }
+
+ // =
+
+ /** SQL = operator applied to Object values (including String; neither
+ * side may be null). */
+ public static boolean eq(Object b0, Object b1) {
+ return b0.equals(b1);
+ }
+
+ /** SQL = operator applied to BigDecimal values (neither may be null). */
+ public static boolean eq(BigDecimal b0, BigDecimal b1) {
+ return b0.stripTrailingZeros().equals(b1.stripTrailingZeros());
+ }
+
+ // <>
+
+ /** SQL <> operator applied to Object values (including String;
+ * neither side may be null). */
+ public static boolean ne(Object b0, Object b1) {
+ return !b0.equals(b1);
+ }
+
+ /** SQL <> operator applied to BigDecimal values. */
+ public static boolean ne(BigDecimal b0, BigDecimal b1) {
+ return b0.compareTo(b1) != 0;
+ }
+
+ // <
+
+ /** SQL < operator applied to boolean values. */
+ public static boolean lt(boolean b0, boolean b1) {
+ return compare(b0, b1) < 0;
+ }
+
+ /** SQL < operator applied to String values. */
+ public static boolean lt(String b0, String b1) {
+ return b0.compareTo(b1) < 0;
+ }
+
+ /** SQL < operator applied to ByteString values. */
+ public static boolean lt(ByteString b0, ByteString b1) {
+ return b0.compareTo(b1) < 0;
+ }
+
+ /** SQL < operator applied to BigDecimal values. */
+ public static boolean lt(BigDecimal b0, BigDecimal b1) {
+ return b0.compareTo(b1) < 0;
+ }
+
+ // <=
+
+ /** SQL ≤ operator applied to boolean values. */
+ public static boolean le(boolean b0, boolean b1) {
+ return compare(b0, b1) <= 0;
+ }
+
+ /** SQL ≤ operator applied to String values. */
+ public static boolean le(String b0, String b1) {
+ return b0.compareTo(b1) <= 0;
+ }
+
+ /** SQL ≤ operator applied to ByteString values. */
+ public static boolean le(ByteString b0, ByteString b1) {
+ return b0.compareTo(b1) <= 0;
+ }
+
+ /** SQL ≤ operator applied to BigDecimal values. */
+ public static boolean le(BigDecimal b0, BigDecimal b1) {
+ return b0.compareTo(b1) <= 0;
+ }
+
+ // >
+
+ /** SQL > operator applied to boolean values. */
+ public static boolean gt(boolean b0, boolean b1) {
+ return compare(b0, b1) > 0;
+ }
+
+ /** SQL > operator applied to String values. */
+ public static boolean gt(String b0, String b1) {
+ return b0.compareTo(b1) > 0;
+ }
+
+ /** SQL > operator applied to ByteString values. */
+ public static boolean gt(ByteString b0, ByteString b1) {
+ return b0.compareTo(b1) > 0;
+ }
+
+ /** SQL > operator applied to BigDecimal values. */
+ public static boolean gt(BigDecimal b0, BigDecimal b1) {
+ return b0.compareTo(b1) > 0;
+ }
+
+ // >=
+
+ /** SQL ≥ operator applied to boolean values. */
+ public static boolean ge(boolean b0, boolean b1) {
+ return compare(b0, b1) >= 0;
+ }
+
+ /** SQL ≥ operator applied to String values. */
+ public static boolean ge(String b0, String b1) {
+ return b0.compareTo(b1) >= 0;
+ }
+
+ /** SQL ≥ operator applied to ByteString values. */
+ public static boolean ge(ByteString b0, ByteString b1) {
+ return b0.compareTo(b1) >= 0;
+ }
+
+ /** SQL ≥ operator applied to BigDecimal values. */
+ public static boolean ge(BigDecimal b0, BigDecimal b1) {
+ return b0.compareTo(b1) >= 0;
+ }
+
+ // +
+
+ /** SQL <code>+</code> operator applied to int values. */
+ public static int plus(int b0, int b1) {
+ return b0 + b1;
+ }
+
+ /** SQL <code>+</code> operator applied to int values; left side may be
+ * null. */
+ public static Integer plus(Integer b0, int b1) {
+ return b0 == null ? null : (b0 + b1);
+ }
+
+ /** SQL <code>+</code> operator applied to int values; right side may be
+ * null. */
+ public static Integer plus(int b0, Integer b1) {
+ return b1 == null ? null : (b0 + b1);
+ }
+
+ /** SQL <code>+</code> operator applied to nullable int values. */
+ public static Integer plus(Integer b0, Integer b1) {
+ return (b0 == null || b1 == null) ? null : (b0 + b1);
+ }
+
+ /** SQL <code>+</code> operator applied to nullable long and int values. */
+ public static Long plus(Long b0, Integer b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() + b1.longValue());
+ }
+
+ /** SQL <code>+</code> operator applied to nullable int and long values. */
+ public static Long plus(Integer b0, Long b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() + b1.longValue());
+ }
+
+ /** SQL <code>+</code> operator applied to BigDecimal values. */
+ public static BigDecimal plus(BigDecimal b0, BigDecimal b1) {
+ return (b0 == null || b1 == null) ? null : b0.add(b1);
+ }
+
+ // -
+
+ /** SQL <code>-</code> operator applied to int values. */
+ public static int minus(int b0, int b1) {
+ return b0 - b1;
+ }
+
+ /** SQL <code>-</code> operator applied to int values; left side may be
+ * null. */
+ public static Integer minus(Integer b0, int b1) {
+ return b0 == null ? null : (b0 - b1);
+ }
+
+ /** SQL <code>-</code> operator applied to int values; right side may be
+ * null. */
+ public static Integer minus(int b0, Integer b1) {
+ return b1 == null ? null : (b0 - b1);
+ }
+
+ /** SQL <code>-</code> operator applied to nullable int values. */
+ public static Integer minus(Integer b0, Integer b1) {
+ return (b0 == null || b1 == null) ? null : (b0 - b1);
+ }
+
+ /** SQL <code>-</code> operator applied to nullable long and int values. */
+ public static Long minus(Long b0, Integer b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() - b1.longValue());
+ }
+
+ /** SQL <code>-</code> operator applied to nullable int and long values. */
+ public static Long minus(Integer b0, Long b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() - b1.longValue());
+ }
+
+ /** SQL <code>-</code> operator applied to BigDecimal values. */
+ public static BigDecimal minus(BigDecimal b0, BigDecimal b1) {
+ return (b0 == null || b1 == null) ? null : b0.subtract(b1);
+ }
+
+ // /
+
+ /** SQL <code>/</code> operator applied to int values. */
+ public static int divide(int b0, int b1) {
+ return b0 / b1;
+ }
+
+ /** SQL <code>/</code> operator applied to int values; left side may be
+ * null. */
+ public static Integer divide(Integer b0, int b1) {
+ return b0 == null ? null : (b0 / b1);
+ }
+
+ /** SQL <code>/</code> operator applied to int values; right side may be
+ * null. */
+ public static Integer divide(int b0, Integer b1) {
+ return b1 == null ? null : (b0 / b1);
+ }
+
+ /** SQL <code>/</code> operator applied to nullable int values. */
+ public static Integer divide(Integer b0, Integer b1) {
+ return (b0 == null || b1 == null) ? null : (b0 / b1);
+ }
+
+ /** SQL <code>/</code> operator applied to nullable long and int values. */
+ public static Long divide(Long b0, Integer b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() / b1.longValue());
+ }
+
+ /** SQL <code>/</code> operator applied to nullable int and long values. */
+ public static Long divide(Integer b0, Long b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() / b1.longValue());
+ }
+
+ /** SQL <code>/</code> operator applied to BigDecimal values. */
+ public static BigDecimal divide(BigDecimal b0, BigDecimal b1) {
+ // OVERRIDE POINT
+ return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64);
+ }
+
+ // *
+
+ /** SQL <code>*</code> operator applied to int values. */
+ public static int multiply(int b0, int b1) {
+ return b0 * b1;
+ }
+
+ /** SQL <code>*</code> operator applied to int values; left side may be
+ * null. */
+ public static Integer multiply(Integer b0, int b1) {
+ return b0 == null ? null : (b0 * b1);
+ }
+
+ /** SQL <code>*</code> operator applied to int values; right side may be
+ * null. */
+ public static Integer multiply(int b0, Integer b1) {
+ return b1 == null ? null : (b0 * b1);
+ }
+
+ /** SQL <code>*</code> operator applied to nullable int values. */
+ public static Integer multiply(Integer b0, Integer b1) {
+ return (b0 == null || b1 == null) ? null : (b0 * b1);
+ }
+
+ /** SQL <code>*</code> operator applied to nullable long and int values. */
+ public static Long multiply(Long b0, Integer b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() * b1.longValue());
+ }
+
+ /** SQL <code>*</code> operator applied to nullable int and long values. */
+ public static Long multiply(Integer b0, Long b1) {
+ return (b0 == null || b1 == null)
+ ? null
+ : (b0.longValue() * b1.longValue());
+ }
+
+ /** SQL <code>*</code> operator applied to BigDecimal values. */
+ public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) {
+ return (b0 == null || b1 == null) ? null : b0.multiply(b1);
+ }
+
+ // EXP
+
+ /** SQL <code>EXP</code> operator applied to double values. */
+ public static double exp(double b0) {
+ return Math.exp(b0);
+ }
+
+ public static double exp(long b0) {
+ return Math.exp(b0);
+ }
+
+ // POWER
+
+ /** SQL <code>POWER</code> operator applied to double values. */
+ public static double power(double b0, double b1) {
+ return Math.pow(b0, b1);
+ }
+
+ public static double power(long b0, long b1) {
+ return Math.pow(b0, b1);
+ }
+
+ public static double power(long b0, BigDecimal b1) {
+ return Math.pow(b0, b1.doubleValue());
+ }
+
+ // LN
+
+ /** SQL {@code LN(number)} function applied to double values. */
+ public static double ln(double d) {
+ return Math.log(d);
+ }
+
+ /** SQL {@code LN(number)} function applied to long values. */
+ public static double ln(long b0) {
+ return Math.log(b0);
+ }
+
+ /** SQL {@code LN(number)} function applied to BigDecimal values. */
+ public static double ln(BigDecimal d) {
+ return Math.log(d.doubleValue());
+ }
+
+ // LOG10
+
+ /** SQL <code>LOG10(numeric)</code> operator applied to double values. */
+ public static double log10(double b0) {
+ return Math.log10(b0);
+ }
+
+ /** SQL {@code LOG10(number)} function applied to long values. */
+ public static double log10(long b0) {
+ return Math.log10(b0);
+ }
+
+ /** SQL {@code LOG10(number)} function applied to BigDecimal values. */
+ public static double log10(BigDecimal d) {
+ return Math.log10(d.doubleValue());
+ }
+
+ // MOD
+
+ /** SQL <code>MOD</code> operator applied to byte values. */
+ public static byte mod(byte b0, byte b1) {
+ return (byte) (b0 % b1);
+ }
+
+ /** SQL <code>MOD</code> operator applied to short values. */
+ public static short mod(short b0, short b1) {
+ return (short) (b0 % b1);
+ }
+
+ /** SQL <code>MOD</code> operator applied to int values. */
+ public static int mod(int b0, int b1) {
+ return b0 % b1;
+ }
+
+ /** SQL <code>MOD</code> operator applied to long values. */
+ public static long mod(long b0, long b1) {
+ return b0 % b1;
+ }
+
+ // temporary
+ public static BigDecimal mod(BigDecimal b0, int b1) {
+ return mod(b0, BigDecimal.valueOf(b1));
+ }
+
+ // temporary
+ public static int mod(int b0, BigDecimal b1) {
+ return mod(b0, b1.intValue());
+ }
+
+ public static BigDecimal mod(BigDecimal b0, BigDecimal b1) {
+ final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
+ return bigDecimals[1];
+ }
+
+ // ABS
+
+ /** SQL <code>ABS</code> operator applied to byte values. */
+ public static byte abs(byte b0) {
+ return (byte) Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to short values. */
+ public static short abs(short b0) {
+ return (short) Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to int values. */
+ public static int abs(int b0) {
+ return Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to long values. */
+ public static long abs(long b0) {
+ return Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to float values. */
+ public static float abs(float b0) {
+ return Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to double values. */
+ public static double abs(double b0) {
+ return Math.abs(b0);
+ }
+
+ /** SQL <code>ABS</code> operator applied to BigDecimal values. */
+ public static BigDecimal abs(BigDecimal b0) {
+ return b0.abs();
+ }
+
+ // Helpers
+
+ /** Helper for implementing MIN. Somewhat similar to LEAST operator. */
+ public static <T extends Comparable<T>> T lesser(T b0, T b1) {
+ return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0;
+ }
+
+ /** LEAST operator. */
+ public static <T extends Comparable<T>> T least(T b0, T b1) {
+ return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0;
+ }
+
+ public static boolean greater(boolean b0, boolean b1) {
+ return b0 || b1;
+ }
+
+ public static boolean lesser(boolean b0, boolean b1) {
+ return b0 && b1;
+ }
+
+ public static byte greater(byte b0, byte b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static byte lesser(byte b0, byte b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static char greater(char b0, char b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static char lesser(char b0, char b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static short greater(short b0, short b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static short lesser(short b0, short b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static int greater(int b0, int b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static int lesser(int b0, int b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static long greater(long b0, long b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static long lesser(long b0, long b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static float greater(float b0, float b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static float lesser(float b0, float b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ public static double greater(double b0, double b1) {
+ return b0 > b1 ? b0 : b1;
+ }
+
+ public static double lesser(double b0, double b1) {
+ return b0 > b1 ? b1 : b0;
+ }
+
+ /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */
+ public static <T extends Comparable<T>> T greater(T b0, T b1) {
+ return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0;
+ }
+
+ /** GREATEST operator. */
+ public static <T extends Comparable<T>> T greatest(T b0, T b1) {
+ return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0;
+ }
+
+ /** Boolean comparison. */
+ public static int compare(boolean x, boolean y) {
+ return x == y ? 0 : x ? 1 : -1;
+ }
+
+ /** CAST(FLOAT AS VARCHAR). */
+ public static String toString(float x) {
+ if (x == 0) {
+ return "0E0";
+ }
+ BigDecimal bigDecimal =
+ new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros();
+ final String s = bigDecimal.toString();
+ return s.replaceAll("0*E", "E").replace("E+", "E");
+ }
+
+ /** CAST(DOUBLE AS VARCHAR). */
+ public static String toString(double x) {
+ if (x == 0) {
+ return "0E0";
+ }
+ BigDecimal bigDecimal =
+ new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros();
+ final String s = bigDecimal.toString();
+ return s.replaceAll("0*E", "E").replace("E+", "E");
+ }
+
+ /** CAST(DECIMAL AS VARCHAR). */
+ public static String toString(BigDecimal x) {
+ final String s = x.toString();
+ if (s.startsWith("0")) {
+ // we want ".1" not "0.1"
+ return s.substring(1);
+ } else if (s.startsWith("-0")) {
+ // we want "-.1" not "-0.1"
+ return "-" + s.substring(2);
+ } else {
+ return s;
+ }
+ }
+
+ /** CAST(BOOLEAN AS VARCHAR). */
+ public static String toString(boolean x) {
+ // Boolean.toString returns lower case -- no good.
+ return x ? "TRUE" : "FALSE";
+ }
+
+ @NonDeterministic
+ private static Object cannotConvert(Object o, Class toType) {
+ throw new RuntimeException("Cannot convert " + o + " to " + toType);
+ }
+
+ /** CAST(VARCHAR AS BOOLEAN). */
+ public static boolean toBoolean(String s) {
+ s = trim_(s, true, true, ' ');
+ if (s.equalsIgnoreCase("TRUE")) {
+ return true;
+ } else if (s.equalsIgnoreCase("FALSE")) {
+ return false;
+ } else {
+ throw new RuntimeException("Invalid character for cast");
+ }
+ }
+
+ public static boolean toBoolean(Number number) {
+ return !number.equals(0);
+ }
+
+ public static boolean toBoolean(Object o) {
+ return o instanceof Boolean ? (Boolean) o
+ : o instanceof Number ? toBoolean((Number) o)
+ : o instanceof String ? toBoolean((String) o)
+ : (Boolean) cannotConvert(o, boolean.class);
+ }
+
+ // Don't need parseByte etc. - Byte.parseByte is sufficient.
+
+ public static byte toByte(Object o) {
+ return o instanceof Byte ? (Byte) o
+ : o instanceof Number ? toByte((Number) o)
+ : Byte.parseByte(o.toString());
+ }
+
+ public static byte toByte(Number number) {
+ return number.byteValue();
+ }
+
+ public static char toChar(String s) {
+ return s.charAt(0);
+ }
+
+ public static Character toCharBoxed(String s) {
+ return s.charAt(0);
+ }
+
+ public static short toShort(String s) {
+ return Short.parseShort(s.trim());
+ }
+
+ public static short toShort(Number number) {
+ return number.shortValue();
+ }
+
+ public static short toShort(Object o) {
+ return o instanceof Short ? (Short) o
+ : o instanceof Number ? toShort((Number) o)
+ : o instanceof String ? toShort((String) o)
+ : (Short) cannotConvert(o, short.class);
+ }
+
+ public static int toInt(java.util.Date v) {
+ return toInt(v, LOCAL_TZ);
+ }
+
+ public static int toInt(java.util.Date v, TimeZone timeZone) {
+ return (int) (toLong(v, timeZone) / DateTimeUtils.MILLIS_PER_DAY);
+ }
+
+ public static Integer toIntOptional(java.util.Date v) {
+ return v == null ? null : toInt(v);
+ }
+
+ public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) {
+ return v == null
+ ? null
+ : toInt(v, timeZone);
+ }
+
+ public static long toLong(Date v) {
+ return toLong(v, LOCAL_TZ);
+ }
+
+ public static int toInt(java.sql.Time v) {
+ return (int) (toLong(v) % DateTimeUtils.MILLIS_PER_DAY);
+ }
+
+ public static Integer toIntOptional(java.sql.Time v) {
+ return v == null ? null : toInt(v);
+ }
+
+ public static int toInt(String s) {
+ return Integer.parseInt(s.trim());
+ }
+
+ public static int toInt(Number number) {
+ return number.intValue();
+ }
+
+ public static int toInt(Object o) {
+ return o instanceof Integer ? (Integer) o
+ : o instanceof Number ? toInt((Number) o)
+ : o instanceof String ? toInt((String) o)
+ : (Integer) cannotConvert(o, int.class);
+ }
+
+ public static long toLong(Timestamp v) {
+ return toLong(v, LOCAL_TZ);
+ }
+
+ // mainly intended for java.sql.Timestamp but works for other dates also
+ public static long toLong(java.util.Date v, TimeZone timeZone) {
+ final long time = v.getTime();
+ return time + timeZone.getOffset(time);
+ }
+
+ // mainly intended for java.sql.Timestamp but works for other dates also
+ public static Long toLongOptional(java.util.Date v) {
+ return v == null ? null : toLong(v, LOCAL_TZ);
+ }
+
+ public static Long toLongOptional(Timestamp v, TimeZone timeZone) {
+ if (v == null) {
+ return null;
+ }
+ return toLong(v, LOCAL_TZ);
+ }
+
+ public static long toLong(String s) {
+ if (s.startsWith("199") && s.contains(":")) {
+ return Timestamp.valueOf(s).getTime();
+ }
+ return Long.parseLong(s.trim());
+ }
+
+ public static long toLong(Number number) {
+ return number.longValue();
+ }
+
+ public static long toLong(Object o) {
+ return o instanceof Long ? (Long) o
+ : o instanceof Number ? toLong((Number) o)
+ : o instanceof String ? toLong((String) o)
+ : (Long) cannotConvert(o, long.class);
+ }
+
+ public static float toFloat(String s) {
+ return Float.parseFloat(s.trim());
+ }
+
+ public static float toFloat(Number number) {
+ return number.floatValue();
+ }
+
+ public static float toFloat(Object o) {
+ return o instanceof Float ? (Float) o
+ : o instanceof Number ? toFloat((Number) o)
+ : o instanceof String ? toFloat((String) o)
+ : (Float) cannotConvert(o, float.class);
+ }
+
+ public static double toDouble(String s) {
+ return Double.parseDouble(s.trim());
+ }
+
+ public static double toDouble(Number number) {
+ return number.doubleValue();
+ }
+
+ public static double toDouble(Object o) {
+ return o instanceof Double ? (Double) o
+ : o instanceof Number ? toDouble((Number) o)
+ : o instanceof String ? toDouble((String) o)
+ : (Double) cannotConvert(o, double.class);
+ }
+
+ public static BigDecimal toBigDecimal(String s) {
+ return new BigDecimal(s.trim());
+ }
+
+ public static BigDecimal toBigDecimal(Number number) {
+ // There are some values of "long" that cannot be represented as "double".
+ // Not so "int". If it isn't a long, go straight to double.
+ return number instanceof BigDecimal ? (BigDecimal) number
+ : number instanceof BigInteger ? new BigDecimal((BigInteger) number)
+ : number instanceof Long ? new BigDecimal(number.longValue())
+ : new BigDecimal(number.doubleValue());
+ }
+
+ public static BigDecimal toBigDecimal(Object o) {
+ return o instanceof Number ? toBigDecimal((Number) o)
+ : toBigDecimal(o.toString());
+ }
+
+ // Don't need shortValueOf etc. - Short.valueOf is sufficient.
+
+ /** Helper for CAST(... AS VARCHAR(maxLength)). */
+ public static String truncate(String s, int maxLength) {
+ return s == null ? null
+ : s.length() > maxLength ? s.substring(0, maxLength)
+ : s;
+ }
+
+ /** Helper for CAST(... AS VARBINARY(maxLength)). */
+ public static ByteString truncate(ByteString s, int maxLength) {
+ return s == null ? null
+ : s.length() > maxLength ? s.substring(0, maxLength)
+ : s;
+ }
+
+ /** SQL {@code POSITION(seek IN string)} function. */
+ public static int position(String seek, String s) {
+ return s.indexOf(seek) + 1;
+ }
+
+ /** SQL {@code POSITION(seek IN string)} function. */
+ public static int position(ByteString seek, ByteString s) {
+ return s.indexOf(seek) + 1;
+ }
+
+ /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
+ public static long round(long v, long x) {
+ return truncate(v + x / 2, x);
+ }
+
+ /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
+ public static long truncate(long v, long x) {
+ long remainder = v % x;
+ if (remainder < 0) {
+ remainder += x;
+ }
+ return v - remainder;
+ }
+
+ /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
+ public static int round(int v, int x) {
+ return truncate(v + x / 2, x);
+ }
+
+ /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
+ public static int truncate(int v, int x) {
+ int remainder = v % x;
+ if (remainder < 0) {
+ remainder += x;
+ }
+ return v - remainder;
+ }
+
+ /** SQL {@code CURRENT_TIMESTAMP} function. */
+ @NonDeterministic
+ public static long currentTimestamp(DataContext root) {
+ // Cast required for JDK 1.6.
+ return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root);
+ }
+
+ /** SQL {@code CURRENT_TIME} function. */
+ @NonDeterministic
+ public static int currentTime(DataContext root) {
+ int time = (int) (currentTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY);
+ if (time < 0) {
+ time += DateTimeUtils.MILLIS_PER_DAY;
+ }
+ return time;
+ }
+
+ /** SQL {@code CURRENT_DATE} function. */
+ @NonDeterministic
+ public static int currentDate(DataContext root) {
+ final long timestamp = currentTimestamp(root);
+ int date = (int) (timestamp / DateTimeUtils.MILLIS_PER_DAY);
+ final int time = (int) (timestamp % DateTimeUtils.MILLIS_PER_DAY);
+ if (time < 0) {
+ --date;
+ }
+ return date;
+ }
+
+ /** SQL {@code LOCAL_TIMESTAMP} function. */
+ @NonDeterministic
+ public static long localTimestamp(DataContext root) {
+ // Cast required for JDK 1.6.
+ return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root);
+ }
+
+ /** SQL {@code LOCAL_TIME} function. */
+ @NonDeterministic
+ public static int localTime(DataContext root) {
+ return (int) (localTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY);
+ }
+
+ /** Helper for "array element reference". Caller has already ensured that
+ * array and index are not null. Index is 1-based, per SQL. */
+ public static Object arrayItem(List list, int item) {
+ if (item < 1 || item > list.size()) {
+ return null;
+ }
+ return list.get(item - 1);
+ }
+
+ /** Helper for "map element reference". Caller has already ensured that
+ * array and index are not null. Index is 1-based, per SQL. */
+ public static Object mapItem(Map map, Object item) {
+ return map.get(item);
+ }
+
+ /** Implements the {@code [ ... ]} operator on an object whose type is not
+ * known until runtime.
+ */
+ public static Object item(Object object, Object index) {
+ if (object instanceof Map) {
+ return ((Map) object).get(index);
+ }
+ if (object instanceof List && index instanceof Number) {
+ List list = (List) object;
+ return list.get(((Number) index).intValue());
+ }
+ return null;
+ }
+
+ /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */
+ public static boolean isTrue(Boolean b) {
+ return b != null && b;
+ }
+
+ /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */
+ public static boolean isNotFalse(Boolean b) {
+ return b == null || b;
+ }
+
+ /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */
+ public static Boolean not(Boolean b) {
+ return (b == null) ? null : !b;
+ }
+
+ /** Converts a JDBC array to a list. */
+ public static List arrayToList(final java.sql.Array a) {
+ if (a == null) {
+ return null;
+ }
+ try {
+ return Primitive.asList(a.getArray());
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Support the {@code CURRENT VALUE OF sequence} operator. */
+ @NonDeterministic
+ public static long sequenceCurrentValue(String key) {
+ return getAtomicLong(key).get();
+ }
+
+ /** Support the {@code NEXT VALUE OF sequence} operator. */
+ @NonDeterministic
+ public static long sequenceNextValue(String key) {
+ return getAtomicLong(key).incrementAndGet();
+ }
+
+ private static AtomicLong getAtomicLong(String key) {
+ final Map<String, AtomicLong> map = THREAD_SEQUENCES.get();
+ AtomicLong atomic = map.get(key);
+ if (atomic == null) {
+ atomic = new AtomicLong();
+ map.put(key, atomic);
+ }
+ return atomic;
+ }
+
+ /** Support the SLICE function. */
+ public static List slice(List list) {
+ return list;
+ }
+
+ /** Support the ELEMENT function. */
+ public static Object element(List list) {
+ switch (list.size()) {
+ case 0:
+ return null;
+ case 1:
+ return list.get(0);
+ default:
+ throw new RuntimeException("more than one value");
+ }
+ }
+
+ /** Returns a lambda that converts a list to an enumerable. */
+ public static <E> Function1<List<E>, Enumerable<E>> listToEnumerable() {
+ //noinspection unchecked
+ return (Function1<List<E>, Enumerable<E>>) (Function1) LIST_AS_ENUMERABLE;
+ }
+
+}
+
+// End SqlFunctions.java
[34/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
KYLIN-780 Upgrade query module, all query test pass
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f4be1526
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f4be1526
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f4be1526
Branch: refs/heads/0.7
Commit: f4be1526e86d737998e26c6f533a7b615a5e65d0
Parents: 725bde5
Author: Yang Li <li...@apache.org>
Authored: Tue Jun 16 07:53:53 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 6 13:48:09 2015 +0800
----------------------------------------------------------------------
.../hydromatic/optiq/runtime/SqlFunctions.java | 1705 ------
.../adapter/enumerable/EnumerableJoin.java | 187 +
.../apache/calcite/runtime/SqlFunctions.java | 1261 +++++
.../calcite/sql2rel/SqlToRelConverter.java | 4978 ++++++++++++++++++
.../eigenbase/sql2rel/SqlToRelConverter.java | 4795 -----------------
jdbc/pom.xml | 2 +-
.../main/java/org/apache/kylin/jdbc/Driver.java | 12 +-
.../apache/kylin/jdbc/KylinConnectionImpl.java | 12 +-
.../org/apache/kylin/jdbc/KylinEnumerator.java | 2 +-
.../apache/kylin/jdbc/KylinJdbc41Factory.java | 20 +-
.../org/apache/kylin/jdbc/KylinMetaImpl.java | 24 +-
.../org/apache/kylin/jdbc/KylinPrepare.java | 10 +-
.../org/apache/kylin/jdbc/KylinResultSet.java | 6 +-
.../apache/kylin/jdbc/KylinStatementImpl.java | 6 +-
.../org/apache/kylin/jdbc/stub/DataSet.java | 4 +-
.../org/apache/kylin/jdbc/stub/KylinClient.java | 8 +-
.../apache/kylin/jdbc/stub/RemoteClient.java | 2 +-
.../apache/kylin/metadata/model/DataType.java | 3 +-
.../kylin/metadata/model/FunctionDesc.java | 8 +-
.../kylin/metadata/project/ProjectL2Cache.java | 2 +-
pom.xml | 19 +-
query/pom.xml | 6 +-
.../kylin/query/enumerator/CubeEnumerator.java | 20 +-
.../kylin/query/enumerator/HiveEnumerator.java | 8 +-
.../query/enumerator/LookupTableEnumerator.java | 9 +-
.../kylin/query/enumerator/OLAPQuery.java | 12 +-
.../kylin/query/optrule/OLAPAggregateRule.java | 22 +-
.../kylin/query/optrule/OLAPFilterRule.java | 18 +-
.../kylin/query/optrule/OLAPJoinRule.java | 28 +-
.../kylin/query/optrule/OLAPLimitRule.java | 19 +-
.../kylin/query/optrule/OLAPProjectRule.java | 18 +-
.../kylin/query/optrule/OLAPSortRule.java | 19 +-
.../optrule/OLAPToEnumerableConverterRule.java | 13 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 136 +-
.../apache/kylin/query/relnode/OLAPContext.java | 8 +-
.../kylin/query/relnode/OLAPFilterRel.java | 98 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 91 +-
.../kylin/query/relnode/OLAPLimitRel.java | 54 +-
.../kylin/query/relnode/OLAPProjectRel.java | 122 +-
.../org/apache/kylin/query/relnode/OLAPRel.java | 62 +-
.../apache/kylin/query/relnode/OLAPSortRel.java | 62 +-
.../kylin/query/relnode/OLAPTableScan.java | 106 +-
.../relnode/OLAPToEnumerableConverter.java | 57 +-
.../apache/kylin/query/schema/OLAPSchema.java | 6 +-
.../kylin/query/schema/OLAPSchemaFactory.java | 22 +-
.../apache/kylin/query/schema/OLAPTable.java | 60 +-
.../apache/kylin/query/sqlfunc/QuarterBase.java | 38 -
.../apache/kylin/query/sqlfunc/QuarterFunc.java | 34 -
.../apache/kylin/query/test/KylinTestBase.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 16 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 2 +-
.../endpoint/EndpointTupleIterator.java | 2 +-
52 files changed, 7015 insertions(+), 7221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java b/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java
deleted file mode 100644
index db99c83..0000000
--- a/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java
+++ /dev/null
@@ -1,1705 +0,0 @@
-/*
- * OVERRIDE POINT:
- * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);`
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.hydromatic.optiq.runtime;
-
-import net.hydromatic.avatica.ByteString;
-
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Linq4j;
-import net.hydromatic.linq4j.expressions.Primitive;
-import net.hydromatic.linq4j.function.Deterministic;
-import net.hydromatic.linq4j.function.Function1;
-import net.hydromatic.linq4j.function.NonDeterministic;
-
-import net.hydromatic.optiq.DataContext;
-
-import org.eigenbase.util14.DateTimeUtil;
-
-import java.math.*;
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.text.DecimalFormat;
-import java.util.*;
-import java.util.regex.Pattern;
-
-/**
- * Helper methods to implement SQL functions in generated code.
- *
- * <p>Not present: and, or, not (builtin operators are better, because they
- * use lazy evaluation. Implementations do not check for null values; the
- * calling code must do that.</p>
- *
- * <p>Many of the functions do not check for null values. This is intentional.
- * If null arguments are possible, the code-generation framework checks for
- * nulls before calling the functions.</p>
- */
-@SuppressWarnings("UnnecessaryUnboxing")
-@Deterministic
-public class SqlFunctions {
- private static final DecimalFormat DOUBLE_FORMAT =
- new DecimalFormat("0.0E0");
-
- /** The julian date of the epoch, 1970-01-01. */
- public static final int EPOCH_JULIAN = 2440588;
-
- private static final TimeZone LOCAL_TZ = TimeZone.getDefault();
-
- private static final Function1<List<Object>, Enumerable<Object>>
- LIST_AS_ENUMERABLE =
- new Function1<List<Object>, Enumerable<Object>>() {
- public Enumerable<Object> apply(List<Object> list) {
- return Linq4j.asEnumerable(list);
- }
- };
-
- private SqlFunctions() {
- }
-
- /** SQL SUBSTRING(string FROM ... FOR ...) function. */
- public static String substring(String s, int from, int for_) {
- return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
- }
-
- /** SQL SUBSTRING(string FROM ...) function. */
- public static String substring(String s, int from) {
- return s.substring(from - 1);
- }
-
- /** SQL UPPER(string) function. */
- public static String upper(String s) {
- return s.toUpperCase();
- }
-
- /** SQL LOWER(string) function. */
- public static String lower(String s) {
- return s.toLowerCase();
- }
-
- /** SQL INITCAP(string) function. */
- public static String initcap(String s) {
- // Assumes Alpha as [A-Za-z0-9]
- // white space is treated as everything else.
- final int len = s.length();
- boolean start = true;
- final StringBuilder newS = new StringBuilder();
-
- for (int i = 0; i < len; i++) {
- char curCh = s.charAt(i);
- final int c = (int) curCh;
- if (start) { // curCh is whitespace or first character of word.
- if (c > 47 && c < 58) { // 0-9
- start = false;
- } else if (c > 64 && c < 91) { // A-Z
- start = false;
- } else if (c > 96 && c < 123) { // a-z
- start = false;
- curCh = (char) (c - 32); // Uppercase this character
- }
- // else {} whitespace
- } else { // Inside of a word or white space after end of word.
- if (c > 47 && c < 58) { // 0-9
- // noop
- } else if (c > 64 && c < 91) { // A-Z
- curCh = (char) (c + 32); // Lowercase this character
- } else if (c > 96 && c < 123) { // a-z
- // noop
- } else { // whitespace
- start = true;
- }
- }
- newS.append(curCh);
- } // for each character in s
- return newS.toString();
- }
-
- /** SQL CHARACTER_LENGTH(string) function. */
- public static int charLength(String s) {
- return s.length();
- }
-
- /** SQL {@code string || string} operator. */
- public static String concat(String s0, String s1) {
- return s0 + s1;
- }
-
- /** SQL {@code binary || binary} operator. */
- public static ByteString concat(ByteString s0, ByteString s1) {
- return s0.concat(s1);
- }
-
- /** SQL {@code RTRIM} function applied to string. */
- public static String rtrim(String s) {
- return trim_(s, false, true, ' ');
- }
-
- /** SQL {@code LTRIM} function. */
- public static String ltrim(String s) {
- return trim_(s, true, false, ' ');
- }
-
- /** SQL {@code TRIM(... seek FROM s)} function. */
- public static String trim(boolean leading, boolean trailing, String seek,
- String s) {
- return trim_(s, leading, trailing, seek.charAt(0));
- }
-
- /** SQL {@code TRIM} function. */
- private static String trim_(String s, boolean left, boolean right, char c) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return "";
- }
- if (s.charAt(j - 1) != c) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return "";
- }
- if (s.charAt(i) != c) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- public static ByteString trim(ByteString s) {
- return trim_(s, true, true);
- }
-
- /** Helper for CAST. */
- public static ByteString rtrim(ByteString s) {
- return trim_(s, false, true);
- }
-
- /** SQL {@code TRIM} function applied to binary string. */
- private static ByteString trim_(ByteString s, boolean left, boolean right) {
- int j = s.length();
- if (right) {
- for (;;) {
- if (j == 0) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(j - 1) != 0) {
- break;
- }
- --j;
- }
- }
- int i = 0;
- if (left) {
- for (;;) {
- if (i == j) {
- return ByteString.EMPTY;
- }
- if (s.byteAt(i) != 0) {
- break;
- }
- ++i;
- }
- }
- return s.substring(i, j);
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- + r
- + s.substring(start - 1 + r.length());
- }
-
- /** SQL {@code OVERLAY} function. */
- public static String overlay(String s, String r, int start, int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- + r
- + s.substring(start - 1 + length);
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- .concat(r)
- .concat(s.substring(start - 1 + r.length()));
- }
-
- /** SQL {@code OVERLAY} function applied to binary strings. */
- public static ByteString overlay(ByteString s, ByteString r, int start,
- int length) {
- if (s == null || r == null) {
- return null;
- }
- return s.substring(0, start - 1)
- .concat(r)
- .concat(s.substring(start - 1 + length));
- }
-
- /** SQL {@code LIKE} function. */
- public static boolean like(String s, String pattern) {
- final String regex = Like.sqlToRegexLike(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code LIKE} function with escape. */
- public static boolean like(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexLike(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function. */
- public static boolean similar(String s, String pattern) {
- final String regex = Like.sqlToRegexSimilar(pattern, null);
- return Pattern.matches(regex, s);
- }
-
- /** SQL {@code SIMILAR} function with escape. */
- public static boolean similar(String s, String pattern, String escape) {
- final String regex = Like.sqlToRegexSimilar(pattern, escape);
- return Pattern.matches(regex, s);
- }
-
- // =
-
- /** SQL = operator applied to Object values (including String; neither
- * side may be null). */
- public static boolean eq(Object b0, Object b1) {
- return b0.equals(b1);
- }
-
- /** SQL = operator applied to BigDecimal values (neither may be null). */
- public static boolean eq(BigDecimal b0, BigDecimal b1) {
- return b0.stripTrailingZeros().equals(b1.stripTrailingZeros());
- }
-
- // <>
-
- /** SQL <> operator applied to Object values (including String;
- * neither side may be null). */
- public static boolean ne(Object b0, Object b1) {
- return !b0.equals(b1);
- }
-
- /** SQL <> operator applied to BigDecimal values. */
- public static boolean ne(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) != 0;
- }
-
- // <
-
- /** SQL < operator applied to boolean values. */
- public static boolean lt(boolean b0, boolean b1) {
- return compare(b0, b1) < 0;
- }
-
- /** SQL < operator applied to String values. */
- public static boolean lt(String b0, String b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to ByteString values. */
- public static boolean lt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) < 0;
- }
-
- /** SQL < operator applied to BigDecimal values. */
- public static boolean lt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) < 0;
- }
-
- // <=
-
- /** SQL ≤ operator applied to boolean values. */
- public static boolean le(boolean b0, boolean b1) {
- return compare(b0, b1) <= 0;
- }
-
- /** SQL ≤ operator applied to String values. */
- public static boolean le(String b0, String b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to ByteString values. */
- public static boolean le(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- /** SQL ≤ operator applied to BigDecimal values. */
- public static boolean le(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) <= 0;
- }
-
- // >
-
- /** SQL > operator applied to boolean values. */
- public static boolean gt(boolean b0, boolean b1) {
- return compare(b0, b1) > 0;
- }
-
- /** SQL > operator applied to String values. */
- public static boolean gt(String b0, String b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to ByteString values. */
- public static boolean gt(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) > 0;
- }
-
- /** SQL > operator applied to BigDecimal values. */
- public static boolean gt(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) > 0;
- }
-
- // >=
-
- /** SQL ≥ operator applied to boolean values. */
- public static boolean ge(boolean b0, boolean b1) {
- return compare(b0, b1) >= 0;
- }
-
- /** SQL ≥ operator applied to String values. */
- public static boolean ge(String b0, String b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to ByteString values. */
- public static boolean ge(ByteString b0, ByteString b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- /** SQL ≥ operator applied to BigDecimal values. */
- public static boolean ge(BigDecimal b0, BigDecimal b1) {
- return b0.compareTo(b1) >= 0;
- }
-
- // +
-
- /** SQL <code>+</code> operator applied to int values. */
- public static int plus(int b0, int b1) {
- return b0 + b1;
- }
-
- /** SQL <code>+</code> operator applied to int values; left side may be
- * null. */
- public static Integer plus(Integer b0, int b1) {
- return b0 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to int values; right side may be
- * null. */
- public static Integer plus(int b0, Integer b1) {
- return b1 == null ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable int values. */
- public static Integer plus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 + b1);
- }
-
- /** SQL <code>+</code> operator applied to nullable long and int values. */
- public static Long plus(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to nullable int and long values. */
- public static Long plus(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() + b1.longValue());
- }
-
- /** SQL <code>+</code> operator applied to BigDecimal values. */
- public static BigDecimal plus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.add(b1);
- }
-
- // -
-
- /** SQL <code>-</code> operator applied to int values. */
- public static int minus(int b0, int b1) {
- return b0 - b1;
- }
-
- /** SQL <code>-</code> operator applied to int values; left side may be
- * null. */
- public static Integer minus(Integer b0, int b1) {
- return b0 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to int values; right side may be
- * null. */
- public static Integer minus(int b0, Integer b1) {
- return b1 == null ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable int values. */
- public static Integer minus(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 - b1);
- }
-
- /** SQL <code>-</code> operator applied to nullable long and int values. */
- public static Long minus(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to nullable int and long values. */
- public static Long minus(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() - b1.longValue());
- }
-
- /** SQL <code>-</code> operator applied to BigDecimal values. */
- public static BigDecimal minus(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.subtract(b1);
- }
-
- // /
-
- /** SQL <code>/</code> operator applied to int values. */
- public static int divide(int b0, int b1) {
- return b0 / b1;
- }
-
- /** SQL <code>/</code> operator applied to int values; left side may be
- * null. */
- public static Integer divide(Integer b0, int b1) {
- return b0 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to int values; right side may be
- * null. */
- public static Integer divide(int b0, Integer b1) {
- return b1 == null ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable int values. */
- public static Integer divide(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 / b1);
- }
-
- /** SQL <code>/</code> operator applied to nullable long and int values. */
- public static Long divide(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to nullable int and long values. */
- public static Long divide(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() / b1.longValue());
- }
-
- /** SQL <code>/</code> operator applied to BigDecimal values. */
- public static BigDecimal divide(BigDecimal b0, BigDecimal b1) {
- // OVERRIDE POINT
- return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64);
- }
-
- // *
-
- /** SQL <code>*</code> operator applied to int values. */
- public static int multiply(int b0, int b1) {
- return b0 * b1;
- }
-
- /** SQL <code>*</code> operator applied to int values; left side may be
- * null. */
- public static Integer multiply(Integer b0, int b1) {
- return b0 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to int values; right side may be
- * null. */
- public static Integer multiply(int b0, Integer b1) {
- return b1 == null ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable int values. */
- public static Integer multiply(Integer b0, Integer b1) {
- return (b0 == null || b1 == null) ? null : (b0 * b1);
- }
-
- /** SQL <code>*</code> operator applied to nullable long and int values. */
- public static Long multiply(Long b0, Integer b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to nullable int and long values. */
- public static Long multiply(Integer b0, Long b1) {
- return (b0 == null || b1 == null)
- ? null
- : (b0.longValue() * b1.longValue());
- }
-
- /** SQL <code>*</code> operator applied to BigDecimal values. */
- public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) {
- return (b0 == null || b1 == null) ? null : b0.multiply(b1);
- }
-
- // EXP
-
- /** SQL <code>EXP</code> operator applied to double values. */
- public static double exp(double b0) {
- return Math.exp(b0);
- }
-
- public static double exp(long b0) {
- return Math.exp(b0);
- }
-
- // POWER
-
- /** SQL <code>POWER</code> operator applied to double values. */
- public static double power(double b0, double b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, long b1) {
- return Math.pow(b0, b1);
- }
-
- public static double power(long b0, BigDecimal b1) {
- return Math.pow(b0, b1.doubleValue());
- }
-
- // LN
-
- /** SQL {@code LN(number)} function applied to double values. */
- public static double ln(double d) {
- return Math.log(d);
- }
-
- /** SQL {@code LN(number)} function applied to long values. */
- public static double ln(long b0) {
- return Math.log(b0);
- }
-
- /** SQL {@code LN(number)} function applied to BigDecimal values. */
- public static double ln(BigDecimal d) {
- return Math.log(d.doubleValue());
- }
-
- // LOG10
-
- /** SQL <code>LOG10(numeric)</code> operator applied to double values. */
- public static double log10(double b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to long values. */
- public static double log10(long b0) {
- return Math.log10(b0);
- }
-
- /** SQL {@code LOG10(number)} function applied to BigDecimal values. */
- public static double log10(BigDecimal d) {
- return Math.log10(d.doubleValue());
- }
-
- // MOD
-
- /** SQL <code>MOD</code> operator applied to byte values. */
- public static byte mod(byte b0, byte b1) {
- return (byte) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to short values. */
- public static short mod(short b0, short b1) {
- return (short) (b0 % b1);
- }
-
- /** SQL <code>MOD</code> operator applied to int values. */
- public static int mod(int b0, int b1) {
- return b0 % b1;
- }
-
- /** SQL <code>MOD</code> operator applied to long values. */
- public static long mod(long b0, long b1) {
- return b0 % b1;
- }
-
- // temporary
- public static BigDecimal mod(BigDecimal b0, int b1) {
- return mod(b0, BigDecimal.valueOf(b1));
- }
-
- // temporary
- public static int mod(int b0, BigDecimal b1) {
- return mod(b0, b1.intValue());
- }
-
- public static BigDecimal mod(BigDecimal b0, BigDecimal b1) {
- final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
- return bigDecimals[1];
- }
-
- // ABS
-
- /** SQL <code>ABS</code> operator applied to byte values. */
- public static byte abs(byte b0) {
- return (byte) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to short values. */
- public static short abs(short b0) {
- return (short) Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to int values. */
- public static int abs(int b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to long values. */
- public static long abs(long b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to float values. */
- public static float abs(float b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to double values. */
- public static double abs(double b0) {
- return Math.abs(b0);
- }
-
- /** SQL <code>ABS</code> operator applied to BigDecimal values. */
- public static BigDecimal abs(BigDecimal b0) {
- return b0.abs();
- }
-
- // Helpers
-
- /** Helper for implementing MIN. Somewhat similar to LEAST operator. */
- public static <T extends Comparable<T>> T lesser(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- /** LEAST operator. */
- public static <T extends Comparable<T>> T least(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0;
- }
-
- public static boolean greater(boolean b0, boolean b1) {
- return b0 || b1;
- }
-
- public static boolean lesser(boolean b0, boolean b1) {
- return b0 && b1;
- }
-
- public static byte greater(byte b0, byte b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static byte lesser(byte b0, byte b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static char greater(char b0, char b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static char lesser(char b0, char b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static short greater(short b0, short b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static short lesser(short b0, short b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static int greater(int b0, int b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static int lesser(int b0, int b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static long greater(long b0, long b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static long lesser(long b0, long b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static float greater(float b0, float b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static float lesser(float b0, float b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- public static double greater(double b0, double b1) {
- return b0 > b1 ? b0 : b1;
- }
-
- public static double lesser(double b0, double b1) {
- return b0 > b1 ? b1 : b0;
- }
-
- /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */
- public static <T extends Comparable<T>> T greater(T b0, T b1) {
- return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** GREATEST operator. */
- public static <T extends Comparable<T>> T greatest(T b0, T b1) {
- return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0;
- }
-
- /** Boolean comparison. */
- public static int compare(boolean x, boolean y) {
- return x == y ? 0 : x ? 1 : -1;
- }
-
- /** CAST(FLOAT AS VARCHAR). */
- public static String toString(float x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal =
- new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DOUBLE AS VARCHAR). */
- public static String toString(double x) {
- if (x == 0) {
- return "0E0";
- }
- BigDecimal bigDecimal =
- new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros();
- final String s = bigDecimal.toString();
- return s.replaceAll("0*E", "E").replace("E+", "E");
- }
-
- /** CAST(DECIMAL AS VARCHAR). */
- public static String toString(BigDecimal x) {
- final String s = x.toString();
- if (s.startsWith("0")) {
- // we want ".1" not "0.1"
- return s.substring(1);
- } else if (s.startsWith("-0")) {
- // we want "-.1" not "-0.1"
- return "-" + s.substring(2);
- } else {
- return s;
- }
- }
-
- /** CAST(BOOLEAN AS VARCHAR). */
- public static String toString(boolean x) {
- // Boolean.toString returns lower case -- no good.
- return x ? "TRUE" : "FALSE";
- }
-
- @NonDeterministic
- private static Object cannotConvert(Object o, Class toType) {
- throw new RuntimeException("Cannot convert " + o + " to " + toType);
- }
-
- /** CAST(VARCHAR AS BOOLEAN). */
- public static boolean toBoolean(String s) {
- s = trim_(s, true, true, ' ');
- if (s.equalsIgnoreCase("TRUE")) {
- return true;
- } else if (s.equalsIgnoreCase("FALSE")) {
- return false;
- } else {
- throw new RuntimeException("Invalid character for cast");
- }
- }
-
- public static boolean toBoolean(Number number) {
- return !number.equals(0);
- }
-
- public static boolean toBoolean(Object o) {
- return o instanceof Boolean ? (Boolean) o
- : o instanceof Number ? toBoolean((Number) o)
- : o instanceof String ? toBoolean((String) o)
- : (Boolean) cannotConvert(o, boolean.class);
- }
-
- // Don't need parseByte etc. - Byte.parseByte is sufficient.
-
- public static byte toByte(Object o) {
- return o instanceof Byte ? (Byte) o
- : o instanceof Number ? toByte((Number) o)
- : Byte.parseByte(o.toString());
- }
-
- public static byte toByte(Number number) {
- return number.byteValue();
- }
-
- public static char toChar(String s) {
- return s.charAt(0);
- }
-
- public static Character toCharBoxed(String s) {
- return s.charAt(0);
- }
-
- public static short toShort(String s) {
- return Short.parseShort(s.trim());
- }
-
- public static short toShort(Number number) {
- return number.shortValue();
- }
-
- public static short toShort(Object o) {
- return o instanceof Short ? (Short) o
- : o instanceof Number ? toShort((Number) o)
- : o instanceof String ? toShort((String) o)
- : (Short) cannotConvert(o, short.class);
- }
-
- public static int toInt(java.util.Date v) {
- return toInt(v, LOCAL_TZ);
- }
-
- public static int toInt(java.util.Date v, TimeZone timeZone) {
- return (int) (toLong(v, timeZone) / DateTimeUtil.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.util.Date v) {
- return v == null ? null : toInt(v);
- }
-
- public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) {
- return v == null
- ? null
- : toInt(v, timeZone);
- }
-
- public static long toLong(Date v) {
- return toLong(v, LOCAL_TZ);
- }
-
- public static int toInt(java.sql.Time v) {
- return (int) (toLong(v) % DateTimeUtil.MILLIS_PER_DAY);
- }
-
- public static Integer toIntOptional(java.sql.Time v) {
- return v == null ? null : toInt(v);
- }
-
- public static int toInt(String s) {
- return Integer.parseInt(s.trim());
- }
-
- public static int toInt(Number number) {
- return number.intValue();
- }
-
- public static int toInt(Object o) {
- return o instanceof Integer ? (Integer) o
- : o instanceof Number ? toInt((Number) o)
- : o instanceof String ? toInt((String) o)
- : (Integer) cannotConvert(o, int.class);
- }
-
- public static long toLong(Timestamp v) {
- return toLong(v, LOCAL_TZ);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static long toLong(java.util.Date v, TimeZone timeZone) {
- final long time = v.getTime();
- return time + timeZone.getOffset(time);
- }
-
- // mainly intended for java.sql.Timestamp but works for other dates also
- public static Long toLongOptional(java.util.Date v) {
- return v == null ? null : toLong(v, LOCAL_TZ);
- }
-
- public static Long toLongOptional(Timestamp v, TimeZone timeZone) {
- if (v == null) {
- return null;
- }
- return toLong(v, LOCAL_TZ);
- }
-
- public static long toLong(String s) {
- if (s.startsWith("199") && s.contains(":")) {
- return Timestamp.valueOf(s).getTime();
- }
- return Long.parseLong(s.trim());
- }
-
- public static long toLong(Number number) {
- return number.longValue();
- }
-
- public static long toLong(Object o) {
- return o instanceof Long ? (Long) o
- : o instanceof Number ? toLong((Number) o)
- : o instanceof String ? toLong((String) o)
- : (Long) cannotConvert(o, long.class);
- }
-
- public static float toFloat(String s) {
- return Float.parseFloat(s.trim());
- }
-
- public static float toFloat(Number number) {
- return number.floatValue();
- }
-
- public static float toFloat(Object o) {
- return o instanceof Float ? (Float) o
- : o instanceof Number ? toFloat((Number) o)
- : o instanceof String ? toFloat((String) o)
- : (Float) cannotConvert(o, float.class);
- }
-
- public static double toDouble(String s) {
- return Double.parseDouble(s.trim());
- }
-
- public static double toDouble(Number number) {
- return number.doubleValue();
- }
-
- public static double toDouble(Object o) {
- return o instanceof Double ? (Double) o
- : o instanceof Number ? toDouble((Number) o)
- : o instanceof String ? toDouble((String) o)
- : (Double) cannotConvert(o, double.class);
- }
-
- public static BigDecimal toBigDecimal(String s) {
- return new BigDecimal(s.trim());
- }
-
- public static BigDecimal toBigDecimal(Number number) {
- // There are some values of "long" that cannot be represented as "double".
- // Not so "int". If it isn't a long, go straight to double.
- return number instanceof BigDecimal ? (BigDecimal) number
- : number instanceof BigInteger ? new BigDecimal((BigInteger) number)
- : number instanceof Long ? new BigDecimal(number.longValue())
- : new BigDecimal(number.doubleValue());
- }
-
- public static BigDecimal toBigDecimal(Object o) {
- return o instanceof Number ? toBigDecimal((Number) o)
- : toBigDecimal(o.toString());
- }
-
- // Don't need shortValueOf etc. - Short.valueOf is sufficient.
-
- /** Helper for CAST(... AS VARCHAR(maxLength)). */
- public static String truncate(String s, int maxLength) {
- return s == null ? null
- : s.length() > maxLength ? s.substring(0, maxLength)
- : s;
- }
-
- /** Helper for CAST(... AS VARBINARY(maxLength)). */
- public static ByteString truncate(ByteString s, int maxLength) {
- return s == null ? null
- : s.length() > maxLength ? s.substring(0, maxLength)
- : s;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(String seek, String s) {
- return s.indexOf(seek) + 1;
- }
-
- /** SQL {@code POSITION(seek IN string)} function. */
- public static int position(ByteString seek, ByteString s) {
- return s.indexOf(seek) + 1;
- }
-
- /** Cheap, unsafe, long power. power(2, 3) returns 8. */
- public static long powerX(long a, long b) {
- long x = 1;
- while (b > 0) {
- x *= a;
- --b;
- }
- return x;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long round(long v, long x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static long truncate(long v, long x) {
- long remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int round(int v, int x) {
- return truncate(v + x / 2, x);
- }
-
- /** Helper for rounding. Truncate(12345, 1000) returns 12000. */
- public static int truncate(int v, int x) {
- int remainder = v % x;
- if (remainder < 0) {
- remainder += x;
- }
- return v - remainder;
- }
-
- /** Helper for CAST({timestamp} AS VARCHAR(n)). */
- public static String unixTimestampToString(long timestamp) {
- final StringBuilder buf = new StringBuilder(17);
- int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY);
- int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- --date;
- time += DateTimeUtil.MILLIS_PER_DAY;
- }
- unixDateToString(buf, date);
- buf.append(' ');
- unixTimeToString(buf, time);
- return buf.toString();
- }
-
- /** Helper for CAST({timestamp} AS VARCHAR(n)). */
- public static String unixTimeToString(int time) {
- final StringBuilder buf = new StringBuilder(8);
- unixTimeToString(buf, time);
- return buf.toString();
- }
-
- private static void unixTimeToString(StringBuilder buf, int time) {
- int h = time / 3600000;
- int time2 = time % 3600000;
- int m = time2 / 60000;
- int time3 = time2 % 60000;
- int s = time3 / 1000;
- int ms = time3 % 1000;
- int2(buf, h);
- buf.append(':');
- int2(buf, m);
- buf.append(':');
- int2(buf, s);
- }
-
- /** SQL {@code CURRENT_TIMESTAMP} function. */
- @NonDeterministic
- public static long currentTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root);
- }
-
- /** SQL {@code CURRENT_TIME} function. */
- @NonDeterministic
- public static int currentTime(DataContext root) {
- int time = (int) (currentTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- time += DateTimeUtil.MILLIS_PER_DAY;
- }
- return time;
- }
-
- /** SQL {@code CURRENT_DATE} function. */
- @NonDeterministic
- public static int currentDate(DataContext root) {
- final long timestamp = currentTimestamp(root);
- int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY);
- final int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY);
- if (time < 0) {
- --date;
- }
- return date;
- }
-
- /** SQL {@code LOCAL_TIMESTAMP} function. */
- @NonDeterministic
- public static long localTimestamp(DataContext root) {
- // Cast required for JDK 1.6.
- return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root);
- }
-
- /** SQL {@code LOCAL_TIME} function. */
- @NonDeterministic
- public static int localTime(DataContext root) {
- return (int) (localTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY);
- }
-
- private static void int2(StringBuilder buf, int i) {
- buf.append((char) ('0' + (i / 10) % 10));
- buf.append((char) ('0' + i % 10));
- }
-
- private static void int4(StringBuilder buf, int i) {
- buf.append((char) ('0' + (i / 1000) % 10));
- buf.append((char) ('0' + (i / 100) % 10));
- buf.append((char) ('0' + (i / 10) % 10));
- buf.append((char) ('0' + i % 10));
- }
-
- public static int dateStringToUnixDate(String s) {
- int hyphen1 = s.indexOf('-');
- int y;
- int m;
- int d;
- if (hyphen1 < 0) {
- y = Integer.parseInt(s.trim());
- m = 1;
- d = 1;
- } else {
- y = Integer.parseInt(s.substring(0, hyphen1).trim());
- final int hyphen2 = s.indexOf('-', hyphen1 + 1);
- if (hyphen2 < 0) {
- m = Integer.parseInt(s.substring(hyphen1 + 1).trim());
- d = 1;
- } else {
- m = Integer.parseInt(s.substring(hyphen1 + 1, hyphen2).trim());
- d = Integer.parseInt(s.substring(hyphen2 + 1).trim());
- }
- }
- return ymdToUnixDate(y, m, d);
- }
-
- public static int timeStringToUnixDate(String v) {
- return timeStringToUnixDate(v, 0);
- }
-
- public static int timeStringToUnixDate(String v, int start) {
- final int colon1 = v.indexOf(':', start);
- int hour;
- int minute;
- int second;
- int milli;
- if (colon1 < 0) {
- hour = Integer.parseInt(v.trim());
- minute = 1;
- second = 1;
- milli = 0;
- } else {
- hour = Integer.parseInt(v.substring(start, colon1).trim());
- final int colon2 = v.indexOf(':', colon1 + 1);
- if (colon2 < 0) {
- minute = Integer.parseInt(v.substring(colon1 + 1).trim());
- second = 1;
- milli = 0;
- } else {
- minute = Integer.parseInt(v.substring(colon1 + 1, colon2).trim());
- int dot = v.indexOf('.', colon2);
- if (dot < 0) {
- second = Integer.parseInt(v.substring(colon2 + 1).trim());
- milli = 0;
- } else {
- second = Integer.parseInt(v.substring(colon2 + 1, dot).trim());
- milli = Integer.parseInt(v.substring(dot + 1).trim());
- }
- }
- }
- return hour * (int) DateTimeUtil.MILLIS_PER_HOUR
- + minute * (int) DateTimeUtil.MILLIS_PER_MINUTE
- + second * (int) DateTimeUtil.MILLIS_PER_SECOND
- + milli;
- }
-
- public static long timestampStringToUnixDate(String s) {
- final long d;
- final long t;
- s = s.trim();
- int space = s.indexOf(' ');
- if (space >= 0) {
- d = dateStringToUnixDate(s.substring(0, space));
- t = timeStringToUnixDate(s, space + 1);
- } else {
- d = dateStringToUnixDate(s);
- t = 0;
- }
- return d * DateTimeUtil.MILLIS_PER_DAY + t;
- }
-
- /** Helper for CAST({date} AS VARCHAR(n)). */
- public static String unixDateToString(int date) {
- final StringBuilder buf = new StringBuilder(10);
- unixDateToString(buf, date);
- return buf.toString();
- }
-
- private static void unixDateToString(StringBuilder buf, int date) {
- julianToString(buf, date + EPOCH_JULIAN);
- }
-
- private static void julianToString(StringBuilder buf, int julian) {
- // this shifts the epoch back to astronomical year -4800 instead of the
- // start of the Christian era in year AD 1 of the proleptic Gregorian
- // calendar.
- int j = julian + 32044;
- int g = j / 146097;
- int dg = j % 146097;
- int c = (dg / 36524 + 1) * 3 / 4;
- int dc = dg - c * 36524;
- int b = dc / 1461;
- int db = dc % 1461;
- int a = (db / 365 + 1) * 3 / 4;
- int da = db - a * 365;
-
- // integer number of full years elapsed since March 1, 4801 BC
- int y = g * 400 + c * 100 + b * 4 + a;
- // integer number of full months elapsed since the last March 1
- int m = (da * 5 + 308) / 153 - 2;
- // number of days elapsed since day 1 of the month
- int d = da - (m + 4) * 153 / 5 + 122;
- int year = y - 4800 + (m + 2) / 12;
- int month = (m + 2) % 12 + 1;
- int day = d + 1;
- int4(buf, year);
- buf.append('-');
- int2(buf, month);
- buf.append('-');
- int2(buf, day);
- }
-
- public static long unixDateExtract(TimeUnitRange range, long date) {
- return julianExtract(range, (int) date + EPOCH_JULIAN);
- }
-
- private static int julianExtract(TimeUnitRange range, int julian) {
- // this shifts the epoch back to astronomical year -4800 instead of the
- // start of the Christian era in year AD 1 of the proleptic Gregorian
- // calendar.
- int j = julian + 32044;
- int g = j / 146097;
- int dg = j % 146097;
- int c = (dg / 36524 + 1) * 3 / 4;
- int dc = dg - c * 36524;
- int b = dc / 1461;
- int db = dc % 1461;
- int a = (db / 365 + 1) * 3 / 4;
- int da = db - a * 365;
-
- // integer number of full years elapsed since March 1, 4801 BC
- int y = g * 400 + c * 100 + b * 4 + a;
- // integer number of full months elapsed since the last March 1
- int m = (da * 5 + 308) / 153 - 2;
- // number of days elapsed since day 1 of the month
- int d = da - (m + 4) * 153 / 5 + 122;
- int year = y - 4800 + (m + 2) / 12;
- int month = (m + 2) % 12 + 1;
- int day = d + 1;
- switch (range) {
- case YEAR:
- return year;
- case MONTH:
- return month;
- case DAY:
- return day;
- default:
- throw new AssertionError(range);
- }
- }
-
- public static int ymdToUnixDate(int year, int month, int day) {
- final int julian = ymdToJulian(year, month, day);
- return julian - EPOCH_JULIAN;
- }
-
- public static int ymdToJulian(int year, int month, int day) {
- int a = (14 - month) / 12;
- int y = year + 4800 - a;
- int m = month + 12 * a - 3;
- int j = day + (153 * m + 2) / 5
- + 365 * y
- + y / 4
- - y / 100
- + y / 400
- - 32045;
- if (j < 2299161) {
- j = day + (153 * m + 2) / 5 + 365 * y + y / 4 - 32083;
- }
- return j;
- }
-
- public static String intervalYearMonthToString(int v, TimeUnitRange range) {
- final StringBuilder buf = new StringBuilder();
- if (v >= 0) {
- buf.append('+');
- } else {
- buf.append('-');
- v = -v;
- }
- final int y;
- final int m;
- switch (range) {
- case YEAR:
- v = roundUp(v, 12);
- y = v / 12;
- buf.append(y);
- break;
- case YEAR_TO_MONTH:
- y = v / 12;
- buf.append(y);
- buf.append('-');
- m = v % 12;
- number(buf, m, 2);
- break;
- case MONTH:
- m = v;
- buf.append(m);
- break;
- default:
- throw new AssertionError(range);
- }
- return buf.toString();
- }
-
- private static StringBuilder number(StringBuilder buf, int v, int n) {
- for (int k = digitCount(v); k < n; k++) {
- buf.append('0');
- }
- return buf.append(v);
- }
-
- public static int digitCount(int v) {
- for (int n = 1;; n++) {
- v /= 10;
- if (v == 0) {
- return n;
- }
- }
- }
-
- public static String intervalDayTimeToString(long v, TimeUnitRange range,
- int scale) {
- final StringBuilder buf = new StringBuilder();
- if (v >= 0) {
- buf.append('+');
- } else {
- buf.append('-');
- v = -v;
- }
- final long ms;
- final long s;
- final long m;
- final long h;
- final long d;
- switch (range) {
- case DAY_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- buf.append(':');
- number(buf, (int) m, 2);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case DAY_TO_MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- buf.append(':');
- number(buf, (int) m, 2);
- break;
- case DAY_TO_HOUR:
- v = roundUp(v, 1000 * 60 * 60);
- v /= 1000;
- v /= 60;
- v /= 60;
- h = v % 24;
- v /= 24;
- d = v;
- buf.append((int) d);
- buf.append(' ');
- number(buf, (int) h, 2);
- break;
- case DAY:
- v = roundUp(v, 1000 * 60 * 60 * 24);
- d = v / (1000 * 60 * 60 * 24);
- buf.append((int) d);
- break;
- case HOUR:
- v = roundUp(v, 1000 * 60 * 60);
- v /= 1000;
- v /= 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- break;
- case HOUR_TO_MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- buf.append(':');
- number(buf, (int) m, 2);
- break;
- case HOUR_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v % 60;
- v /= 60;
- h = v;
- buf.append((int) h);
- buf.append(':');
- number(buf, (int) m, 2);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case MINUTE_TO_SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v % 60;
- v /= 60;
- m = v;
- buf.append((int) m);
- buf.append(':');
- number(buf, (int) s, 2);
- fraction(buf, scale, ms);
- break;
- case MINUTE:
- v = roundUp(v, 1000 * 60);
- v /= 1000;
- v /= 60;
- m = v;
- buf.append((int) m);
- break;
- case SECOND:
- v = roundUp(v, powerX(10, 3 - scale));
- ms = v % 1000;
- v /= 1000;
- s = v;
- buf.append((int) s);
- fraction(buf, scale, ms);
- break;
- default:
- throw new AssertionError(range);
- }
- return buf.toString();
- }
-
- /**
- * Rounds a dividend to the nearest divisor.
- * For example roundUp(31, 10) yields 30; roundUp(37, 10) yields 40.
- * @param dividend Number to be divided
- * @param divisor Number to divide by
- * @return Rounded dividend
- */
- private static long roundUp(long dividend, long divisor) {
- long remainder = dividend % divisor;
- dividend -= remainder;
- if (remainder * 2 > divisor) {
- dividend += divisor;
- }
- return dividend;
- }
-
- private static int roundUp(int dividend, int divisor) {
- int remainder = dividend % divisor;
- dividend -= remainder;
- if (remainder * 2 > divisor) {
- dividend += divisor;
- }
- return dividend;
- }
-
- private static void fraction(StringBuilder buf, int scale, long ms) {
- if (scale > 0) {
- buf.append('.');
- long v1 = scale == 3 ? ms
- : scale == 2 ? ms / 10
- : scale == 1 ? ms / 100
- : 0;
- number(buf, (int) v1, scale);
- }
- }
-
- /** Helper for "array element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object arrayItem(List list, int item) {
- if (item < 1 || item > list.size()) {
- return null;
- }
- return list.get(item - 1);
- }
-
- /** Helper for "map element reference". Caller has already ensured that
- * array and index are not null. Index is 1-based, per SQL. */
- public static Object mapItem(Map map, Object item) {
- return map.get(item);
- }
-
- /** Implements the {@code [ ... ]} operator on an object whose type is not
- * known until runtime.
- */
- public static Object item(Object object, Object index) {
- if (object instanceof Map) {
- return ((Map) object).get(index);
- }
- if (object instanceof List && index instanceof Number) {
- List list = (List) object;
- return list.get(((Number) index).intValue());
- }
- return null;
- }
-
- /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isTrue(Boolean b) {
- return b != null && b;
- }
-
- /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */
- public static boolean isNotFalse(Boolean b) {
- return b == null || b;
- }
-
- /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */
- public static Boolean not(Boolean b) {
- return (b == null) ? null : !b;
- }
-
- /** Converts a JDBC array to a list. */
- public static List arrayToList(final java.sql.Array a) {
- if (a == null) {
- return null;
- }
- try {
- return Primitive.asList(a.getArray());
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- /** Support the SLICE function. */
- public static List slice(List list) {
- return list;
- }
-
- /** Support the ELEMENT function. */
- public static Object element(List list) {
- switch (list.size()) {
- case 0:
- return null;
- case 1:
- return list.get(0);
- default:
- throw new RuntimeException("more than one value");
- }
- }
-
- /** Returns a lambda that converts a list to an enumerable. */
- public static <E> Function1<List<E>, Enumerable<E>> listToEnumerable() {
- //noinspection unchecked
- return (Function1<List<E>, Enumerable<E>>) (Function1) LIST_AS_ENUMERABLE;
- }
-
- /** A range of time units. The first is more significant than the
- * other (e.g. year-to-day) or the same as the other
- * (e.g. month). */
- public enum TimeUnitRange {
- YEAR,
- YEAR_TO_MONTH,
- MONTH,
- DAY,
- DAY_TO_HOUR,
- DAY_TO_MINUTE,
- DAY_TO_SECOND,
- HOUR,
- HOUR_TO_MINUTE,
- HOUR_TO_SECOND,
- MINUTE,
- MINUTE_TO_SECOND,
- SECOND;
-
- /** Whether this is in the YEAR-TO-MONTH family of intervals. */
- public boolean monthly() {
- return ordinal() <= MONTH.ordinal();
- }
- }
-}
-
-// End SqlFunctions.java
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
new file mode 100644
index 0000000..918b4bf
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
@@ -0,0 +1,187 @@
+/*
+ * OVERRIDE POINT:
+ * - constructor was private instead of public
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.linq4j.tree.BlockBuilder;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.InvalidRelException;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelNodes;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.rules.EquiJoin;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.BuiltInMethod;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Util;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Set;
+
+/** Implementation of {@link org.apache.calcite.rel.core.Join} in
+ * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */
+public class EnumerableJoin extends EquiJoin implements EnumerableRel {
+ public EnumerableJoin( // OVERRIDE POINT, the constructor was private
+ RelOptCluster cluster,
+ RelTraitSet traits,
+ RelNode left,
+ RelNode right,
+ RexNode condition,
+ ImmutableIntList leftKeys,
+ ImmutableIntList rightKeys,
+ JoinRelType joinType,
+ Set<String> variablesStopped)
+ throws InvalidRelException {
+ super(
+ cluster,
+ traits,
+ left,
+ right,
+ condition,
+ leftKeys,
+ rightKeys,
+ joinType,
+ variablesStopped);
+ }
+
+ @Override public EnumerableJoin copy(RelTraitSet traitSet, RexNode condition,
+ RelNode left, RelNode right, JoinRelType joinType,
+ boolean semiJoinDone) {
+ final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
+ assert joinInfo.isEqui();
+ try {
+ return new EnumerableJoin(getCluster(), traitSet, left, right,
+ condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType,
+ variablesStopped);
+ } catch (InvalidRelException e) {
+ // Semantic error not possible. Must be a bug. Convert to
+ // internal error.
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ double rowCount = RelMetadataQuery.getRowCount(this);
+
+ // Joins can be flipped, and for many algorithms, both versions are viable
+ // and have the same cost. To make the results stable between versions of
+ // the planner, make one of the versions slightly more expensive.
+ switch (joinType) {
+ case RIGHT:
+ rowCount = addEpsilon(rowCount);
+ break;
+ default:
+ if (RelNodes.COMPARATOR.compare(left, right) > 0) {
+ rowCount = addEpsilon(rowCount);
+ }
+ }
+
+ // Cheaper if the smaller number of rows is coming from the LHS.
+ // Model this by adding L log L to the cost.
+ final double rightRowCount = right.getRows();
+ final double leftRowCount = left.getRows();
+ if (Double.isInfinite(leftRowCount)) {
+ rowCount = leftRowCount;
+ } else {
+ rowCount += Util.nLogN(leftRowCount);
+ }
+ if (Double.isInfinite(rightRowCount)) {
+ rowCount = rightRowCount;
+ } else {
+ rowCount += rightRowCount;
+ }
+ return planner.getCostFactory().makeCost(rowCount, 0, 0);
+ }
+
+ private double addEpsilon(double d) {
+ assert d >= 0d;
+ final double d0 = d;
+ if (d < 10) {
+ // For small d, adding 1 would change the value significantly.
+ d *= 1.001d;
+ if (d != d0) {
+ return d;
+ }
+ }
+ // For medium d, add 1. Keeps integral values integral.
+ ++d;
+ if (d != d0) {
+ return d;
+ }
+ // For large d, adding 1 might not change the value. Add .1%.
+ // If d is NaN, this still will probably not change the value. That's OK.
+ d *= 1.001d;
+ return d;
+ }
+
+ public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
+ BlockBuilder builder = new BlockBuilder();
+ final Result leftResult =
+ implementor.visitChild(this, 0, (EnumerableRel) left, pref);
+ Expression leftExpression =
+ builder.append(
+ "left", leftResult.block);
+ final Result rightResult =
+ implementor.visitChild(this, 1, (EnumerableRel) right, pref);
+ Expression rightExpression =
+ builder.append(
+ "right", rightResult.block);
+ final PhysType physType =
+ PhysTypeImpl.of(
+ implementor.getTypeFactory(), getRowType(), pref.preferArray());
+ final PhysType keyPhysType =
+ leftResult.physType.project(
+ leftKeys, JavaRowFormat.LIST);
+ return implementor.result(
+ physType,
+ builder.append(
+ Expressions.call(
+ leftExpression,
+ BuiltInMethod.JOIN.method,
+ Expressions.list(
+ rightExpression,
+ leftResult.physType.generateAccessor(leftKeys),
+ rightResult.physType.generateAccessor(rightKeys),
+ EnumUtils.joinSelector(joinType,
+ physType,
+ ImmutableList.of(
+ leftResult.physType, rightResult.physType)))
+ .append(
+ Util.first(keyPhysType.comparer(),
+ Expressions.constant(null)))
+ .append(
+ Expressions.constant(joinType.generatesNullsOnLeft()))
+ .append(
+ Expressions.constant(
+ joinType.generatesNullsOnRight())))).toBlock());
+ }
+
+}
+
+// End EnumerableJoin.java
[12/50] [abbrv] incubator-kylin git commit: KYLIN-540 compile issue
Posted by li...@apache.org.
KYLIN-540 compile issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c47813f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c47813f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c47813f5
Branch: refs/heads/0.7
Commit: c47813f59b449bdbd24acc627571ec2219c50035
Parents: b08df2b
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jun 27 09:15:18 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Jun 27 09:15:18 2015 +0800
----------------------------------------------------------------------
job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c47813f5/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index 13103ac..a373456 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -729,7 +729,7 @@ public class CubeMetadataUpgrade {
if (verify) {
instance = new CubeMetadataUpgrade(exportFolder);
instance.verify();
- instance.addDictionaryForFK();
+ instance.copyDictionaryForFK();
} else {
File oldMetaFolder = new File(exportFolder);
if (!oldMetaFolder.exists()) {
[49/50] [abbrv] incubator-kylin git commit: minor,
fix CliCommandExecutor to return command output
Posted by li...@apache.org.
minor, fix CliCommandExecutor to return command output
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/73621543
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/73621543
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/73621543
Branch: refs/heads/0.7
Commit: 73621543e320707c76c077057d0521a11b16d308
Parents: 44c09a2
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Jul 8 13:37:18 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Jul 8 13:37:18 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/util/CliCommandExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73621543/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index 20bcba5..336b206 100644
--- a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -131,7 +131,7 @@ public class CliCommandExecutor {
String line;
StringBuilder result = new StringBuilder();
while ((line = reader.readLine()) != null) {
- result.append("line").append('\n');
+ result.append(line).append('\n');
if (logAppender != null) {
logAppender.log(line);
}
[32/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
new file mode 100644
index 0000000..2508629
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -0,0 +1,4978 @@
+/*
+ * OVERRIDE POINT:
+ * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
+ * - isTrimUnusedFields(), override to false
+ * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql2rel;
+
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptQuery;
+import org.apache.calcite.plan.RelOptSamplingParameters;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.prepare.RelOptTableImpl;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Collect;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Sample;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
+import org.apache.calcite.rel.logical.LogicalTableModify;
+import org.apache.calcite.rel.logical.LogicalTableScan;
+import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCallBinding;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexFieldCollation;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.sql.JoinConditionType;
+import org.apache.calcite.sql.JoinType;
+import org.apache.calcite.sql.SemiJoinType;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlBasicCall;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlDelete;
+import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlInsert;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlMerge;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlSampleSpec;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.SqlSetOperator;
+import org.apache.calcite.sql.SqlUpdate;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
+import org.apache.calcite.sql.fun.SqlInOperator;
+import org.apache.calcite.sql.fun.SqlRowOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.calcite.sql.validate.AggregatingSelectScope;
+import org.apache.calcite.sql.validate.CollectNamespace;
+import org.apache.calcite.sql.validate.DelegatingScope;
+import org.apache.calcite.sql.validate.ListScope;
+import org.apache.calcite.sql.validate.ParameterScope;
+import org.apache.calcite.sql.validate.SelectScope;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+import org.apache.calcite.sql.validate.SqlQualified;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
+import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql.validate.SqlValidatorImpl;
+import org.apache.calcite.sql.validate.SqlValidatorNamespace;
+import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.NumberUtil;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.mapping.Mappings;
+import org.apache.calcite.util.trace.CalciteTrace;
+import org.apache.calcite.sql.SqlNumericLiteral;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/**
+ * Converts a SQL parse tree (consisting of
+ * {@link org.apache.calcite.sql.SqlNode} objects) into a relational algebra
+ * expression (consisting of {@link org.apache.calcite.rel.RelNode} objects).
+ *
+ * <p>The public entry points are: {@link #convertQuery},
+ * {@link #convertExpression(SqlNode)}.
+ */
+public class SqlToRelConverter {
+ //~ Static fields/initializers ---------------------------------------------
+
+ protected static final Logger SQL2REL_LOGGER =
+ CalciteTrace.getSqlToRelTracer();
+
+ private static final Function<SubQuery, SqlNode> FN =
+ new Function<SubQuery, SqlNode>() {
+ public SqlNode apply(SubQuery input) {
+ return input.node;
+ }
+ };
+ private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
+ //~ Instance fields --------------------------------------------------------
+
+ protected final SqlValidator validator;
+ protected final RexBuilder rexBuilder;
+ protected final Prepare.CatalogReader catalogReader;
+ protected final RelOptCluster cluster;
+ private DefaultValueFactory defaultValueFactory;
+ private SubqueryConverter subqueryConverter;
+ protected final List<RelNode> leaves = new ArrayList<RelNode>();
+ private final List<SqlDynamicParam> dynamicParamSqlNodes =
+ new ArrayList<SqlDynamicParam>();
+ private final SqlOperatorTable opTab;
+ private boolean shouldConvertTableAccess;
+ protected final RelDataTypeFactory typeFactory;
+ private final SqlNodeToRexConverter exprConverter;
+ private boolean decorrelationEnabled;
+ private boolean trimUnusedFields;
+ private boolean shouldCreateValuesRel;
+ private boolean isExplain;
+ private int nDynamicParamsInExplain;
+
+ /**
+ * Fields used in name resolution for correlated subqueries.
+ */
+ private final Map<String, DeferredLookup> mapCorrelToDeferred =
+ new HashMap<String, DeferredLookup>();
+ private int nextCorrel = 0;
+
+ private static final String CORREL_PREFIX = "$cor";
+
+ /**
+ * Stack of names of datasets requested by the <code>
+ * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
+ */
+ private final Stack<String> datasetStack = new Stack<String>();
+
+ /**
+ * Mapping of non-correlated subqueries that have been converted to their
+ * equivalent constants. Used to avoid re-evaluating the subquery if it's
+ * already been evaluated.
+ */
+ private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
+ new HashMap<SqlNode, RexNode>();
+
+ public final RelOptTable.ViewExpander viewExpander;
+
+ //~ Constructors -----------------------------------------------------------
+ /**
+ * Creates a converter.
+ *
+ * @param viewExpander Preparing statement
+ * @param validator Validator
+ * @param catalogReader Schema
+ * @param planner Planner
+ * @param rexBuilder Rex builder
+ * @param convertletTable Expression converter
+ */
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptPlanner planner,
+ RexBuilder rexBuilder,
+ SqlRexConvertletTable convertletTable) {
+ this.viewExpander = viewExpander;
+ this.opTab =
+ (validator
+ == null) ? SqlStdOperatorTable.instance()
+ : validator.getOperatorTable();
+ this.validator = validator;
+ this.catalogReader = catalogReader;
+ this.defaultValueFactory = new NullDefaultValueFactory();
+ this.subqueryConverter = new NoOpSubqueryConverter();
+ this.rexBuilder = rexBuilder;
+ this.typeFactory = rexBuilder.getTypeFactory();
+ RelOptQuery query = new RelOptQuery(planner);
+ this.cluster = query.createCluster(typeFactory, rexBuilder);
+ this.shouldConvertTableAccess = true;
+ this.exprConverter =
+ new SqlNodeToRexConverterImpl(convertletTable);
+ decorrelationEnabled = true;
+ trimUnusedFields = false;
+ shouldCreateValuesRel = true;
+ isExplain = false;
+ nDynamicParamsInExplain = 0;
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ /**
+ * @return the RelOptCluster in use.
+ */
+ public RelOptCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Returns the row-expression builder.
+ */
+ public RexBuilder getRexBuilder() {
+ return rexBuilder;
+ }
+
+ /**
+ * Returns the number of dynamic parameters encountered during translation;
+ * this must only be called after {@link #convertQuery}.
+ *
+ * @return number of dynamic parameters
+ */
+ public int getDynamicParamCount() {
+ return dynamicParamSqlNodes.size();
+ }
+
+ /**
+ * Returns the type inferred for a dynamic parameter.
+ *
+ * @param index 0-based index of dynamic parameter
+ * @return inferred type, never null
+ */
+ public RelDataType getDynamicParamType(int index) {
+ SqlNode sqlNode = dynamicParamSqlNodes.get(index);
+ if (sqlNode == null) {
+ throw Util.needToImplement("dynamic param type inference");
+ }
+ return validator.getValidatedNodeType(sqlNode);
+ }
+
+ /**
+ * Returns the current count of the number of dynamic parameters in an
+ * EXPLAIN PLAN statement.
+ *
+ * @param increment if true, increment the count
+ * @return the current count before the optional increment
+ */
+ public int getDynamicParamCountInExplain(boolean increment) {
+ int retVal = nDynamicParamsInExplain;
+ if (increment) {
+ ++nDynamicParamsInExplain;
+ }
+ return retVal;
+ }
+
+ /**
+ * @return mapping of non-correlated subqueries that have been converted to
+ * the constants that they evaluate to
+ */
+ public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
+ return mapConvertedNonCorrSubqs;
+ }
+
+ /**
+ * Adds to the current map of non-correlated converted subqueries the
+ * elements from another map that contains non-correlated subqueries that
+ * have been converted by another SqlToRelConverter.
+ *
+ * @param alreadyConvertedNonCorrSubqs the other map
+ */
+ public void addConvertedNonCorrSubqs(
+ Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+ mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
+ }
+
+ /**
+ * Set a new DefaultValueFactory. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param factory new DefaultValueFactory
+ */
+ public void setDefaultValueFactory(DefaultValueFactory factory) {
+ defaultValueFactory = factory;
+ }
+
+ /**
+ * Sets a new SubqueryConverter. To have any effect, this must be called
+ * before any convert method.
+ *
+ * @param converter new SubqueryConverter
+ */
+ public void setSubqueryConverter(SubqueryConverter converter) {
+ subqueryConverter = converter;
+ }
+
+ /**
+ * Indicates that the current statement is part of an EXPLAIN PLAN statement
+ *
+ * @param nDynamicParams number of dynamic parameters in the statement
+ */
+ public void setIsExplain(int nDynamicParams) {
+ isExplain = true;
+ nDynamicParamsInExplain = nDynamicParams;
+ }
+
+ /**
+ * Controls whether table access references are converted to physical rels
+ * immediately. The optimizer doesn't like leaf rels to have
+ * {@link Convention#NONE}. However, if we are doing further conversion
+ * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
+ * defer conversion. To have any effect, this must be called before any
+ * convert method.
+ *
+ * @param enabled true for immediate conversion (the default); false to
+ * generate logical LogicalTableScan instances
+ */
+ public void enableTableAccessConversion(boolean enabled) {
+ shouldConvertTableAccess = enabled;
+ }
+
+ /**
+ * Controls whether instances of
+ * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
+ * may not be supported by all physical implementations. To have any effect,
+ * this must be called before any convert method.
+ *
+ * @param enabled true to allow LogicalValues to be generated (the default);
+ * false to force substitution of Project+OneRow instead
+ */
+ public void enableValuesRelCreation(boolean enabled) {
+ shouldCreateValuesRel = enabled;
+ }
+
+ private void checkConvertedType(SqlNode query, RelNode result) {
+ if (!query.isA(SqlKind.DML)) {
+ // Verify that conversion from SQL to relational algebra did
+ // not perturb any type information. (We can't do this if the
+ // SQL statement is something like an INSERT which has no
+ // validator type information associated with its result,
+ // hence the namespace check above.)
+ RelDataType convertedRowType = result.getRowType();
+ if (!checkConvertedRowType(query, convertedRowType)) {
+ RelDataType validatedRowType =
+ validator.getValidatedNodeType(query);
+ validatedRowType = uniquifyFields(validatedRowType);
+ throw Util.newInternal("Conversion to relational algebra failed to "
+ + "preserve datatypes:\n"
+ + "validated type:\n"
+ + validatedRowType.getFullTypeString()
+ + "\nconverted type:\n"
+ + convertedRowType.getFullTypeString()
+ + "\nrel:\n"
+ + RelOptUtil.toString(result));
+ }
+ }
+ }
+
+ public RelNode flattenTypes(
+ RelNode rootRel,
+ boolean restructure) {
+ RelStructuredTypeFlattener typeFlattener =
+ new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
+ return typeFlattener.rewrite(rootRel, restructure);
+ }
+
+ /**
+ * If subquery is correlated and decorrelation is enabled, performs
+ * decorrelation.
+ *
+ * @param query Query
+ * @param rootRel Root relational expression
+ * @return New root relational expression after decorrelation
+ */
+ public RelNode decorrelate(SqlNode query, RelNode rootRel) {
+ if (!enableDecorrelation()) {
+ return rootRel;
+ }
+ final RelNode result = decorrelateQuery(rootRel);
+ if (result != rootRel) {
+ checkConvertedType(query, result);
+ }
+ return result;
+ }
+
+ /**
+ * Walks over a tree of relational expressions, replacing each
+ * {@link RelNode} with a 'slimmed down' relational expression that projects
+ * only the fields required by its consumer.
+ *
+ * <p>This may make things easier for the optimizer, by removing crud that
+ * would expand the search space, but is difficult for the optimizer itself
+ * to do it, because optimizer rules must preserve the number and type of
+ * fields. Hence, this transform that operates on the entire tree, similar
+ * to the {@link RelStructuredTypeFlattener type-flattening transform}.
+ *
+ * <p>Currently this functionality is disabled in farrago/luciddb; the
+ * default implementation of this method does nothing.
+ *
+ * @param rootRel Relational expression that is at the root of the tree
+ * @return Trimmed relational expression
+ */
+ public RelNode trimUnusedFields(RelNode rootRel) {
+ // Trim fields that are not used by their consumer.
+ if (isTrimUnusedFields()) {
+ final RelFieldTrimmer trimmer = newFieldTrimmer();
+ rootRel = trimmer.trim(rootRel);
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(
+ RelOptUtil.dumpPlan(
+ "Plan after trimming unused fields",
+ rootRel,
+ false,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ }
+ }
+ return rootRel;
+ }
+
+ /**
+ * Creates a RelFieldTrimmer.
+ *
+ * @return Field trimmer
+ */
+ protected RelFieldTrimmer newFieldTrimmer() {
+ return new RelFieldTrimmer(validator);
+ }
+
+ /**
+ * Converts an unvalidated query's parse tree into a relational expression.
+ *
+ * @param query Query to convert
+ * @param needsValidation Whether to validate the query before converting;
+ * <code>false</code> if the query has already been
+ * validated.
+ * @param top Whether the query is top-level, say if its result
+ * will become a JDBC result set; <code>false</code> if
+ * the query will be part of a view.
+ */
+ public RelNode convertQuery(
+ SqlNode query,
+ final boolean needsValidation,
+ final boolean top) {
+ if (needsValidation) {
+ query = validator.validate(query);
+ }
+
+ RelNode result = convertQueryRecursive(query, top, null);
+ checkConvertedType(query, result);
+
+ boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
+ if (dumpPlan) {
+ SQL2REL_LOGGER.fine(
+ RelOptUtil.dumpPlan(
+ "Plan after converting SqlNode to RelNode",
+ result,
+ false,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ }
+
+ return result;
+ }
+
+ protected boolean checkConvertedRowType(
+ SqlNode query,
+ RelDataType convertedRowType) {
+ RelDataType validatedRowType = validator.getValidatedNodeType(query);
+ validatedRowType = uniquifyFields(validatedRowType);
+
+ return RelOptUtil.equal(
+ "validated row type",
+ validatedRowType,
+ "converted row type",
+ convertedRowType,
+ false);
+ }
+
+ protected RelDataType uniquifyFields(RelDataType rowType) {
+ return validator.getTypeFactory().createStructType(
+ RelOptUtil.getFieldTypeList(rowType),
+ SqlValidatorUtil.uniquify(rowType.getFieldNames()));
+ }
+
+ /**
+ * Converts a SELECT statement's parse tree into a relational expression.
+ */
+ public RelNode convertSelect(SqlSelect select) {
+ final SqlValidatorScope selectScope = validator.getWhereScope(select);
+ final Blackboard bb = createBlackboard(selectScope, null);
+ convertSelectImpl(bb, select);
+ return bb.root;
+ }
+
+ /**
+ * Factory method for creating translation workspace.
+ */
+ protected Blackboard createBlackboard(
+ SqlValidatorScope scope,
+ Map<String, RexNode> nameToNodeMap) {
+ return new Blackboard(scope, nameToNodeMap);
+ }
+
+ /**
+ * Implementation of {@link #convertSelect(SqlSelect)}; derived class may
+ * override.
+ */
+ protected void convertSelectImpl(
+ final Blackboard bb,
+ SqlSelect select) {
+ convertFrom(
+ bb,
+ select.getFrom());
+ convertWhere(
+ bb,
+ select.getWhere());
+
+ List<SqlNode> orderExprList = new ArrayList<SqlNode>();
+ List<RelFieldCollation> collationList =
+ new ArrayList<RelFieldCollation>();
+ gatherOrderExprs(
+ bb,
+ select,
+ select.getOrderList(),
+ orderExprList,
+ collationList);
+ final RelCollation collation =
+ cluster.traitSetOf().canonize(RelCollationImpl.of(collationList));
+
+ if (validator.isAggregate(select)) {
+ convertAgg(
+ bb,
+ select,
+ orderExprList);
+ } else {
+ convertSelectList(
+ bb,
+ select,
+ orderExprList);
+ }
+
+ if (select.isDistinct()) {
+ distinctify(bb, true);
+ }
+ convertOrder(
+ select, bb, collation, orderExprList, select.getOffset(),
+ select.getFetch());
+ bb.setRoot(bb.root, true);
+ }
+
+ /**
+ * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
+ * a relational expression to make the results unique.
+ *
+ * <p>If the SELECT clause contains duplicate expressions, adds
+ * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are
+ * grouping on the minimal set of keys. The performance gain isn't huge, but
+ * it is difficult to detect these duplicate expressions later.
+ *
+ * @param bb Blackboard
+ * @param checkForDupExprs Check for duplicate expressions
+ */
+ private void distinctify(
+ Blackboard bb,
+ boolean checkForDupExprs) {
+ // Look for duplicate expressions in the project.
+ // Say we have 'select x, y, x, z'.
+ // Then dups will be {[2, 0]}
+ // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
+ RelNode rel = bb.root;
+ if (checkForDupExprs && (rel instanceof LogicalProject)) {
+ LogicalProject project = (LogicalProject) rel;
+ final List<RexNode> projectExprs = project.getProjects();
+ List<Integer> origins = new ArrayList<Integer>();
+ int dupCount = 0;
+ for (int i = 0; i < projectExprs.size(); i++) {
+ int x = findExpr(projectExprs.get(i), projectExprs, i);
+ if (x >= 0) {
+ origins.add(x);
+ ++dupCount;
+ } else {
+ origins.add(i);
+ }
+ }
+ if (dupCount == 0) {
+ distinctify(bb, false);
+ return;
+ }
+
+ final Map<Integer, Integer> squished = Maps.newHashMap();
+ final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
+ final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ if (origins.get(i) == i) {
+ squished.put(i, newProjects.size());
+ newProjects.add(RexInputRef.of2(i, fields));
+ }
+ }
+ rel =
+ new LogicalProject(
+ cluster,
+ rel,
+ Pair.left(newProjects),
+ Pair.right(newProjects),
+ LogicalProject.Flags.BOXED);
+
+ bb.root = rel;
+ distinctify(bb, false);
+ rel = bb.root;
+
+ // Create the expressions to reverse the mapping.
+ // Project($0, $1, $0, $2).
+ final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
+ for (int i = 0; i < fields.size(); i++) {
+ final int origin = origins.get(i);
+ RelDataTypeField field = fields.get(i);
+ undoProjects.add(
+ Pair.of(
+ (RexNode) new RexInputRef(
+ squished.get(origin), field.getType()),
+ field.getName()));
+ }
+
+ rel =
+ new LogicalProject(
+ cluster,
+ rel,
+ Pair.left(undoProjects),
+ Pair.right(undoProjects),
+ LogicalProject.Flags.BOXED);
+
+ bb.setRoot(
+ rel,
+ false);
+
+ return;
+ }
+
+ // Usual case: all of the expressions in the SELECT clause are
+ // different.
+ final ImmutableBitSet groupSet =
+ ImmutableBitSet.range(rel.getRowType().getFieldCount());
+ rel =
+ createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
+ ImmutableList.<AggregateCall>of());
+
+ bb.setRoot(
+ rel,
+ false);
+ }
+
+ private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
+ for (int i = 0; i < count; i++) {
+ RexNode expr = exprs.get(i);
+ if (expr.toString().equals(seek.toString())) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Converts a query's ORDER BY clause, if any.
+ *
+ * @param select Query
+ * @param bb Blackboard
+ * @param collation Collation list
+ * @param orderExprList Method populates this list with orderBy expressions
+ * not present in selectList
+ * @param offset Expression for number of rows to discard before
+ * returning first row
+ * @param fetch Expression for number of rows to fetch
+ */
+ protected void convertOrder(
+ SqlSelect select,
+ Blackboard bb,
+ RelCollation collation,
+ List<SqlNode> orderExprList,
+ SqlNode offset,
+ SqlNode fetch) {
+ if (select.getOrderList() == null) {
+ assert collation.getFieldCollations().isEmpty();
+ if (offset == null && fetch == null) {
+ return;
+ }
+ }
+
+ // Create a sorter using the previously constructed collations.
+ bb.setRoot(
+ new Sort(
+ cluster,
+ cluster.traitSetOf(Convention.NONE, collation),
+ bb.root,
+ collation,
+ offset == null ? null : convertExpression(offset),
+ fetch == null ? null : convertExpression(fetch)),
+ false);
+
+ // If extra expressions were added to the project list for sorting,
+ // add another project to remove them.
+ if (orderExprList.size() > 0) {
+ List<RexNode> exprs = new ArrayList<RexNode>();
+ final RelDataType rowType = bb.root.getRowType();
+ final int fieldCount =
+ rowType.getFieldCount() - orderExprList.size();
+ for (int i = 0; i < fieldCount; i++) {
+ exprs.add(rexBuilder.makeInputRef(bb.root, i));
+ }
+ bb.setRoot(
+ new LogicalProject(
+ cluster,
+ cluster.traitSetOf(RelCollationImpl.PRESERVE),
+ bb.root,
+ exprs,
+ cluster.getTypeFactory().createStructType(
+ rowType.getFieldList().subList(0, fieldCount)),
+ Project.Flags.BOXED),
+ false);
+ }
+ }
+
+ /**
+ * Returns whether a given node contains a {@link SqlInOperator}.
+ *
+ * @param node a RexNode tree
+ */
+ private static boolean containsInOperator(
+ SqlNode node) {
+ try {
+ SqlVisitor<Void> visitor =
+ new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visit(call);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
+ }
+ }
+
+ /**
+ * Push down all the NOT logical operators into any IN/NOT IN operators.
+ *
+ * @param sqlNode the root node from which to look for NOT operators
+ * @return the transformed SqlNode representation with NOT pushed down.
+ */
+ private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+ SqlCall sqlCall = (SqlCall) sqlNode;
+ if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
+ || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+ for (int i = 0; i < sqlOperands.length; i++) {
+ sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ }
+ return sqlNode;
+ } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode childNode = sqlCall.operand(0);
+ assert childNode instanceof SqlCall;
+ SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+ if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+ SqlNode[] andOperands = childSqlCall.getOperands();
+ SqlNode[] orOperands = new SqlNode[andOperands.length];
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ andOperands[i]);
+ }
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = pushDownNotForIn(orOperands[i]);
+ }
+ return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
+ orOperands[0], orOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+ SqlNode[] orOperands = childSqlCall.getOperands();
+ SqlNode[] andOperands = new SqlNode[orOperands.length];
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] =
+ SqlStdOperatorTable.NOT.createCall(
+ SqlParserPos.ZERO,
+ orOperands[i]);
+ }
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = pushDownNotForIn(andOperands[i]);
+ }
+ return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
+ andOperands[0], andOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode[] notOperands = childSqlCall.getOperands();
+ assert notOperands.length == 1;
+ return pushDownNotForIn(notOperands[0]);
+ } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+ SqlNode[] inOperands = childSqlCall.getOperands();
+ SqlInOperator inOp =
+ (SqlInOperator) childSqlCall.getOperator();
+ if (inOp.isNotIn()) {
+ return SqlStdOperatorTable.IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ } else {
+ return SqlStdOperatorTable.NOT_IN.createCall(
+ SqlParserPos.ZERO,
+ inOperands[0],
+ inOperands[1]);
+ }
+ } else {
+ // childSqlCall is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // sqlNode is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // tree rooted at sqlNode does not contain inOperator
+ return sqlNode;
+ }
+ }
+
+ /**
+ * Converts a WHERE clause.
+ *
+ * @param bb Blackboard
+ * @param where WHERE clause, may be null
+ */
+ private void convertWhere(
+ final Blackboard bb,
+ final SqlNode where) {
+ if (where == null) {
+ return;
+ }
+ SqlNode newWhere = pushDownNotForIn(where);
+ replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ final RexNode convertedWhere = bb.convertExpression(newWhere);
+
+ // only allocate filter if the condition is not TRUE
+ if (!convertedWhere.isAlwaysTrue()) {
+ bb.setRoot(
+ RelOptUtil.createFilter(bb.root, convertedWhere),
+ false);
+ }
+ }
+
+ private void replaceSubqueries(
+ final Blackboard bb,
+ final SqlNode expr,
+ RelOptUtil.Logic logic) {
+ findSubqueries(bb, expr, logic, false);
+ for (SubQuery node : bb.subqueryList) {
+ substituteSubquery(bb, node);
+ }
+ }
+
+ private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
+ final RexNode expr = subQuery.expr;
+ if (expr != null) {
+ // Already done.
+ return;
+ }
+
+ final SqlBasicCall call;
+ final RelNode rel;
+ final SqlNode query;
+ final Pair<RelNode, Boolean> converted;
+ switch (subQuery.node.getKind()) {
+ case CURSOR:
+ convertCursor(bb, subQuery);
+ return;
+
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
+ subQuery.expr = bb.register(rel, JoinRelType.INNER);
+ return;
+
+ case IN:
+ call = (SqlBasicCall) subQuery.node;
+ final SqlNode[] operands = call.getOperands();
+
+ SqlNode leftKeyNode = operands[0];
+ query = operands[1];
+
+ final List<RexNode> leftKeys;
+ switch (leftKeyNode.getKind()) {
+ case ROW:
+ leftKeys = Lists.newArrayList();
+ for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
+ leftKeys.add(bb.convertExpression(sqlExpr));
+ }
+ break;
+ default:
+ leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
+ }
+
+ final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
+ if (query instanceof SqlNodeList) {
+ SqlNodeList valueList = (SqlNodeList) query;
+ if (!containsNullLiteral(valueList)
+ && valueList.size() < getInSubqueryThreshold()) {
+ // We're under the threshold, so convert to OR.
+ subQuery.expr =
+ convertInToOr(
+ bb,
+ leftKeys,
+ valueList,
+ isNotIn);
+ return;
+ }
+
+ // Otherwise, let convertExists translate
+ // values list into an inline table for the
+ // reference to Q below.
+ }
+
+ // Project out the search columns from the left side
+
+ // Q1:
+ // "select from emp where emp.deptno in (select col1 from T)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp inner join (select distinct col1 from T)) q
+ // on emp.deptno = q.col1
+ //
+ // Q2:
+ // "select from emp where emp.deptno not in (Q)"
+ //
+ // is converted to
+ //
+ // "select from
+ // emp left outer join (select distinct col1, TRUE from T) q
+ // on emp.deptno = q.col1
+ // where emp.deptno <> null
+ // and q.indicator <> TRUE"
+ //
+ final boolean outerJoin = bb.subqueryNeedsOuterJoin
+ || isNotIn
+ || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
+ converted =
+ convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic,
+ outerJoin);
+ if (converted.right) {
+ // Generate
+ // emp CROSS JOIN (SELECT COUNT(*) AS c,
+ // COUNT(deptno) AS ck FROM dept)
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RelNode seek = converted.left.getInput(0); // fragile
+ final int keyCount = leftKeys.size();
+ final List<Integer> args = ImmutableIntList.range(0, keyCount);
+ LogicalAggregate aggregate =
+ new LogicalAggregate(cluster, seek, false, ImmutableBitSet.of(),
+ null,
+ ImmutableList.of(
+ new AggregateCall(SqlStdOperatorTable.COUNT, false,
+ ImmutableList.<Integer>of(), longType, null),
+ new AggregateCall(SqlStdOperatorTable.COUNT, false,
+ args, longType, null)));
+ LogicalJoin join =
+ new LogicalJoin(cluster, bb.root, aggregate,
+ rexBuilder.makeLiteral(true), JoinRelType.INNER,
+ ImmutableSet.<String>of());
+ bb.setRoot(join, false);
+ }
+ RexNode rex =
+ bb.register(converted.left,
+ outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
+
+ subQuery.expr = translateIn(subQuery, bb.root, rex);
+ if (isNotIn) {
+ subQuery.expr =
+ rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ }
+ return;
+
+ case EXISTS:
+ // "select from emp where exists (select a from T)"
+ //
+ // is converted to the following if the subquery is correlated:
+ //
+ // "select from emp left outer join (select AGG_TRUE() as indicator
+ // from T group by corr_var) q where q.indicator is true"
+ //
+ // If there is no correlation, the expression is replaced with a
+ // boolean indicating whether the subquery returned 0 or >= 1 row.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.getOperands()[0];
+ converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS,
+ subQuery.logic, true);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
+ return;
+ }
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ case SCALAR_QUERY:
+ // Convert the subquery. If it's non-correlated, convert it
+ // to a constant expression.
+ call = (SqlBasicCall) subQuery.node;
+ query = call.getOperands()[0];
+ converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR,
+ subQuery.logic, true);
+ assert !converted.right;
+ if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
+ return;
+ }
+ rel = convertToSingleValueSubq(query, converted.left);
+ subQuery.expr = bb.register(rel, JoinRelType.LEFT);
+ return;
+
+ case SELECT:
+ // This is used when converting multiset queries:
+ //
+ // select * from unnest(select multiset[deptno] from emps);
+ //
+ converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR,
+ subQuery.logic, true);
+ assert !converted.right;
+ subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
+ return;
+
+ default:
+ throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
+ }
+ }
+
+ private RexNode translateIn(SubQuery subQuery, RelNode root,
+ final RexNode rex) {
+ switch (subQuery.logic) {
+ case TRUE:
+ return rexBuilder.makeLiteral(true);
+
+ case UNKNOWN_AS_FALSE:
+ assert rex instanceof RexRangeRef;
+ final int fieldCount = rex.getType().getFieldCount();
+ RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
+ rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
+
+ // Then append the IS NOT NULL(leftKeysForIn).
+ //
+ // RexRangeRef contains the following fields:
+ // leftKeysForIn,
+ // rightKeysForIn (the original subquery select list),
+ // nullIndicator
+ //
+ // The first two lists contain the same number of fields.
+ final int k = (fieldCount - 1) / 2;
+ for (int i = 0; i < k; i++) {
+ rexNode =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.AND,
+ rexNode,
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.IS_NOT_NULL,
+ rexBuilder.makeFieldAccess(rex, i)));
+ }
+ return rexNode;
+
+ case TRUE_FALSE_UNKNOWN:
+ case UNKNOWN_AS_TRUE:
+ // select e.deptno,
+ // case
+ // when ct.c = 0 then false
+ // when dt.i is not null then true
+ // when e.deptno is null then null
+ // when ct.ck < ct.c then null
+ // else false
+ // end
+ // from e
+ // cross join (select count(*) as c, count(deptno) as ck from v) as ct
+ // left join (select distinct deptno, true as i from v) as dt
+ // on e.deptno = dt.deptno
+ final Join join = (Join) root;
+ final Project left = (Project) join.getLeft();
+ final RelNode leftLeft = ((Join) left.getInput(0)).getLeft();
+ final int leftLeftCount = leftLeft.getRowType().getFieldCount();
+ final RelDataType nullableBooleanType =
+ typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
+ final RelDataType longType =
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
+ final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
+ final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
+ final RexNode iRef =
+ rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+
+ final RexLiteral zero =
+ rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
+ final RexNode unknownLiteral =
+ rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
+
+ final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
+ falseLiteral,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
+ trueLiteral);
+ final JoinInfo joinInfo = join.analyzeCondition();
+ for (int leftKey : joinInfo.leftKeys) {
+ final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
+ unknownLiteral);
+ }
+ args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
+ unknownLiteral,
+ falseLiteral);
+
+ return rexBuilder.makeCall(
+ nullableBooleanType,
+ SqlStdOperatorTable.CASE,
+ args.build());
+
+ default:
+ throw new AssertionError(subQuery.logic);
+ }
+ }
+
+ private static boolean containsNullLiteral(SqlNodeList valueList) {
+ for (SqlNode node : valueList.getList()) {
+ if (node instanceof SqlLiteral) {
+ SqlLiteral lit = (SqlLiteral) node;
+ if (lit.getValue() == null) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Determines if a subquery is non-correlated and if so, converts it to a
+ * constant.
+ *
+ * @param subQuery the call that references the subquery
+ * @param bb blackboard used to convert the subquery
+ * @param converted RelNode tree corresponding to the subquery
+ * @param isExists true if the subquery is part of an EXISTS expression
+ * @return if the subquery can be converted to a constant
+ */
+ private boolean convertNonCorrelatedSubQuery(
+ SubQuery subQuery,
+ Blackboard bb,
+ RelNode converted,
+ boolean isExists) {
+ SqlCall call = (SqlBasicCall) subQuery.node;
+ if (subqueryConverter.canConvertSubquery()
+ && isSubQueryNonCorrelated(converted, bb)) {
+ // First check if the subquery has already been converted
+ // because it's a nested subquery. If so, don't re-evaluate
+ // it again.
+ RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
+ if (constExpr == null) {
+ constExpr =
+ subqueryConverter.convertSubquery(
+ call,
+ this,
+ isExists,
+ isExplain);
+ }
+ if (constExpr != null) {
+ subQuery.expr = constExpr;
+ mapConvertedNonCorrSubqs.put(call, constExpr);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Converts the RelNode tree for a select statement to a select that
+ * produces a single value.
+ *
+ * @param query the query
+ * @param plan the original RelNode tree corresponding to the statement
+ * @return the converted RelNode tree
+ */
+ public RelNode convertToSingleValueSubq(
+ SqlNode query,
+ RelNode plan) {
+ // Check whether query is guaranteed to produce a single value.
+ if (query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) query;
+ SqlNodeList selectList = select.getSelectList();
+ SqlNodeList groupList = select.getGroup();
+
+ if ((selectList.size() == 1)
+ && ((groupList == null) || (groupList.size() == 0))) {
+ SqlNode selectExpr = selectList.get(0);
+ if (selectExpr instanceof SqlCall) {
+ SqlCall selectExprCall = (SqlCall) selectExpr;
+ if (selectExprCall.getOperator()
+ instanceof SqlAggFunction) {
+ return plan;
+ }
+ }
+ }
+ }
+
+ // If not, project SingleValueAgg
+ return RelOptUtil.createSingleValueAggRel(
+ cluster,
+ plan);
+ }
+
+ /**
+ * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
+ *
+ * @param leftKeys LHS
+ * @param valuesList RHS
+ * @param isNotIn is this a NOT IN operator
+ * @return converted expression
+ */
+ private RexNode convertInToOr(
+ final Blackboard bb,
+ final List<RexNode> leftKeys,
+ SqlNodeList valuesList,
+ boolean isNotIn) {
+ List<RexNode> comparisons = new ArrayList<RexNode>();
+ for (SqlNode rightVals : valuesList) {
+ RexNode rexComparison;
+ if (leftKeys.size() == 1) {
+ rexComparison =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.EQUALS,
+ leftKeys.get(0),
+ bb.convertExpression(rightVals));
+ } else {
+ assert rightVals instanceof SqlCall;
+ final SqlBasicCall call = (SqlBasicCall) rightVals;
+ assert (call.getOperator() instanceof SqlRowOperator)
+ && call.getOperands().length == leftKeys.size();
+ rexComparison =
+ RexUtil.composeConjunction(
+ rexBuilder,
+ Iterables.transform(
+ Pair.zip(leftKeys, call.getOperandList()),
+ new Function<Pair<RexNode, SqlNode>, RexNode>() {
+ public RexNode apply(Pair<RexNode, SqlNode> pair) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ pair.left, bb.convertExpression(pair.right));
+ }
+ }),
+ false);
+ }
+ comparisons.add(rexComparison);
+ }
+
+ RexNode result =
+ RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+ assert result != null;
+
+ if (isNotIn) {
+ result =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.NOT,
+ result);
+ }
+
+ return result;
+ }
+
+ /**
+ * Gets the list size threshold under which {@link #convertInToOr} is used.
+ * Lists of this size or greater will instead be converted to use a join
+ * against an inline table
+ * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a
+ * predicate. A threshold of 0 forces usage of an inline table in all cases; a
+ * threshold of Integer.MAX_VALUE forces usage of OR in all cases
+ *
+ * @return threshold, default 20
+ */
+ protected int getInSubqueryThreshold() {
+ // OVERRIDE POINT
+ return Integer.MAX_VALUE; // was 20
+ }
+
+ /**
+ * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
+ * produces an indicator variable, and the result is a relational expression
+ * which outer joins that indicator to the original query. After performing
+ * the outer join, the condition will be TRUE if the EXISTS condition holds,
+ * NULL otherwise.
+ *
+ * @param seek A query, for example 'select * from emp' or
+ * 'values (1,2,3)' or '('Foo', 34)'.
+ * @param subqueryType Whether sub-query is IN, EXISTS or scalar
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept an
+ * approximation (say representing UNKNOWN as FALSE)
+ * @param needsOuterJoin Whether an outer join is needed
+ * @return join expression
+ * @pre extraExpr == null || extraName != null
+ */
+ private Pair<RelNode, Boolean> convertExists(
+ SqlNode seek,
+ RelOptUtil.SubqueryType subqueryType,
+ RelOptUtil.Logic logic,
+ boolean needsOuterJoin) {
+ final SqlValidatorScope seekScope =
+ (seek instanceof SqlSelect)
+ ? validator.getSelectScope((SqlSelect) seek)
+ : null;
+ final Blackboard seekBb = createBlackboard(seekScope, null);
+ RelNode seekRel = convertQueryOrInList(seekBb, seek);
+
+ return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic,
+ needsOuterJoin);
+ }
+
+ private RelNode convertQueryOrInList(
+ Blackboard bb,
+ SqlNode seek) {
+ // NOTE: Once we start accepting single-row queries as row constructors,
+ // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
+ // Z)). The SQL standard resolves the ambiguity by saying that a lone
+ // select should be interpreted as a table expression, not a row
+ // expression. The semantic difference is that a table expression can
+ // return multiple rows.
+ if (seek instanceof SqlNodeList) {
+ return convertRowValues(
+ bb,
+ seek,
+ ((SqlNodeList) seek).getList(),
+ false,
+ null);
+ } else {
+ return convertQueryRecursive(seek, false, null);
+ }
+ }
+
+ private RelNode convertRowValues(
+ Blackboard bb,
+ SqlNode rowList,
+ Collection<SqlNode> rows,
+ boolean allowLiteralsOnly,
+ RelDataType targetRowType) {
+ // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
+ // literals into a single LogicalValues; this gives the optimizer a smaller
+ // input tree. For everything else (computed expressions, row
+ // subqueries), we union each row in as a projection on top of a
+ // LogicalOneRow.
+
+ final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
+ ImmutableList.builder();
+ final RelDataType rowType;
+ if (targetRowType != null) {
+ rowType = targetRowType;
+ } else {
+ rowType =
+ SqlTypeUtil.promoteToRowType(
+ typeFactory,
+ validator.getValidatedNodeType(rowList),
+ null);
+ }
+
+ List<RelNode> unionInputs = new ArrayList<RelNode>();
+ for (SqlNode node : rows) {
+ SqlBasicCall call;
+ if (isRowConstructor(node)) {
+ call = (SqlBasicCall) node;
+ ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
+ for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ operand.e,
+ bb,
+ rowType,
+ operand.i);
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
+ if ((rexLiteral == null) || !shouldCreateValuesRel) {
+ // fallback to convertRowConstructor
+ tuple = null;
+ break;
+ }
+ tuple.add(rexLiteral);
+ }
+ if (tuple != null) {
+ tupleList.add(tuple.build());
+ continue;
+ }
+ } else {
+ RexLiteral rexLiteral =
+ convertLiteralInValuesList(
+ node,
+ bb,
+ rowType,
+ 0);
+ if ((rexLiteral != null) && shouldCreateValuesRel) {
+ tupleList.add(ImmutableList.of(rexLiteral));
+ continue;
+ } else {
+ if ((rexLiteral == null) && allowLiteralsOnly) {
+ return null;
+ }
+ }
+
+ // convert "1" to "row(1)"
+ call =
+ (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
+ SqlParserPos.ZERO,
+ node);
+ }
+ unionInputs.add(convertRowConstructor(bb, call));
+ }
+ LogicalValues values =
+ new LogicalValues(
+ cluster,
+ rowType,
+ tupleList.build());
+ RelNode resultRel;
+ if (unionInputs.isEmpty()) {
+ resultRel = values;
+ } else {
+ if (!values.getTuples().isEmpty()) {
+ unionInputs.add(values);
+ }
+ LogicalUnion union =
+ new LogicalUnion(
+ cluster,
+ unionInputs,
+ true);
+ resultRel = union;
+ }
+ leaves.add(resultRel);
+ return resultRel;
+ }
+
+ private RexLiteral convertLiteralInValuesList(
+ SqlNode sqlNode,
+ Blackboard bb,
+ RelDataType rowType,
+ int iField) {
+ if (!(sqlNode instanceof SqlLiteral)) {
+ return null;
+ }
+ RelDataTypeField field = rowType.getFieldList().get(iField);
+ RelDataType type = field.getType();
+ if (type.isStruct()) {
+ // null literals for weird stuff like UDT's need
+ // special handling during type flattening, so
+ // don't use LogicalValues for those
+ return null;
+ }
+
+ RexNode literalExpr =
+ exprConverter.convertLiteral(
+ bb,
+ (SqlLiteral) sqlNode);
+
+ if (!(literalExpr instanceof RexLiteral)) {
+ assert literalExpr.isA(SqlKind.CAST);
+ RexNode child = ((RexCall) literalExpr).getOperands().get(0);
+ assert RexLiteral.isNullLiteral(child);
+
+ // NOTE jvs 22-Nov-2006: we preserve type info
+ // in LogicalValues digest, so it's OK to lose it here
+ return (RexLiteral) child;
+ }
+
+ RexLiteral literal = (RexLiteral) literalExpr;
+
+ Comparable value = literal.getValue();
+
+ if (SqlTypeUtil.isExactNumeric(type)) {
+ BigDecimal roundedValue =
+ NumberUtil.rescaleBigDecimal(
+ (BigDecimal) value,
+ type.getScale());
+ return rexBuilder.makeExactLiteral(
+ roundedValue,
+ type);
+ }
+
+ if ((value instanceof NlsString)
+ && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+ // pad fixed character type
+ NlsString unpadded = (NlsString) value;
+ return rexBuilder.makeCharLiteral(
+ new NlsString(
+ Util.rpad(unpadded.getValue(), type.getPrecision()),
+ unpadded.getCharsetName(),
+ unpadded.getCollation()));
+ }
+ return literal;
+ }
+
+ private boolean isRowConstructor(SqlNode node) {
+ if (!(node.getKind() == SqlKind.ROW)) {
+ return false;
+ }
+ SqlCall call = (SqlCall) node;
+ return call.getOperator().getName().equalsIgnoreCase("row");
+ }
+
+ /**
+ * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
+ * inside SQL parse tree. Does not traverse inside queries.
+ *
+ * @param bb blackboard
+ * @param node the SQL parse tree
+ * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
+ * FALSE, UNKNOWN) will be required, or whether we can accept
+ * an approximation (say representing UNKNOWN as FALSE)
+ * @param registerOnlyScalarSubqueries if set to true and the parse tree
+ * corresponds to a variation of a select
+ * node, only register it if it's a scalar
+ * subquery
+ */
+ private void findSubqueries(
+ Blackboard bb,
+ SqlNode node,
+ RelOptUtil.Logic logic,
+ boolean registerOnlyScalarSubqueries) {
+ final SqlKind kind = node.getKind();
+ switch (kind) {
+ case EXISTS:
+ case SELECT:
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ case CURSOR:
+ case SCALAR_QUERY:
+ if (!registerOnlyScalarSubqueries
+ || (kind == SqlKind.SCALAR_QUERY)) {
+ bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
+ }
+ return;
+ case IN:
+ if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
+ logic = logic.negate();
+ }
+ break;
+ case NOT:
+ logic = logic.negate();
+ break;
+ }
+ if (node instanceof SqlCall) {
+ if (kind == SqlKind.OR
+ || kind == SqlKind.NOT) {
+ // It's always correct to outer join subquery with
+ // containing query; however, when predicates involve Or
+ // or NOT, outer join might be necessary.
+ bb.subqueryNeedsOuterJoin = true;
+ }
+ for (SqlNode operand : ((SqlCall) node).getOperandList()) {
+ if (operand != null) {
+ // In the case of an IN expression, locate scalar
+ // subqueries so we can convert them to constants
+ findSubqueries(
+ bb,
+ operand,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
+ }
+ } else if (node instanceof SqlNodeList) {
+ for (SqlNode child : (SqlNodeList) node) {
+ findSubqueries(
+ bb,
+ child,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubqueries);
+ }
+ }
+
+ // Now that we've located any scalar subqueries inside the IN
+ // expression, register the IN expression itself. We need to
+ // register the scalar subqueries first so they can be converted
+ // before the IN expression is converted.
+ if (kind == SqlKind.IN) {
+ if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
+ && !validator.getValidatedNodeType(node).isNullable()) {
+ logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
+ }
+ // TODO: This conversion is only valid in the WHERE clause
+ if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE
+ && !bb.subqueryNeedsOuterJoin) {
+ logic = RelOptUtil.Logic.TRUE;
+ }
+ bb.registerSubquery(node, logic);
+ }
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format.
+ *
+ * @param node Expression to translate
+ * @return Converted expression
+ */
+ public RexNode convertExpression(
+ SqlNode node) {
+ Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
+ Blackboard bb =
+ createBlackboard(
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
+ null);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts an expression from {@link SqlNode} to {@link RexNode} format,
+ * mapping identifier references to predefined expressions.
+ *
+ * @param node Expression to translate
+ * @param nameToNodeMap map from String to {@link RexNode}; when an
+ * {@link SqlIdentifier} is encountered, it is used as a
+ * key and translated to the corresponding value from
+ * this map
+ * @return Converted expression
+ */
+ public RexNode convertExpression(
+ SqlNode node,
+ Map<String, RexNode> nameToNodeMap) {
+ final Map<String, RelDataType> nameToTypeMap =
+ new HashMap<String, RelDataType>();
+ for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
+ nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
+ }
+ Blackboard bb =
+ createBlackboard(
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
+ nameToNodeMap);
+ return bb.convertExpression(node);
+ }
+
+ /**
+ * Converts a non-standard expression.
+ *
+ * <p>This method is an extension-point that derived classes can override. If
+ * this method returns a null result, the normal expression translation
+ * process will proceed. The default implementation always returns null.
+ *
+ * @param node Expression
+ * @param bb Blackboard
+ * @return null to proceed with the usual expression translation process
+ */
+ protected RexNode convertExtendedExpression(
+ SqlNode node,
+ Blackboard bb) {
+ return null;
+ }
+
+ private RexNode convertOver(Blackboard bb, SqlNode node) {
+ SqlCall call = (SqlCall) node;
+ SqlCall aggCall = call.operand(0);
+ SqlNode windowOrRef = call.operand(1);
+ final SqlWindow window =
+ validator.resolveWindow(windowOrRef, bb.scope, true);
+ final SqlNodeList partitionList = window.getPartitionList();
+ final ImmutableList.Builder<RexNode> partitionKeys =
+ ImmutableList.builder();
+ for (SqlNode partition : partitionList) {
+ partitionKeys.add(bb.convertExpression(partition));
+ }
+ RexNode lowerBound = bb.convertExpression(window.getLowerBound());
+ RexNode upperBound = bb.convertExpression(window.getUpperBound());
+ SqlNodeList orderList = window.getOrderList();
+ if ((orderList.size() == 0) && !window.isRows()) {
+ // A logical range requires an ORDER BY clause. Use the implicit
+ // ordering of this relation. There must be one, otherwise it would
+ // have failed validation.
+ orderList = bb.scope.getOrderList();
+ if (orderList == null) {
+ throw new AssertionError(
+ "Relation should have sort key for implicit ORDER BY");
+ }
+ }
+ final ImmutableList.Builder<RexFieldCollation> orderKeys =
+ ImmutableList.builder();
+ final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
+ for (SqlNode order : orderList) {
+ flags.clear();
+ RexNode e = bb.convertSortExpression(order, flags);
+ orderKeys.add(new RexFieldCollation(e, flags));
+ }
+ try {
+ Util.permAssert(bb.window == null, "already in window agg mode");
+ bb.window = window;
+ RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
+ rexAgg =
+ rexBuilder.ensureType(
+ validator.getValidatedNodeType(call), rexAgg, false);
+
+ // Walk over the tree and apply 'over' to all agg functions. This is
+ // necessary because the returned expression is not necessarily a call
+ // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
+ final RexShuttle visitor =
+ new HistogramShuttle(
+ partitionKeys.build(), orderKeys.build(),
+ RexWindowBound.create(window.getLowerBound(), lowerBound),
+ RexWindowBound.create(window.getUpperBound(), upperBound),
+ window);
+ return rexAgg.accept(visitor);
+ } finally {
+ bb.window = null;
+ }
+ }
+
+ /**
+ * Converts a FROM clause into a relational expression.
+ *
+ * @param bb Scope within which to resolve identifiers
+ * @param from FROM clause of a query. Examples include:
+ *
+ * <ul>
+ * <li>a single table ("SALES.EMP"),
+ * <li>an aliased table ("EMP AS E"),
+ * <li>a list of tables ("EMP, DEPT"),
+ * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
+ * DEPT.DEPTNO"),
+ * <li>a VALUES clause ("VALUES ('Fred', 20)"),
+ * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
+ * <li>or any combination of the above.
+ * </ul>
+ */
+ protected void convertFrom(
+ Blackboard bb,
+ SqlNode from) {
+ SqlCall call;
+ final SqlNode[] operands;
+ switch (from.getKind()) {
+ case AS:
+ operands = ((SqlBasicCall) from).getOperands();
+ convertFrom(bb, operands[0]);
+ return;
+
+ case WITH_ITEM:
+ convertFrom(bb, ((SqlWithItem) from).query);
+ return;
+
+ case WITH:
+ convertFrom(bb, ((SqlWith) from).body);
+ return;
+
+ case TABLESAMPLE:
+ operands = ((SqlBasicCall) from).getOperands();
+ SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
+ if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
+ String sampleName =
+ ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
+ .getName();
+ datasetStack.push(sampleName);
+ convertFrom(bb, operands[0]);
+ datasetStack.pop();
+ } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
+ SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
+ (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
+ convertFrom(bb, operands[0]);
+ RelOptSamplingParameters params =
+ new RelOptSamplingParameters(
+ tableSampleSpec.isBernoulli(),
+ tableSampleSpec.getSamplePercentage(),
+ tableSampleSpec.isRepeatable(),
+ tableSampleSpec.getRepeatableSeed());
+ bb.setRoot(new Sample(cluster, bb.root, params), false);
+ } else {
+ throw Util.newInternal(
+ "unknown TABLESAMPLE type: " + sampleSpec);
+ }
+ return;
+
+ case IDENTIFIER:
+ final SqlValidatorNamespace fromNamespace =
+ validator.getNamespace(from).resolve();
+ if (fromNamespace.getNode() != null) {
+ convertFrom(bb, fromNamespace.getNode());
+ return;
+ }
+ final String datasetName =
+ datasetStack.isEmpty() ? null : datasetStack.peek();
+ boolean[] usedDataset = {false};
+ RelOptTable table =
+ SqlValidatorUtil.getRelOptTable(
+ fromNamespace,
+ catalogReader,
+ datasetName,
+ usedDataset);
+ final RelNode tableRel;
+ if (shouldConvertTableAccess) {
+ tableRel = toRel(table);
+ } else {
+ tableRel = new LogicalTableScan(cluster, table);
+ }
+ bb.setRoot(tableRel, true);
+ if (usedDataset[0]) {
+ bb.setDataset(datasetName);
+ }
+ return;
+
+ case JOIN:
+ final SqlJoin join = (SqlJoin) from;
+ final Blackboard fromBlackboard =
+ createBlackboard(validator.getJoinScope(from), null);
+ SqlNode left = join.getLeft();
+ SqlNode right = join.getRight();
+ final boolean isNatural = join.isNatural();
+ final JoinType joinType = join.getJoinType();
+ final Blackboard leftBlackboard =
+ createBlackboard(
+ Util.first(validator.getJoinScope(left),
+ ((DelegatingScope) bb.scope).getParent()), null);
+ final Blackboard rightBlackboard =
+ createBlackboard(
+ Util.first(validator.getJoinScope(right),
+ ((DelegatingScope) bb.scope).getParent()), null);
+ convertFrom(leftBlackboard, left);
+ RelNode leftRel = leftBlackboard.root;
+ convertFrom(rightBlackboard, right);
+ RelNode rightRel = rightBlackboard.root;
+ JoinRelType convertedJoinType = convertJoinType(joinType);
+ RexNode conditionExp;
+ if (isNatural) {
+ final List<String> columnList =
+ SqlValidatorUtil.deriveNaturalJoinColumnList(
+ validator.getNamespace(left).getRowType(),
+ validator.getNamespace(right).getRowType());
+ conditionExp = convertUsing(leftRel, rightRel, columnList);
+ } else {
+ conditionExp =
+ convertJoinCondition(
+ fromBlackboard,
+ join.getCondition(),
+ join.getConditionType(),
+ leftRel,
+ rightRel);
+ }
+
+ final RelNode joinRel =
+ createJoin(
+ fromBlackboard,
+ leftRel,
+ rightRel,
+ conditionExp,
+ convertedJoinType);
+ bb.setRoot(joinRel, false);
+ return;
+
+ case SELECT:
+ case INTERSECT:
+ case EXCEPT:
+ case UNION:
+ final RelNode rel = convertQueryRecursive(from, false, null);
+ bb.setRoot(rel, true);
+ return;
+
+ case VALUES:
+ convertValuesImpl(bb, (SqlCall) from, null);
+ return;
+
+ case UNNEST:
+ final SqlNode node = ((SqlCall) from).operand(0);
+ replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ final RelNode childRel =
+ RelOptUtil.createProject(
+ (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
+ Collections.singletonList(bb.convertExpression(node)),
+ Collections.singletonList(validator.deriveAlias(node, 0)),
+ true);
+
+ Uncollect uncollect =
+ new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
+ childRel);
+ bb.setRoot(uncollect, true);
+ return;
+
+ case COLLECTION_TABLE:
+ call = (SqlCall) from;
+
+ // Dig out real call; TABLE() wrapper is just syntactic.
+ assert call.getOperandList().size() == 1;
+ call = call.operand(0);
+ convertCollectionTable(bb, call);
+ return;
+
+ default:
+ throw Util.newInternal("not a join operator " + from);
+ }
+ }
+
+ protected void convertCollectionTable(
+ Blackboard bb,
+ SqlCall call) {
+ final SqlOperator operator = call.getOperator();
+ if (operator == SqlStdOperatorTable.TABLESAMPLE) {
+ final String sampleName =
+ SqlLiteral.stringValue(call.operand(0));
+ datasetStack.push(sampleName);
+ SqlCall cursorCall = call.operand(1);
+ SqlNode query = cursorCall.operand(0);
+ RelNode converted = convertQuery(query, false, false);
+ bb.setRoot(converted, false);
+ datasetStack.pop();
+ return;
+ }
+ replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+
+ // Expand table macro if possible. It's more efficient than
+ // LogicalTableFunctionScan.
+ if (operator instanceof SqlUserDefinedTableMacro) {
+ final SqlUserDefinedTableMacro udf =
+ (SqlUserDefinedTableMacro) operator;
+ final TranslatableTable table = udf.getTable(typeFactory,
+ call.getOperandList());
+ final RelDataType rowType = table.getRowType(typeFactory);
+ RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
+ RelNode converted = toRel(relOptTable);
+ bb.setRoot(converted, true);
+ return;
+ }
+
+ Type elementType;
+ if (operator instanceof SqlUserDefinedTableFunction) {
+ SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
+ elementType = udtf.getElementType(typeFactory, call.getOperandList());
+ } else {
+ elementType = null;
+ }
+
+ RexNode rexCall = bb.convertExpression(call);
+ final List<RelNode> inputs = bb.retrieveCursors();
+ Set<RelColumnMapping> columnMappings =
+ getColumnMappings(operator);
+ LogicalTableFunctionScan callRel =
+ new LogicalTableFunctionScan(
+ cluster,
+ inputs,
+ rexCall,
+ elementType,
+ validator.getValidatedNodeType(call),
+ columnMappings);
+ bb.setRoot(callRel, true);
+ afterTableFunction(bb, call, callRel);
+ }
+
+ protected void afterTableFunction(
+ SqlToRelConverter.Blackboard bb,
+ SqlCall call,
+ LogicalTableFunctionScan callRel) {
+ }
+
+ private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
+ SqlReturnTypeInference rti = op.getReturnTypeInference();
+ if (rti == null) {
+ return null;
+ }
+ if (rti instanceof TableFunctionReturnTypeInference) {
+ TableFunctionReturnTypeInference tfrti =
+ (TableFunctionReturnTypeInference) rti;
+ return tfrti.getColumnMappings();
+ } else {
+ return null;
+ }
+ }
+
+ protected RelNode createJoin(
+ Blackboard bb,
+ RelNode leftRel,
+ RelNode rightRel,
+ RexNode joinCond,
+ JoinRelType joinType) {
+ assert joinCond != null;
+
+ Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(rightRel);
+ if (correlatedVariables.size() > 0) {
+ final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
+ final List<String> correlNames = Lists.newArrayList();
+
+ // All correlations must refer the same namespace since correlation
+ // produces exactly one correlation source.
+ // The same source might be referenced by different variables since
+ // DeferredLookups are not de-duplicated at create time.
+ SqlValidatorNamespace prevNs = null;
+
+ for (String correlName : correlatedVariables) {
+ DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
+ RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
+ String originalRelName = lookup.getOriginalRelName();
+ String originalFieldName = fieldAccess.getField().getName();
+
+ int[] nsIndexes = {-1};
+ final SqlValidatorScope[] ancestorScopes = {null};
+ SqlValidatorNamespace foundNs =
+ lookup.bb.scope.resolve(
+ ImmutableList.of(originalRelName),
+ ancestorScopes,
+ nsIndexes);
+
+ assert foundNs != null;
+ assert nsIndexes.length == 1;
+
+ int childNamespaceIndex = nsIndexes[0];
+
+ SqlValidatorScope ancestorScope = ancestorScopes[0];
+ boolean correlInCurrentScope = ancestorScope == bb.scope;
+
+ if (!correlInCurrentScope) {
+ continue;
+ }
+
+ if (prevNs == null) {
+ prevNs = foundNs;
+ } else {
+ assert prevNs == foundNs : "All correlation variables should resolve"
+ + " to the same namespace."
+ + " Prev ns=" + prevNs
+ + ", new ns=" + foundNs;
+ }
+
+ int namespaceOffset = 0;
+ if (childNamespaceIndex > 0) {
+ // If not the first child, need to figure out the width
+ // of output types from all the preceding namespaces
+ assert ancestorScope instanceof ListScope;
+ List<SqlValidatorNamespace> children =
+ ((ListScope) ancestorScope).getChildren();
+
+ for (int i = 0; i < childNamespaceIndex; i++) {
+ SqlValidatorNamespace child = children.get(i);
+ namespaceOffset +=
+ child.getRowType().getFieldCount();
+ }
+ }
+
+ RelDataTypeField field =
+ catalogReader.field(foundNs.getRowType(), originalFieldName);
+ int pos = namespaceOffset + field.getIndex();
+
+ assert field.getType()
+ == lookup.getFieldAccess(correlName).getField().getType();
+
+ assert pos != -1;
+
+ if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
+ // bb.root is an aggregate and only projects group by
+ // keys.
+ Map<Integer, Integer> exprProjection =
+ bb.mapRootRelToFieldProjection.get(bb.root);
+
+ // subquery can reference group by keys projected from
+ // the root of the outer relation.
+ if (exprProjection.containsKey(pos)) {
+ pos = exprProjection.get(pos);
+ } else {
+ // correl not grouped
+ throw Util.newInternal(
+ "Identifier '" + originalRelName + "."
+ + originalFieldName + "' is not a group expr");
+ }
+ }
+
+ requiredColumns.set(pos);
+ correlNames.add(correlName);
+ }
+
+ if (!correlNames.isEmpty()) {
+ if (correlNames.size() > 1) {
+ // The same table was referenced more than once.
+ // So we deduplicate
+ RelShuttle dedup =
+ new DeduplicateCorrelateVariables(rexBuilder,
+ correlNames.get(0),
+ ImmutableSet.copyOf(Util.skip(correlNames)));
+ rightRel = rightRel.accept(dedup);
+ }
+ LogicalCorrelate corr = new LogicalCorrelate(
+ rightRel.getCluster(),
+ leftRel,
+ rightRel,
+ new CorrelationId(correlNames.get(0)),
+ requiredColumns.build(),
+ SemiJoinType.of(joinType));
+ if (!joinCond.isAlwaysTrue()) {
+ return RelOptUtil.createFilter(corr, joinCond);
+ }
+ return corr;
+ }
+ }
+
+ final List<RexNode> extraLeftExprs = new ArrayList<RexNode>();
+ final List<RexNode> extraRightExprs = new ArrayList<RexNode>();
+ final int leftCount = leftRel.getRowType().getFieldCount();
+ final int rightCount = rightRel.getRowType().getFieldCount();
+ if (!containsGet(joinCond)) {
+ joinCond = pushDownJoinConditions(
+ joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs);
+ }
+ if (!extraLeftExprs.isEmpty()) {
+ final List<RelDataTypeField> fields =
+ leftRel.getRowType().getFieldList();
+ leftRel = RelOptUtil.createProject(
+ leftRel,
+ new AbstractList<Pair<RexNode, String>>() {
+ @Override public int size() {
+ return leftCount + extraLeftExprs.size();
+ }
+
+ @Override public Pair<RexNode, String> get(int index) {
+ if (index < leftCount) {
+ RelDataTypeField field = fields.get(index);
+ return Pair.<RexNode, String>of(
+ new RexInputRef(index, field.getType()),
+ field.getName());
+ } else {
+ return Pair.<RexNode, String>of(
+ extraLeftExprs.get(index - leftCount), null);
+ }
+ }
+ },
+ true);
+ }
+ if (!extraRightExprs.isEmpty()) {
+ final List<RelDataTypeField> fields =
+ rightRel.getRowType().getFieldList();
+ final int newLeftCount = leftCount + extraLeftExprs.size();
+ rightRel = RelOptUtil.createProject(
+ rightRel,
+ new AbstractList<Pair<RexNode, String>>() {
+ @Override public int size() {
+ return rightCount + extraRightExprs.size();
+ }
+
+ @Override public Pair<RexNode, String> get(int index) {
+ if (index < rightCount) {
+ RelDataTypeField field = fields.get(index);
+ return Pair.<RexNode, String>of(
+ new RexInputRef(index, field.getType()),
+ field.getName());
+ } else {
+ return Pair.of(
+ RexUtil.shift(
+ extraRightExprs.get(index - rightCount),
+ -newLeftCount),
+ null);
+ }
+ }
+ },
+ true);
+ }
+ RelNode join = createJoin(
+ leftRel,
+ rightRel,
+ joinCond,
+ joinType,
+ ImmutableSet.<String>of());
+ if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) {
+ Mappings.TargetMapping mapping =
+ Mappings.createShiftMapping(
+ leftCount + extraLeftExprs.size()
+ + rightCount + extraRightExprs.size(),
+ 0, 0, leftCount,
+ leftCount, leftCount + extraLeftExprs.size(), rightCount);
+ return RelOptUtil.project(join, mapping);
+ }
+ return join;
+ }
+
+ private static boolean containsGet(RexNode node) {
+ try {
+ node.accept(
+ new RexVisitorImpl<Void>(true) {
+ @Override public Void visitCall(RexCall call) {
+ if (call.getOperator() == RexBuilder.GET_OPERATOR) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ });
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ /**
+ * Pushes down parts of a join condition. For example, given
+ * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above
+ * "emp" that computes the expression
+ * "emp.deptno + 1". The resulting join condition is a simple combination
+ * of AND, equals, and input fields.
+ */
+ private RexNode pushDownJoinConditions(
+ RexNode node,
+ int leftCount,
+ int rightCount,
+ List<RexNode> extraLeftExprs,
+ List<RexNode> extraRightExprs) {
+ switch (node.getKind()) {
+ case AND:
+ case OR:
+ case EQUALS:
+ RexCall call = (RexCall) node;
+ List<RexNode> list = new ArrayList<RexNode>();
+ List<RexNode> operands = Lists.newArrayList(call.getOperands());
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ final int left2 = leftCount + extraLeftExprs.size();
+ final int right2 = rightCount + extraRightExprs.size();
+ final RexNode e =
+ pushDownJoinConditions(
+ operand,
+ leftCount,
+ rightCount,
+ extraLeftExprs,
+ extraRightExprs);
+ final List<RexNode> remainingOperands = Util.skip(operands, i + 1);
+ final int left3 = leftCount + extraLeftExprs.size();
+ final int right3 = rightCount + extraRightExprs.size();
+ fix(remainingOperands, left2, left3);
+ fix(list, left2, left3);
+ list.add(e);
+ }
+ if (!list.equals(call.getOperands())) {
+ return call.clone(call.getType(), list);
+ }
+ return call;
+ case INPUT_REF:
+ case LITERAL:
+ return node;
+ default:
+ ImmutableBitSet bits = RelOptUtil.InputFinder.bits(node);
+ final int mid = leftCount + extraLeftExprs.size();
+ switch (Side.of(bits, mid)) {
+ case LEFT:
+ fix(extraRightExprs, mid, mid + 1);
+ extraLeftExprs.add(node);
+ return new RexInputRef(mid, node.getType());
+ case RIGHT:
+ final int index2 = mid + rightCount + extraRightExprs.size();
+ extraRightExprs.add(node);
+ return new RexInputRef(index2, node.getType());
+ case BOTH:
+ case EMPTY:
+ default:
+ return node;
+ }
+ }
+ }
+
+ private void fix(List<RexNode> operands, int before, int after) {
+ if (before == after) {
+ return;
+ }
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode node = operands.get(i);
+ operands.set(i, RexUtil.shift(node, before, after - before));
+ }
+ }
+
+ /**
+ * Categorizes whether a bit set contains bits left and right of a
+ * line.
+ */
+ enum Side {
+ LEFT, RIGHT, BOTH, EMPTY;
+
+ static Side of(ImmutableBitSet bitSet, int middle) {
+ final int firstBit = bitSet.nextSetBit(0);
+ if (firstBit < 0) {
+ return EMPTY;
+ }
+ if (firstBit >= middle) {
+ return RIGHT;
+ }
+ if (bitSet.nextSetBit(middle) < 0) {
+ return LEFT;
+ }
+ return BOTH;
+ }
+ }
+
+ /**
+ * Determines whether a subquery is non-correlated. Note that a
+ * non-correlated subquery can contain correlated references, provided those
+ * references do not reference select statements that are parents of the
+ * subquery.
+ *
+ * @param subq the subquery
+ * @param bb blackboard used while converting the subquery, i.e., the
+ * blackboard of the parent query of this subquery
+ * @return true if the subquery is non-correlated.
+ */
+ private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
+ Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
+ for (String correlName : correlatedVariables) {
+ DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
+ String originalRelName = lookup.getOriginalRelName();
+
+ int[] nsIndexes = {-1};
+ final SqlValidatorScope[] ancestorScopes = {null};
+ SqlValidatorNamespace foundNs =
+ lookup.bb.scope.resolve(
+ ImmutableList.of(originalRelName),
+ ancestorScopes,
+ nsIndexes);
+
+ assert foundNs != null;
+ assert nsIndexes.length == 1;
+
+ SqlValidatorScope ancestorScope = ancestorScopes[0];
+
+ // If the correlated reference is in a scope that's "above" the
+ // subquery, then this is a correlated subquery.
+ SqlValidatorScope parentScope = bb.scope;
+ do {
+ if (ancestorScope == parentScope) {
+ return false;
+ }
+ if (parentScope instanceof DelegatingScope) {
+ parentScope = ((DelegatingScope) parentScope).getParent();
+ } else {
+ break;
+ }
+ } while (parentScope != null);
+ }
+ return true;
+ }
+
+ /**
+ * Returns a list of fields to be prefixed to each relational expression.
+ *
+ * @return List of system fields
+ */
+ protected List<RelDataTypeField> getSystemFields() {
+ return Collections.emptyList();
+ }
+
+ private RexNode convertJoinCondition(
+ Blackboard bb,
+ SqlNode condition,
+ JoinConditionType conditionType,
+ RelNode leftRel,
+ RelNode rightRel) {
+ if (condition == null) {
+ return rexBuilder.makeLiteral(true);
+ }
+ bb.setRoot(ImmutableList.of(leftRel, rightRel));
+ replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
+ switch (conditionType) {
+ case ON:
+ bb.setRoot(ImmutableList.of(leftRel, rightRel));
+ return bb.convertExpression(condition);
+ case USING:
+ SqlNodeList list = (SqlNodeList) condition;
+ List<String> nameList = new ArrayList<String>();
+ for (SqlNode columnName : list) {
+ final SqlIdentifier id = (SqlIdentifier) columnName;
+ String name = id.getSimple();
+ nameList.add(name);
+ }
+ return convertUsing(leftRel, rightRel, nameList);
+ default:
+ throw Util.unexpected(conditionType);
+ }
+ }
+
+ /**
+ * Returns an expression for matching columns of a USING clause or inferred
+ * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
+ * b.y". Returns null if the column list is empty.
+ *
+ * @param leftRel Left input to the join
+ * @param rightRel Right input to the join
+ * @param nameList List of column names to join on
+ * @return Expression to match columns from name list, or true if name list
+ * is empty
+ */
+ private RexNode convertUsing(
+ RelNode leftRel,
+ RelNode rightRel,
+ List<String> nameList) {
+ final List<RexNode> list = Lists.newArrayList();
+ for (String name : nameList) {
+ final RelDataType leftRowType = leftRel.getRowType();
+ RelDataTypeField leftField = catalogReader.field(leftRowType, name);
+ RexNode left =
+ rexBuilder.makeInputRef(
+ leftField.getType(),
+ leftFiel
<TRUNCATED>
[45/50] [abbrv] incubator-kylin git commit: KYLIN-867 backup and
export metadata on finishing building II
Posted by li...@apache.org.
KYLIN-867 backup and export metadata on finishing building II
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/88baae92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/88baae92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/88baae92
Branch: refs/heads/0.7
Commit: 88baae927354c52dc37ef776fb432b8cff92d7e4
Parents: 8d08e6f
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jul 6 17:48:51 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 6 17:48:51 2015 +0800
----------------------------------------------------------------------
job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/88baae92/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 6dacc08..b0bf818 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -135,7 +135,7 @@ public class BuildIIWithEngineTest {
ii.setStatus(RealizationStatusEnum.READY);
iiManager.updateII(ii);
}
- // backup();
+ backup();
}
@Test
[29/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
index 06cad92..59179df 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
@@ -18,23 +18,25 @@
package org.apache.kylin.query.relnode;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Stack;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
/**
- *
- * @author xjiang
- *
*/
public interface OLAPRel extends RelNode {
@@ -137,24 +139,52 @@ public interface OLAPRel extends RelNode {
*/
public static class JavaImplementor extends EnumerableRelImplementor {
- private OLAPContext parentContext;
+ private IdentityHashMap<EnumerableRel, OLAPContext> relContexts = Maps.newIdentityHashMap();
+ private boolean calciteDebug = System.getProperty("calcite.debug") != null;
public JavaImplementor(EnumerableRelImplementor enumImplementor) {
super(enumImplementor.getRexBuilder(), new LinkedHashMap<String, Object>());
}
- public OLAPContext getParentContext() {
- return parentContext;
+ public EnumerableRel createEnumerable(OLAPRel parent) {
+ ArrayList<EnumerableRel> enumInputs = null;
+ List<RelNode> children = parent.getInputs();
+ if (children != null) {
+ enumInputs = Lists.newArrayListWithCapacity(children.size());
+ for (RelNode child : children) {
+ enumInputs.add(createEnumerable((OLAPRel) child));
+ }
+ }
+
+ EnumerableRel result = parent.implementEnumerable(enumInputs);
+ relContexts.put(result, parent.getContext());
+ return result;
}
@Override
public EnumerableRel.Result visitChild(EnumerableRel parent, int ordinal, EnumerableRel child, EnumerableRel.Prefer prefer) {
- if (parent instanceof OLAPRel) {
- OLAPRel olapRel = (OLAPRel) parent;
- this.parentContext = olapRel.getContext();
+ // OLAPTableScan is shared instance when the same table appears multiple times in the tree.
+ // Its context must be set (or corrected) right before visiting.
+ if (child instanceof OLAPTableScan) {
+ OLAPContext parentContext = relContexts.get(parent);
+ if (parentContext != null) {
+ ((OLAPTableScan) child).overrideContext(parentContext);
+ }
}
+
+ if (calciteDebug) {
+ OLAPContext context;
+ if (child instanceof OLAPRel)
+ context = ((OLAPRel) child).getContext();
+ else
+ context = relContexts.get(child);
+ System.out.println(context + " - " + child);
+ }
+
return super.visitChild(parent, ordinal, child, prefer);
}
}
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
index 6287f68..fa5dc1d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
@@ -18,32 +18,30 @@
package org.apache.kylin.query.relnode;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableSortRel;
-
-import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelFieldCollation;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexNode;
-
-import com.google.common.base.Preconditions;
+import java.util.List;
+
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableSort;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rex.RexNode;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
+import com.google.common.base.Preconditions;
+
/**
- * @author xjiang
- *
*/
-public class OLAPSortRel extends SortRel implements EnumerableRel, OLAPRel {
+public class OLAPSortRel extends Sort implements OLAPRel {
private ColumnRowType columnRowType;
private OLAPContext context;
@@ -66,26 +64,26 @@ public class OLAPSortRel extends SortRel implements EnumerableRel, OLAPRel {
@Override
public void implementOLAP(OLAPImplementor implementor) {
- implementor.visitChild(getChild(), this);
+ implementor.visitChild(getInput(), this);
this.context = implementor.getContext();
this.columnRowType = buildColumnRowType();
}
private ColumnRowType buildColumnRowType() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
return inputColumnRowType;
}
@Override
public void implementRewrite(RewriteImplementor implementor) {
- implementor.visitChild(this, getChild());
+ implementor.visitChild(this, getInput());
for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
int index = fieldCollation.getFieldIndex();
StorageContext.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
- OLAPRel olapChild = (OLAPRel) this.getChild();
+ OLAPRel olapChild = (OLAPRel) this.getInput();
TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
MeasureDesc measure = findMeasure(orderCol);
if (measure != null) {
@@ -116,17 +114,9 @@ public class OLAPSortRel extends SortRel implements EnumerableRel, OLAPRel {
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- OLAPRel childRel = (OLAPRel) getChild();
- childRel.replaceTraitSet(EnumerableConvention.INSTANCE);
-
- EnumerableSortRel enumSort = new EnumerableSortRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE, collation), getChild(), collation, offset, fetch);
-
- Result res = enumSort.implement(implementor, pref);
-
- childRel.replaceTraitSet(OLAPRel.CONVENTION);
-
- return res;
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
+ return new EnumerableSort(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE, collation), //
+ sole(inputs), collation, offset, fetch);
}
@Override
@@ -141,7 +131,7 @@ public class OLAPSortRel extends SortRel implements EnumerableRel, OLAPRel {
@Override
public boolean hasSubQuery() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
return olapChild.hasSubQuery();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index af76511..fc1fdf5 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -22,32 +22,32 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import net.hydromatic.linq4j.expressions.Blocks;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.linq4j.expressions.Primitive;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.PhysType;
-import net.hydromatic.optiq.rules.java.PhysTypeImpl;
-
-import org.apache.kylin.query.schema.OLAPSchema;
-import org.apache.kylin.query.schema.OLAPTable;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.RelWriter;
-import org.eigenbase.rel.TableAccessRelBase;
-import org.eigenbase.rel.rules.*;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.relopt.volcano.AbstractConverter.ExpandConversionRule;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.reltype.RelDataTypeField;
-
-import com.google.common.base.Preconditions;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.adapter.enumerable.PhysType;
+import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
+import org.apache.calcite.linq4j.tree.Primitive;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.volcano.AbstractConverter.ExpandConversionRule;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
+import org.apache.calcite.rel.rules.FilterJoinRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.JoinCommuteRule;
+import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.optrule.OLAPAggregateRule;
@@ -57,11 +57,14 @@ import org.apache.kylin.query.optrule.OLAPLimitRule;
import org.apache.kylin.query.optrule.OLAPProjectRule;
import org.apache.kylin.query.optrule.OLAPSortRule;
import org.apache.kylin.query.optrule.OLAPToEnumerableConverterRule;
+import org.apache.kylin.query.schema.OLAPSchema;
+import org.apache.kylin.query.schema.OLAPTable;
+
+import com.google.common.base.Preconditions;
/**
- * @author xjiang
*/
-public class OLAPTableScan extends TableAccessRelBase implements OLAPRel, EnumerableRel {
+public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
private final OLAPTable olapTable;
private final String tableName;
@@ -94,6 +97,10 @@ public class OLAPTableScan extends TableAccessRelBase implements OLAPRel, Enumer
return context;
}
+ void overrideContext(OLAPContext context) {
+ this.context = context;
+ }
+
@Override
public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
Preconditions.checkArgument(inputs.isEmpty());
@@ -115,20 +122,19 @@ public class OLAPTableScan extends TableAccessRelBase implements OLAPRel, Enumer
planner.addRule(OLAPSortRule.INSTANCE);
// since join is the entry point, we can't push filter past join
- planner.removeRule(PushFilterPastJoinRule.FILTER_ON_JOIN);
- planner.removeRule(PushFilterPastJoinRule.JOIN);
+ planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
+ planner.removeRule(FilterJoinRule.JOIN);
// TODO : since we don't have statistic of table, the optimization of join is too cost
- planner.removeRule(SwapJoinRule.INSTANCE);
- planner.removeRule(PushJoinThroughJoinRule.LEFT);
- planner.removeRule(PushJoinThroughJoinRule.RIGHT);
+ planner.removeRule(JoinCommuteRule.INSTANCE);
+ planner.removeRule(JoinPushThroughJoinRule.LEFT);
+ planner.removeRule(JoinPushThroughJoinRule.RIGHT);
- // for columns in having clause will enable table scan filter rule
- // cause kylin does not depend on MPP
- planner.removeRule(PushFilterPastProjectRule.INSTANCE);
+ // for columns in having clause will enable table scan filter rule cause kylin does not depend on MPP
+ planner.removeRule(FilterProjectTransposeRule.INSTANCE);
// distinct count will be split into a separated query that is joined with the left query
- planner.removeRule(RemoveDistinctAggregateRule.INSTANCE);
-
+ planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
+
// see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser
planner.removeRule(ExpandConversionRule.INSTANCE);
}
@@ -185,26 +191,22 @@ public class OLAPTableScan extends TableAccessRelBase implements OLAPRel, Enumer
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- if (!(implementor instanceof JavaImplementor))
- throw new IllegalStateException("implementor is not JavaImplementor");
- JavaImplementor javaImplementor = (JavaImplementor) implementor;
-
- int ctxId = this.context.id;
- if (javaImplementor.getParentContext() != null) {
- ctxId = javaImplementor.getParentContext().id;
- }
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
+ return this;
+ }
- PhysType physType = PhysTypeImpl.of(javaImplementor.getTypeFactory(), this.rowType, pref.preferArray());
+ @Override
+ public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
+ PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), this.rowType, pref.preferArray());
String execFunction = genExecFunc();
- return javaImplementor.result(physType, Blocks.toBlock(Expressions.call(table.getExpression(OLAPTable.class), execFunction, javaImplementor.getRootExpression(), Expressions.constant(ctxId))));
+ MethodCallExpression exprCall = Expressions.call(table.getExpression(OLAPTable.class), execFunction, implementor.getRootExpression(), Expressions.constant(context.id));
+ return implementor.result(physType, Blocks.toBlock(exprCall));
}
private String genExecFunc() {
- // if the table to scan is not the fact table of cube, then it's a
- // lookup table
+ // if the table to scan is not the fact table of cube, then it's a lookup table
if (context.hasJoin == false && tableName.equalsIgnoreCase(context.realization.getFactTable()) == false) {
return "executeLookupTableQuery";
} else {
@@ -231,7 +233,7 @@ public class OLAPTableScan extends TableAccessRelBase implements OLAPRel, Enumer
for (Map.Entry<String, RelDataType> rewriteField : rewriteFields.entrySet()) {
String fieldName = rewriteField.getKey();
- RelDataTypeField field = rowType.getField(fieldName, true);
+ RelDataTypeField field = rowType.getField(fieldName, true, false);
if (field != null) {
RelDataType fieldType = field.getType();
rewriteField.setValue(fieldType);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 54cafe5..af69ec5 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -20,31 +20,31 @@ package org.apache.kylin.query.relnode;
import java.util.List;
-import net.hydromatic.linq4j.expressions.Blocks;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.PhysType;
-import net.hydromatic.optiq.rules.java.PhysTypeImpl;
-
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.adapter.enumerable.PhysType;
+import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterImpl;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.convert.ConverterRelImpl;
-import org.eigenbase.relopt.ConventionTraitDef;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
/**
- * @author xjiang
*/
-public class OLAPToEnumerableConverter extends ConverterRelImpl implements EnumerableRel {
+public class OLAPToEnumerableConverter extends ConverterImpl implements EnumerableRel {
public OLAPToEnumerableConverter(RelOptCluster cluster, RelTraitSet traits, RelNode input) {
super(cluster, ConventionTraitDef.INSTANCE, traits, input);
@@ -64,7 +64,7 @@ public class OLAPToEnumerableConverter extends ConverterRelImpl implements Enume
public Result implement(EnumerableRelImplementor enumImplementor, Prefer pref) {
// post-order travel children
OLAPRel.OLAPImplementor olapImplementor = new OLAPRel.OLAPImplementor();
- olapImplementor.visitChild(getChild(), this);
+ olapImplementor.visitChild(getInput(), this);
// find cube from olap context
try {
@@ -84,13 +84,20 @@ public class OLAPToEnumerableConverter extends ConverterRelImpl implements Enume
// rewrite query if necessary
OLAPRel.RewriteImplementor rewriteImplementor = new OLAPRel.RewriteImplementor();
- rewriteImplementor.visitChild(this, getChild());
-
- // build java implementation
- EnumerableRel child = (EnumerableRel) getChild();
- OLAPRel.JavaImplementor javaImplementor = new OLAPRel.JavaImplementor(enumImplementor);
- return javaImplementor.visitChild(this, 0, child, pref);
+ rewriteImplementor.visitChild(this, getInput());
+
+ // implement as EnumerableRel
+ OLAPRel.JavaImplementor impl = new OLAPRel.JavaImplementor(enumImplementor);
+ EnumerableRel inputAsEnum = impl.createEnumerable((OLAPRel) getInput());
+ this.replaceInput(0, inputAsEnum);
+
+ if (System.getProperty("calcite.debug") != null) {
+ String dumpPlan = RelOptUtil.dumpPlan("", this, false, SqlExplainLevel.DIGEST_ATTRIBUTES);
+ System.out.println("EXECUTION PLAN AFTER REWRITE");
+ System.out.println(dumpPlan);
+ }
+ return impl.visitChild(this, 0, inputAsEnum, pref);
}
private Result buildHiveResult(EnumerableRelImplementor enumImplementor, Prefer pref, OLAPContext context) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
index 332a5ff..27d456f 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
@@ -22,9 +22,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import net.hydromatic.optiq.Table;
-import net.hydromatic.optiq.impl.AbstractSchema;
-
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.impl.AbstractSchema;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.metadata.MetadataManager;
@@ -33,7 +32,6 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
/**
- * @author xjiang
*/
public class OLAPSchema extends AbstractSchema {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 1845beb..f8711b1 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -25,12 +25,10 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.SchemaFactory;
-import net.hydromatic.optiq.SchemaPlus;
-
-import org.eigenbase.util14.ConversionUtil;
-
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaFactory;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.util.ConversionUtil;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.DatabaseDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -38,7 +36,6 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
/**
- * @author xjiang
*/
public class OLAPSchemaFactory implements SchemaFactory {
@@ -52,7 +49,8 @@ public class OLAPSchemaFactory implements SchemaFactory {
* object and does not require additional codec.
*
* Ref SaffronProperties.defaultCharset
- * SqlUtil.translateCharacterSetName() NlsString constructor()
+ * Ref SqlUtil.translateCharacterSetName()
+ * Ref NlsString constructor()
*/
System.setProperty("saffron.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
System.setProperty("saffron.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
@@ -109,13 +107,7 @@ public class OLAPSchemaFactory implements SchemaFactory {
out.write(" \"factory\": \"org.apache.kylin.query.schema.OLAPSchemaFactory\",\n");
out.write(" \"operand\": {\n");
out.write(" \"" + SCHEMA_PROJECT + "\": \"" + project + "\"\n");
- out.write(" },\n");
- out.write(" \"functions\": [\n");
- out.write(" {\n");
- out.write(" \"name\": \"QUARTER\",\n");
- out.write(" \"className\": \"org.apache.kylin.query.sqlfunc.QuarterFunc\"\n");
- out.write(" }\n");
- out.write(" ]\n");
+ out.write(" }\n");
out.write(" }\n");
if (++counter != schemaCounts.size()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index cd04230..f4eaa08 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,33 +19,30 @@
package org.apache.kylin.query.schema;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.linq4j.QueryProvider;
-import net.hydromatic.linq4j.Queryable;
-import net.hydromatic.optiq.DataContext;
-import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.Statistic;
-import net.hydromatic.optiq.Statistics;
-import net.hydromatic.optiq.TranslatableTable;
-import net.hydromatic.optiq.impl.AbstractTableQueryable;
-import net.hydromatic.optiq.impl.java.AbstractQueryableTable;
-
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.relopt.RelOptTable.ToRelContext;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.sql.type.SqlTypeName;
-import org.eigenbase.sql.type.SqlTypeUtil;
-
-import com.google.common.collect.Lists;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.adapter.java.AbstractQueryableTable;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptTable.ToRelContext;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Statistic;
+import org.apache.calcite.schema.Statistics;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.impl.AbstractTableQueryable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -55,10 +52,9 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.enumerator.OLAPQuery.EnumeratorTypeEnum;
import org.apache.kylin.query.relnode.OLAPTableScan;
+import com.google.common.collect.Lists;
+
/**
- *
- * @author xjiang
- *
*/
public class OLAPTable extends AbstractQueryableTable implements TranslatableTable {
@@ -122,12 +118,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
if (this.rowType == null) {
// always build exposedColumns and rowType together
this.exposedColumns = listSourceColumns();
- this.rowType = deduceRowType(typeFactory);
+ this.rowType = deriveRowType(typeFactory);
}
return this.rowType;
}
- private RelDataType deduceRowType(RelDataTypeFactory typeFactory) {
+ private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : exposedColumns) {
RelDataType sqlType = createSqlType(typeFactory, column);
@@ -153,8 +149,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
else
result = typeFactory.createSqlType(sqlTypeName);
- // due to left join and uncertain data quality, dimension value can be
- // null
+ // due to left join and uncertain data quality, dimension value can be null
if (column.isNullable()) {
result = typeFactory.createTypeWithNullability(result, true);
} else {
@@ -177,8 +172,9 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
metFields.add(fieldName);
ColumnDesc fakeCountCol = new ColumnDesc();
fakeCountCol.setName(fieldName);
- fakeCountCol.setDatatype(func.getSQLType());
- fakeCountCol.setNullable(false);
+ fakeCountCol.setDatatype(func.getSQLType().toString());
+ if (func.isCount())
+ fakeCountCol.setNullable(false);
fakeCountCol.init(sourceTable);
exposedColumns.add(fakeCountCol);
}
@@ -215,7 +211,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
@Override
public Statistic getStatistic() {
- List<BitSet> keys = new ArrayList<BitSet>();
+ List<ImmutableBitSet> keys = new ArrayList<ImmutableBitSet>();
return Statistics.of(100, keys);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterBase.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterBase.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterBase.java
deleted file mode 100644
index 1bd0dff..0000000
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterBase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.sqlfunc;
-
-import net.hydromatic.optiq.runtime.SqlFunctions;
-import net.hydromatic.optiq.runtime.SqlFunctions.TimeUnitRange;
-
-/**
- * @author xjiang
- *
- */
-public abstract class QuarterBase {
-
- /**
- * According to jvm spec, it return self method before parent.
- * So, we keep Date in parent and int in child
- */
- public static long eval(int date) {
- long month = SqlFunctions.unixDateExtract(TimeUnitRange.MONTH, date);
- return month / 4 + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterFunc.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterFunc.java
deleted file mode 100644
index 179caf1..0000000
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/QuarterFunc.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.sqlfunc;
-
-import java.sql.Date;
-
-/**
- * @author xjiang
- *
- */
-public class QuarterFunc extends QuarterBase {
- private QuarterFunc() {
- }
-
- public static long eval(Date date) {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
index 55699d3..819d2f9 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
@@ -423,7 +423,7 @@ public class KylinTestBase {
int count = executeQuery(sql, true);
if (debug) {
- System.clearProperty("optiq.debug");
+ System.clearProperty("calcite.debug");
}
return count;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 027202d..09499ee 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,14 +41,19 @@ import java.util.Set;
import javax.sql.DataSource;
-import net.hydromatic.avatica.ColumnMetaData.Rep;
-
+import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.metrics.QueryMetrics;
import org.apache.kylin.rest.model.ColumnMeta;
@@ -67,13 +72,6 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.query.relnode.OLAPContext;
-
/**
* @author xduo
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 4589e4d..268c098 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -315,7 +315,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
List<String> names = rowValueDecoder.getNames();
MeasureDesc[] measures = rowValueDecoder.getMeasures();
for (int i = 0; i < measures.length; i++) {
- String dataType = measures[i].getFunction().getSQLType();
+ String dataType = measures[i].getFunction().getSQLType().getName();
info.setField(names.get(i), null, dataType, index++);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 1d2a355..f865dff 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -239,7 +239,7 @@ public class EndpointTupleIterator implements ITupleIterator {
}
for (FunctionDesc measure : measures) {
- info.setField(measure.getRewriteFieldName(), null, measure.getSQLType(), index++);
+ info.setField(measure.getRewriteFieldName(), null, measure.getSQLType().getName(), index++);
}
return info;
[16/50] [abbrv] incubator-kylin git commit: KYLIN-839 bug fix
Posted by li...@apache.org.
KYLIN-839 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7fdcd31d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7fdcd31d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7fdcd31d
Branch: refs/heads/0.7
Commit: 7fdcd31d07b6f1b5000e96a90018747158a59505
Parents: 9a467fd
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jun 29 19:11:54 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jun 29 19:11:54 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/dict/lookup/SnapshotTable.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7fdcd31d/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 130a4c2..28db5b4 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -209,6 +209,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
int rowNum = in.readInt();
if (rowNum > 0) {
int n = in.readInt();
+ rowIndices = new ArrayList<int[]>(rowNum);
+
if (this.useDictionary == true) {
this.dict = new TrieDictionary<String>();
dict.readFields(in);
@@ -222,7 +224,6 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
}
} else {
List<String[]> rows = new ArrayList<String[]>(rowNum);
- ArrayList<int[]> allRowIndices = new ArrayList<int[]>();
TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
for (int i = 0; i < rowNum; i++) {
@@ -239,9 +240,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
for (int i = 0; i < n; i++) {
rowIndex[i] = dict.getIdFromValue(row[i]);
}
- allRowIndices.add(rowIndex);
+ this.rowIndices.add(rowIndex);
}
- this.rowIndices = allRowIndices;
}
}
}
[05/50] [abbrv] incubator-kylin git commit: hot fix: filter
evaluation performance issue caused by TableDesc
Posted by li...@apache.org.
hot fix: filter evaluation performance issue caused by TableDesc
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e0733c96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e0733c96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e0733c96
Branch: refs/heads/0.7
Commit: e0733c96db5ed908ecb2fead8067beb42d0fd445
Parents: e8132b7
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 21:20:25 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:20:25 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/TableDesc.java | 13 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 10 +-
.../kylin/storage/filter/FilterPerfTest.java | 132 +++++++++++++++++++
3 files changed, 146 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 14ea390..30c4d76 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -18,9 +18,6 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-import java.util.Comparator;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -28,6 +25,9 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
+import java.util.Arrays;
+import java.util.Comparator;
+
/**
* Table Metadata from Source. All name should be uppercase.
* <p/>
@@ -43,6 +43,8 @@ public class TableDesc extends RootPersistentEntity {
private DatabaseDesc database;
+ private String identity = null;
+
public ColumnDesc findColumnByName(String name) {
//ignore the db name and table name if exists
int lastIndexOfDot = name.lastIndexOf(".");
@@ -73,7 +75,10 @@ public class TableDesc extends RootPersistentEntity {
}
public String getIdentity() {
- return String.format("%s.%s", this.getDatabase().toUpperCase(), this.getName()).toUpperCase();
+ if (identity == null) {
+ identity = String.format("%s.%s", this.getDatabase().toUpperCase(), this.getName()).toUpperCase();
+ }
+ return identity;
}
public static String concatResourcePath(String tableIdentity) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 5023580..92e5174 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -18,10 +18,6 @@
package org.apache.kylin.storage.tuple;
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.cube.CubeManager;
@@ -31,6 +27,10 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
/**
* @author xjiang
*/
@@ -82,7 +82,7 @@ public class Tuple implements ITuple {
return info.getDataType(fieldName);
}
- private void setFieldObjectValue(String fieldName, Object fieldValue) {
+ public void setFieldObjectValue(String fieldName, Object fieldValue) {
int index = info.getFieldIndex(fieldName);
values[index] = fieldValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0733c96/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
new file mode 100644
index 0000000..e8826cd
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterPerfTest.java
@@ -0,0 +1,132 @@
+package org.apache.kylin.storage.filter;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ */
+public class FilterPerfTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void foo() throws IOException, InterruptedException {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ TblColRef format = new TblColRef(tableDesc.findColumnByName("LSTG_FORMAT_NAME"));
+ TblColRef categ = new TblColRef(tableDesc.findColumnByName("LEAF_CATEG_ID"));
+ TblColRef site = new TblColRef(tableDesc.findColumnByName("LSTG_SITE_ID"));
+
+ List<TupleFilter> ands = Lists.newArrayList();
+ ands.add(buildOrFilterWithMultipleValues(format, Lists.newArrayList("Auction", "FP-GTC", "Others")));
+ ands.add(buildOrFilterWithMultipleValues(categ, Lists.newArrayList("48027", "164261", "113802", "118687")));
+ ands.add(buildOrFilterWithMultipleValues(site, Lists.newArrayList("0", "15", "3")));
+ TupleFilter filter = buildAndFilter(ands);
+
+ TupleInfo info = new TupleInfo();
+ ColumnDesc[] columns = tableDesc.getColumns();
+ for (int i = 0; i < columns.length; i++) {
+ ColumnDesc column = columns[i];
+ info.setField(column.getName(), new TblColRef(column), column.getDatatype(), i);
+ }
+
+ List<String> lines = Files.readLines(new File("../examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv"), Charset.defaultCharset());
+ List<Tuple> tuples = Lists.newArrayList();
+
+ for (String line : lines) {
+ String[] tokens = line.split(",");
+ if (tokens.length != columns.length) {
+ System.out.println("invalid line");
+ continue;
+ }
+ Tuple t = new Tuple(info);
+ for (int k = 0; k < columns.length; k++) {
+ ColumnDesc column = columns[k];
+ t.setFieldObjectValue(column.getName(), tokens[k]);
+ }
+ tuples.add(t);
+ }
+
+ for (int i = 0; i < 5; i++) {
+ Thread.sleep(1000);
+ System.out.println(i);
+ }
+
+ Iterator<Tuple> itr = Iterables.cycle(tuples).iterator();
+ int TOTAL_LOOP = 1000000;
+ int loopCount = 0;
+ int matchCount = 0;
+ long startTime = System.currentTimeMillis();
+ while (itr.hasNext()) {
+ if (filter.evaluate(itr.next())) {
+ matchCount++;
+ }
+
+ if (++loopCount > TOTAL_LOOP) {
+ break;
+ }
+ }
+ System.out.println("Total match count: " + matchCount);
+ System.out.println("ellapsed time: " + (System.currentTimeMillis() - startTime));
+ }
+
+ private TupleFilter buildOrFilterWithMultipleValues(TblColRef column, List<String> values) {
+ List<TupleFilter> ors = Lists.newArrayList();
+ for (String v : values) {
+ ors.add(buildCompareFilter(column, v));
+ }
+ return buildOrFilter(ors);
+ }
+
+ protected CompareTupleFilter buildCompareFilter(TblColRef column, String v) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = new ConstantTupleFilter(v);
+ compareFilter.addChild(constantFilter);
+ return compareFilter;
+ }
+
+ protected TupleFilter buildAndFilter(List<TupleFilter> ands) {
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ for (TupleFilter x : ands) {
+ andFilter.addChild(x);
+ }
+ return andFilter;
+ }
+
+ protected TupleFilter buildOrFilter(List<TupleFilter> ors) {
+ LogicalTupleFilter orFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ for (TupleFilter x : ors) {
+ orFilter.addChild(x);
+ }
+ return orFilter;
+ }
+
+}
[38/50] [abbrv] incubator-kylin git commit: minor bug fix,
JDBC KylinMeta to report correct catalog metadata
Posted by li...@apache.org.
minor bug fix, JDBC KylinMeta to report correct catalog metadata
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ce77dd29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ce77dd29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ce77dd29
Branch: refs/heads/0.7
Commit: ce77dd29264bf932b4fe58e9a1de4e405846b691
Parents: b28e9f4
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Jul 6 13:39:49 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 6 13:50:16 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/jdbc/KylinMeta.java | 18 ++++++++++++------
.../java/org/apache/kylin/jdbc/DriverTest.java | 3 +++
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ce77dd29/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index 72d7fb1..821fba5 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -89,19 +89,19 @@ public class KylinMeta extends MetaImpl {
@Override
public MetaResultSet getCatalogs() {
List<KMetaCatalog> catalogs = getMetaProject().catalogs;
- return createResultSet(catalogs, MetaCatalog.class, "TABLE_CATALOG");
+ return createResultSet(catalogs, KMetaCatalog.class, "TABLE_CAT");
}
@Override
public MetaResultSet getSchemas(String catalog, Pat schemaPattern) {
List<KMetaSchema> schemas = getMetaProject().getSchemas(catalog, schemaPattern);
- return createResultSet(schemas, MetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG");
+ return createResultSet(schemas, KMetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG");
}
@Override
public MetaResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
List<KMetaTable> tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList);
- return createResultSet(tables, MetaTable.class, //
+ return createResultSet(tables, KMetaTable.class, //
"TABLE_CAT", //
"TABLE_SCHEM", //
"TABLE_NAME", //
@@ -117,7 +117,7 @@ public class KylinMeta extends MetaImpl {
@Override
public MetaResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
List<KMetaColumn> columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern);
- return createResultSet(columns, MetaColumn.class, //
+ return createResultSet(columns, KMetaColumn.class, //
"TABLE_CAT", //
"TABLE_SCHEM", //
"TABLE_NAME", //
@@ -285,13 +285,19 @@ public class KylinMeta extends MetaImpl {
}
}
- public static class KMetaCatalog extends MetaCatalog implements NamedWithChildren {
+ public static class KMetaCatalog implements NamedWithChildren {
+ public final String tableCat;
public final List<KMetaSchema> schemas;
public KMetaCatalog(String tableCatalog, List<KMetaSchema> schemas) {
- super(tableCatalog);
+ this.tableCat = tableCatalog;
this.schemas = schemas;
}
+
+ @Override
+ public String getName() {
+ return tableCat;
+ }
@Override
public List<? extends NamedWithChildren> getChildren() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ce77dd29/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index c17efb1..3a2ef75 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -100,14 +100,17 @@ public class DriverTest {
ResultSet catalogs = conn.getMetaData().getCatalogs();
System.out.println("CATALOGS");
+ printResultSetMetaData(catalogs);
printResultSet(catalogs);
ResultSet schemas = conn.getMetaData().getSchemas();
System.out.println("SCHEMAS");
+ printResultSetMetaData(schemas);
printResultSet(schemas);
ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
System.out.println("TABLES");
+ printResultSetMetaData(tables);
printResultSet(tables);
for (int j = 0; j < 3; j++) {
[04/50] [abbrv] incubator-kylin git commit: KYLIN-844 add
BackdoorToggles to control fuzzy key
Posted by li...@apache.org.
KYLIN-844 add BackdoorToggles to control fuzzy key
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e8132b73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e8132b73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e8132b73
Branch: refs/heads/0.7
Commit: e8132b73ab00c1474f78821d62b240c51c25eadd
Parents: ad4a28a
Author: honma <ho...@ebay.com>
Authored: Wed Jun 24 12:02:06 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 24 21:16:52 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 53 ++++++++++++++++
.../org/apache/kylin/common/util/BytesUtil.java | 12 ++--
.../apache/kylin/common/util/BytesUtilTest.java | 9 +++
.../apache/kylin/job/tools/RowCounterCLI.java | 67 ++++++++++++++++++++
.../kylin/rest/controller/QueryController.java | 67 ++++++++++++--------
.../apache/kylin/rest/request/SQLRequest.java | 13 ++++
.../kylin/storage/hbase/HBaseKeyRange.java | 26 +++++---
.../coprocessor/observer/ObserverEnabler.java | 12 +++-
8 files changed, 218 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
new file mode 100644
index 0000000..315543f
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.common.debug;
+
+import java.util.Map;
+
+/**
+ */
+public class BackdoorToggles {
+
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal();
+
+ public static void setToggles(Map<String, String> toggles) {
+ _backdoorToggles.set(toggles);
+ }
+
+ public static String getToggle(String key) {
+ Map<String, String> toggles = _backdoorToggles.get();
+ if (toggles == null) {
+ return null;
+ } else {
+ return toggles.get(key);
+ }
+ }
+
+ public static void cleanToggles() {
+ _backdoorToggles.remove();
+ }
+
+ /**
+ * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
+ *
+ *
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
+
+ /**
+ * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 41946ca..decdc3a 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -18,16 +18,15 @@
package org.apache.kylin.common.util;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.Writable;
+
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.Writable;
-
public class BytesUtil {
public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@@ -371,6 +370,10 @@ public class BytesUtil {
return toHex(array);
}
+
+ /**
+ * this method only works for hex strings
+ */
public static byte[] fromReadableText(String text) {
String[] tokens = text.split("\\\\x");
byte[] ret = new byte[tokens.length - 1];
@@ -399,6 +402,7 @@ public class BytesUtil {
return sb.toString();
}
+
public static void main(String[] args) throws Exception {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 4c88cbe..2a3f0ef 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -53,4 +53,13 @@ public class BytesUtilTest extends TestCase {
assertEquals(y[1], false);
}
+ @Test
+ public void testReadable()
+ {
+ String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
+ byte[] bytes = BytesUtil.fromReadableText(x);
+ String y = BytesUtil.toHex(bytes);
+ assertEquals(x,y);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
new file mode 100644
index 0000000..b80063d
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
@@ -0,0 +1,67 @@
+package org.apache.kylin.job.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ */
+public class RowCounterCLI {
+ private static final Logger logger = LoggerFactory.getLogger(RowCounterCLI.class);
+
+ public static void main(String[] args) throws IOException {
+
+ if (args == null || args.length != 3) {
+ System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]");
+ }
+
+ System.out.println(args[0]);
+ String htableName = args[0];
+ System.out.println(args[1]);
+ byte[] startKey = BytesUtil.fromReadableText(args[1]);
+ System.out.println(args[2]);
+ byte[] endKey = BytesUtil.fromReadableText(args[2]);
+
+ if (startKey == null) {
+ System.out.println("startkey is null ");
+ } else {
+ System.out.println("startkey lenght: " + startKey.length);
+ }
+
+ System.out.println("start key in binary: " + Bytes.toStringBinary(startKey));
+ System.out.println("end key in binary: " + Bytes.toStringBinary(endKey));
+
+ Configuration conf = HBaseConfiguration.create();
+
+ Scan scan = new Scan();
+ scan.setCaching(1024);
+ scan.setCacheBlocks(true);
+ scan.setStartRow(startKey);
+ scan.setStopRow(endKey);
+
+ logger.info("My Scan " + scan.toString());
+
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
+
+ Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
+ int counter = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ counter++;
+ if (counter % 1000 == 1) {
+ System.out.println("number of rows: " + counter);
+ }
+ }
+ System.out.println("number of rows: " + counter);
+ tableInterface.close();
+ conn.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 165660e..7cd14ff 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -18,50 +18,45 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import javax.servlet.http.HttpServletResponse;
-
+import com.codahale.metrics.annotation.Timed;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
-
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.model.Query;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.MetaRequest;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.util.QueryUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import com.codahale.metrics.annotation.Timed;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.model.Query;
-import org.apache.kylin.rest.model.SelectedColumnMeta;
-import org.apache.kylin.rest.model.TableMeta;
-import org.apache.kylin.rest.request.PrepareSqlRequest;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.rest.request.SaveSqlRequest;
-import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryUtil;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
/**
* Handle query requests.
@@ -86,13 +81,15 @@ public class QueryController extends BasicController {
@ResponseBody
@Timed(name = "query")
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
- long startTimestamp = System.currentTimeMillis();
+ initDebugToggles(sqlRequest);
+ long startTimestamp = System.currentTimeMillis();
SQLResponse response = doQuery(sqlRequest);
response.setDuration(System.currentTimeMillis() - startTimestamp);
-
queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
+ cleanupDebugToggles();
+
return response;
}
@@ -261,4 +258,18 @@ public class QueryController extends BasicController {
this.cacheManager = cacheManager;
}
+ private void initDebugToggles(SQLRequest sqlRequest) {
+
+ Map<String, String> toggles = sqlRequest.getBackdoorToggles();
+ if (toggles == null || toggles.size() == 0) {
+ return;
+ }
+
+ BackdoorToggles.setToggles(toggles);
+ }
+
+ private void cleanupDebugToggles() {
+ BackdoorToggles.cleanToggles();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index a75ddfb..901badd 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.request;
+import java.util.Map;
+
public class SQLRequest {
private String sql;
@@ -26,9 +28,20 @@ public class SQLRequest {
private Integer limit = 0;
private boolean acceptPartial = true;
+ private Map<String,String> backdoorToggles;
+
public SQLRequest() {
}
+ public Map<String, String> getBackdoorToggles() {
+ return backdoorToggles;
+ }
+
+ public void setBackdoorToggles(Map<String, String> backdoorToggles) {
+ this.backdoorToggles = backdoorToggles;
+ }
+
+
public String getProject() {
return project;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index cb2dce7..278c50c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -18,12 +18,9 @@
package org.apache.kylin.storage.hbase;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.DateFormat;
@@ -36,9 +33,10 @@ import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.util.*;
/**
*
@@ -47,6 +45,8 @@ import com.google.common.collect.Maps;
*/
public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyRange.class);
+
private static final int FUZZY_VALUE_CAP = 20;
private static final byte[] ZERO_TAIL_BYTES = new byte[] { 0 };
@@ -160,6 +160,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
+ buf.append(System.lineSeparator());
}
this.fuzzyKeyString = buf.toString();
}
@@ -167,6 +168,15 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_DISABLE_FUZZY_KEY)) != null) {
+ if (Boolean.valueOf(toggle)) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+ }
+
FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
FuzzyMaskEncoder fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e8132b73/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index ce9ba9d..6acc40e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -72,8 +73,17 @@ public class ObserverEnabler {
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
+
+ //debug/profiling purpose
+ String toggle;
+ if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_OBSERVER_BEHAVIOR)) == null) {
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString();//default behavior
+ } else {
+ logger.info("The execution of this query will use " + toggle + " as observer's behavior");
+ }
+
scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
- scan.setAttribute(AggregateRegionObserver.BEHAVIOR, ObserverBehavior.SCAN_FILTER_AGGR.toString().getBytes());
+ scan.setAttribute(AggregateRegionObserver.BEHAVIOR, toggle.getBytes());
scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));
[08/50] [abbrv] incubator-kylin git commit: KYLIN-844 default
observer behavior if no attribute hint found
Posted by li...@apache.org.
KYLIN-844 default observer behavior if no attribute hint found
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/de7b13b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/de7b13b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/de7b13b0
Branch: refs/heads/0.7
Commit: de7b13b0b877aded419e9d057f10dbeff3cc76b1
Parents: cc25a04
Author: honma <ho...@ebay.com>
Authored: Thu Jun 25 13:26:02 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Jun 25 13:26:02 2015 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/observer/AggregateRegionObserver.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/de7b13b0/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index aea2cea..8771bf4 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -48,7 +48,6 @@ public class AggregateRegionObserver extends BaseRegionObserver {
static final String FILTER = "_Filter";
static final String BEHAVIOR = "_Behavior";
-
@Override
public final RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
@@ -85,8 +84,11 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
+ ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR;
byte[] behavior = scan.getAttribute(BEHAVIOR);
- ObserverBehavior observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ if (behavior != null && behavior.length != 0) {
+ observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ }
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
@@ -95,7 +97,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner,observerBehavior);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner, observerBehavior);
}
} finally {
region.closeRegionOperation();
[14/50] [abbrv] incubator-kylin git commit: KYLIN-839 use dictionary
in building snapshot
Posted by li...@apache.org.
KYLIN-839 use dictionary in building snapshot
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/08c8e910
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/08c8e910
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/08c8e910
Branch: refs/heads/0.7
Commit: 08c8e9103fa757330d8e4848552e563be9bcbf2e
Parents: b97d0e5
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jun 29 10:52:49 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jun 29 10:52:49 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/lookup/SnapshotTable.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/08c8e910/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 35cbb84..4764dd0 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -49,7 +49,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
@JsonProperty("column_delimeter")
private String columnDelimeter;
@JsonProperty("useDictionary")
- private Boolean useDictionary;
+ private boolean useDictionary;
private ArrayList<String[]> rows;
@@ -60,7 +60,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
SnapshotTable(ReadableTable table) throws IOException {
this.signature = table.getSignature();
this.columnDelimeter = table.getColumnDelimeter();
- this.useDictionary = Boolean.TRUE;
+ this.useDictionary = true;
}
public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
@@ -162,7 +162,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
int n = rows.get(0).length;
out.writeInt(n);
- if (this.useDictionary == Boolean.TRUE) {
+ if (this.useDictionary == true) {
Dictionary<String> dict = buildDictionary();
dict.write(out);
for (int i = 0; i < rows.size(); i++) {
@@ -199,7 +199,7 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
rows = new ArrayList<String[]>(rowNum);
if (rowNum > 0) {
int n = in.readInt();
- if (this.useDictionary == Boolean.TRUE) {
+ if (this.useDictionary == true) {
Dictionary<String> dict = new TrieDictionary<String>();
dict.readFields(in);
for (int i = 0; i < rowNum; i++) {
[46/50] [abbrv] incubator-kylin git commit: KYLIN-881 Upgrade to
Calcite 1.3.0
Posted by li...@apache.org.
KYLIN-881 Upgrade to Calcite 1.3.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f2b92061
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f2b92061
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f2b92061
Branch: refs/heads/0.7
Commit: f2b92061444f48fa016c3bc6a4dfff9331e63b41
Parents: 88baae9
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Jul 7 16:07:23 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Jul 7 16:07:23 2015 +0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableJoin.java | 26 +-
.../apache/calcite/runtime/SqlFunctions.java | 119 +++-
.../calcite/sql2rel/SqlToRelConverter.java | 660 ++++++++++---------
.../java/org/apache/kylin/jdbc/KylinClient.java | 2 +-
.../org/apache/kylin/jdbc/KylinJdbcFactory.java | 5 +-
.../java/org/apache/kylin/jdbc/KylinMeta.java | 38 +-
.../kylin/jdbc/KylinPreparedStatement.java | 50 +-
.../org/apache/kylin/jdbc/KylinResultSet.java | 9 +-
pom.xml | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 2 +
.../kylin/query/optrule/OLAPFilterRule.java | 4 +-
.../kylin/query/optrule/OLAPLimitRule.java | 5 +-
.../kylin/query/optrule/OLAPProjectRule.java | 6 +-
.../kylin/query/relnode/OLAPFilterRel.java | 4 +-
.../kylin/query/relnode/OLAPProjectRel.java | 12 +-
.../kylin/query/relnode/OLAPTableScan.java | 6 +-
16 files changed, 584 insertions(+), 366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
index 918b4bf..2b2ca8d 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java
@@ -1,6 +1,6 @@
/*
* OVERRIDE POINT:
- * - constructor was private instead of public
+ * - constructor was private instead of protected
*/
/*
@@ -31,10 +31,10 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelNodes;
+import org.apache.calcite.rel.core.EquiJoin;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.rules.EquiJoin;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.BuiltInMethod;
import org.apache.calcite.util.ImmutableIntList;
@@ -47,7 +47,10 @@ import java.util.Set;
/** Implementation of {@link org.apache.calcite.rel.core.Join} in
* {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */
public class EnumerableJoin extends EquiJoin implements EnumerableRel {
- public EnumerableJoin( // OVERRIDE POINT, the constructor was private
+ /** Creates an EnumerableJoin.
+ *
+ * <p>Use {@link #create} unless you know what you're doing. */
+ public EnumerableJoin(
RelOptCluster cluster,
RelTraitSet traits,
RelNode left,
@@ -70,6 +73,23 @@ public class EnumerableJoin extends EquiJoin implements EnumerableRel {
variablesStopped);
}
+ /** Creates an EnumerableJoin. */
+ public static EnumerableJoin create(
+ RelNode left,
+ RelNode right,
+ RexNode condition,
+ ImmutableIntList leftKeys,
+ ImmutableIntList rightKeys,
+ JoinRelType joinType,
+ Set<String> variablesStopped)
+ throws InvalidRelException {
+ final RelOptCluster cluster = left.getCluster();
+ final RelTraitSet traitSet =
+ cluster.traitSetOf(EnumerableConvention.INSTANCE);
+ return new EnumerableJoin(cluster, traitSet, left, right, condition,
+ leftKeys, rightKeys, joinType, variablesStopped);
+ }
+
@Override public EnumerableJoin copy(RelTraitSet traitSet, RexNode condition,
RelNode left, RelNode right, JoinRelType joinType,
boolean semiJoinDone) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 3b1a2e0..80b9fdd 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -549,7 +549,6 @@ public class SqlFunctions {
/** SQL <code>/</code> operator applied to BigDecimal values. */
public static BigDecimal divide(BigDecimal b0, BigDecimal b1) {
- // OVERRIDE POINT
return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64);
}
@@ -693,6 +692,124 @@ public class SqlFunctions {
return bigDecimals[1];
}
+ // FLOOR
+
+ public static double floor(double b0) {
+ return Math.floor(b0);
+ }
+
+ public static float floor(float b0) {
+ return (float) Math.floor(b0);
+ }
+
+ public static BigDecimal floor(BigDecimal b0) {
+ return b0.setScale(0, BigDecimal.ROUND_FLOOR);
+ }
+
+ /** SQL <code>FLOOR</code> operator applied to byte values. */
+ public static byte floor(byte b0, byte b1) {
+ return (byte) floor((int) b0, (int) b1);
+ }
+
+ /** SQL <code>FLOOR</code> operator applied to short values. */
+ public static short floor(short b0, short b1) {
+ return (short) floor((int) b0, (int) b1);
+ }
+
+ /** SQL <code>FLOOR</code> operator applied to int values. */
+ public static int floor(int b0, int b1) {
+ int r = b0 % b1;
+ if (r < 0) {
+ r += b1;
+ }
+ return b0 - r;
+ }
+
+ /** SQL <code>FLOOR</code> operator applied to long values. */
+ public static long floor(long b0, long b1) {
+ long r = b0 % b1;
+ if (r < 0) {
+ r += b1;
+ }
+ return b0 - r;
+ }
+
+ // temporary
+ public static BigDecimal floor(BigDecimal b0, int b1) {
+ return floor(b0, BigDecimal.valueOf(b1));
+ }
+
+ // temporary
+ public static int floor(int b0, BigDecimal b1) {
+ return floor(b0, b1.intValue());
+ }
+
+ public static BigDecimal floor(BigDecimal b0, BigDecimal b1) {
+ final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
+ BigDecimal r = bigDecimals[1];
+ if (r.signum() < 0) {
+ r = r.add(b1);
+ }
+ return b0.subtract(r);
+ }
+
+ // CEIL
+
+ public static double ceil(double b0) {
+ return Math.ceil(b0);
+ }
+
+ public static float ceil(float b0) {
+ return (float) Math.ceil(b0);
+ }
+
+ public static BigDecimal ceil(BigDecimal b0) {
+ return b0.setScale(0, BigDecimal.ROUND_CEILING);
+ }
+
+ /** SQL <code>CEIL</code> operator applied to byte values. */
+ public static byte ceil(byte b0, byte b1) {
+ return floor((byte) (b0 + b1 - 1), b1);
+ }
+
+ /** SQL <code>CEIL</code> operator applied to short values. */
+ public static short ceil(short b0, short b1) {
+ return floor((short) (b0 + b1 - 1), b1);
+ }
+
+ /** SQL <code>CEIL</code> operator applied to int values. */
+ public static int ceil(int b0, int b1) {
+ int r = b0 % b1;
+ if (r > 0) {
+ r -= b1;
+ }
+ return b0 - r;
+ }
+
+ /** SQL <code>CEIL</code> operator applied to long values. */
+ public static long ceil(long b0, long b1) {
+ return floor(b0 + b1 - 1, b1);
+ }
+
+ // temporary
+ public static BigDecimal ceil(BigDecimal b0, int b1) {
+ return ceil(b0, BigDecimal.valueOf(b1));
+ }
+
+ // temporary
+ public static int ceil(int b0, BigDecimal b1) {
+ return ceil(b0, b1.intValue());
+ }
+
+ public static BigDecimal ceil(BigDecimal b0, BigDecimal b1) {
+ final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1);
+ BigDecimal r = bigDecimals[1];
+ if (r.signum() > 0) {
+ r = r.subtract(b1);
+ }
+ return b0.subtract(r);
+ }
+
// ABS
/** SQL <code>ABS</code> operator applied to byte values. */
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 2508629..c643f94 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -27,14 +27,13 @@ import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptQuery;
import org.apache.calcite.plan.RelOptSamplingParameters;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.prepare.RelOptTableImpl;
import org.apache.calcite.rel.RelCollation;
-import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelShuttle;
@@ -47,7 +46,6 @@ import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.Sample;
-import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.core.Uncollect;
import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.calcite.rel.logical.LogicalCorrelate;
@@ -55,12 +53,14 @@ import org.apache.calcite.rel.logical.LogicalIntersect;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.logical.LogicalMinus;
import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalSort;
import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
import org.apache.calcite.rel.logical.LogicalTableModify;
import org.apache.calcite.rel.logical.LogicalTableScan;
import org.apache.calcite.rel.logical.LogicalUnion;
import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rel.metadata.RelColumnMapping;
+import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -80,6 +80,8 @@ import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.rex.RexWindowBound;
import org.apache.calcite.schema.ModifiableTable;
+import org.apache.calcite.schema.ModifiableView;
+import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.TranslatableTable;
import org.apache.calcite.sql.JoinConditionType;
import org.apache.calcite.sql.JoinType;
@@ -101,6 +103,7 @@ import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlMerge;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlNumericLiteral;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.SqlSampleSpec;
@@ -109,6 +112,7 @@ import org.apache.calcite.sql.SqlSelectKeyword;
import org.apache.calcite.sql.SqlSetOperator;
import org.apache.calcite.sql.SqlUpdate;
import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlValuesOperator;
import org.apache.calcite.sql.SqlWindow;
import org.apache.calcite.sql.SqlWith;
import org.apache.calcite.sql.SqlWithItem;
@@ -146,9 +150,9 @@ import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mappings;
import org.apache.calcite.util.trace.CalciteTrace;
-import org.apache.calcite.sql.SqlNumericLiteral;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@@ -189,12 +193,6 @@ public class SqlToRelConverter {
protected static final Logger SQL2REL_LOGGER =
CalciteTrace.getSqlToRelTracer();
- private static final Function<SubQuery, SqlNode> FN =
- new Function<SubQuery, SqlNode>() {
- public SqlNode apply(SubQuery input) {
- return input.node;
- }
- };
private static final BigDecimal TWO = BigDecimal.valueOf(2L);
//~ Instance fields --------------------------------------------------------
@@ -205,9 +203,8 @@ public class SqlToRelConverter {
protected final RelOptCluster cluster;
private DefaultValueFactory defaultValueFactory;
private SubqueryConverter subqueryConverter;
- protected final List<RelNode> leaves = new ArrayList<RelNode>();
- private final List<SqlDynamicParam> dynamicParamSqlNodes =
- new ArrayList<SqlDynamicParam>();
+ protected final List<RelNode> leaves = new ArrayList<>();
+ private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
private final SqlOperatorTable opTab;
private boolean shouldConvertTableAccess;
protected final RelDataTypeFactory typeFactory;
@@ -222,7 +219,7 @@ public class SqlToRelConverter {
* Fields used in name resolution for correlated subqueries.
*/
private final Map<String, DeferredLookup> mapCorrelToDeferred =
- new HashMap<String, DeferredLookup>();
+ new HashMap<>();
private int nextCorrel = 0;
private static final String CORREL_PREFIX = "$cor";
@@ -231,7 +228,7 @@ public class SqlToRelConverter {
* Stack of names of datasets requested by the <code>
* TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
*/
- private final Stack<String> datasetStack = new Stack<String>();
+ private final Stack<String> datasetStack = new Stack<>();
/**
* Mapping of non-correlated subqueries that have been converted to their
@@ -239,7 +236,7 @@ public class SqlToRelConverter {
* already been evaluated.
*/
private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
- new HashMap<SqlNode, RexNode>();
+ new HashMap<>();
public final RelOptTable.ViewExpander viewExpander;
@@ -254,6 +251,7 @@ public class SqlToRelConverter {
* @param rexBuilder Rex builder
* @param convertletTable Expression converter
*/
+ @Deprecated // will be removed before 2.0
public SqlToRelConverter(
RelOptTable.ViewExpander viewExpander,
SqlValidator validator,
@@ -261,6 +259,17 @@ public class SqlToRelConverter {
RelOptPlanner planner,
RexBuilder rexBuilder,
SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader,
+ RelOptCluster.create(planner, rexBuilder), convertletTable);
+ }
+
+ /* Creates a converter. */
+ public SqlToRelConverter(
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable) {
this.viewExpander = viewExpander;
this.opTab =
(validator
@@ -270,10 +279,9 @@ public class SqlToRelConverter {
this.catalogReader = catalogReader;
this.defaultValueFactory = new NullDefaultValueFactory();
this.subqueryConverter = new NoOpSubqueryConverter();
- this.rexBuilder = rexBuilder;
+ this.rexBuilder = cluster.getRexBuilder();
this.typeFactory = rexBuilder.getTypeFactory();
- RelOptQuery query = new RelOptQuery(planner);
- this.cluster = query.createCluster(typeFactory, rexBuilder);
+ this.cluster = Preconditions.checkNotNull(cluster);
this.shouldConvertTableAccess = true;
this.exprConverter =
new SqlNodeToRexConverterImpl(convertletTable);
@@ -532,6 +540,9 @@ public class SqlToRelConverter {
}
RelNode result = convertQueryRecursive(query, top, null);
+ if (top && isStream(query)) {
+ result = new LogicalDelta(cluster, result.getTraitSet(), result);
+ }
checkConvertedType(query, result);
boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
@@ -547,6 +558,11 @@ public class SqlToRelConverter {
return result;
}
+ private static boolean isStream(SqlNode query) {
+ return query instanceof SqlSelect
+ && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ }
+
protected boolean checkConvertedRowType(
SqlNode query,
RelDataType convertedRowType) {
@@ -554,11 +570,7 @@ public class SqlToRelConverter {
validatedRowType = uniquifyFields(validatedRowType);
return RelOptUtil.equal(
- "validated row type",
- validatedRowType,
- "converted row type",
- convertedRowType,
- false);
+ "validated row type", validatedRowType, "converted row type", convertedRowType, false);
}
protected RelDataType uniquifyFields(RelDataType rowType) {
@@ -600,9 +612,8 @@ public class SqlToRelConverter {
bb,
select.getWhere());
- List<SqlNode> orderExprList = new ArrayList<SqlNode>();
- List<RelFieldCollation> collationList =
- new ArrayList<RelFieldCollation>();
+ final List<SqlNode> orderExprList = new ArrayList<>();
+ final List<RelFieldCollation> collationList = new ArrayList<>();
gatherOrderExprs(
bb,
select,
@@ -610,7 +621,7 @@ public class SqlToRelConverter {
orderExprList,
collationList);
final RelCollation collation =
- cluster.traitSetOf().canonize(RelCollationImpl.of(collationList));
+ cluster.traitSet().canonize(RelCollations.of(collationList));
if (validator.isAggregate(select)) {
convertAgg(
@@ -656,7 +667,7 @@ public class SqlToRelConverter {
if (checkForDupExprs && (rel instanceof LogicalProject)) {
LogicalProject project = (LogicalProject) rel;
final List<RexNode> projectExprs = project.getProjects();
- List<Integer> origins = new ArrayList<Integer>();
+ final List<Integer> origins = new ArrayList<>();
int dupCount = 0;
for (int i = 0; i < projectExprs.size(); i++) {
int x = findExpr(projectExprs.get(i), projectExprs, i);
@@ -682,13 +693,8 @@ public class SqlToRelConverter {
}
}
rel =
- new LogicalProject(
- cluster,
- rel,
- Pair.left(newProjects),
- Pair.right(newProjects),
- LogicalProject.Flags.BOXED);
-
+ LogicalProject.create(rel, Pair.left(newProjects),
+ Pair.right(newProjects));
bb.root = rel;
distinctify(bb, false);
rel = bb.root;
@@ -707,13 +713,8 @@ public class SqlToRelConverter {
}
rel =
- new LogicalProject(
- cluster,
- rel,
- Pair.left(undoProjects),
- Pair.right(undoProjects),
- LogicalProject.Flags.BOXED);
-
+ LogicalProject.create(rel, Pair.left(undoProjects),
+ Pair.right(undoProjects));
bb.setRoot(
rel,
false);
@@ -772,11 +773,7 @@ public class SqlToRelConverter {
// Create a sorter using the previously constructed collations.
bb.setRoot(
- new Sort(
- cluster,
- cluster.traitSetOf(Convention.NONE, collation),
- bb.root,
- collation,
+ LogicalSort.create(bb.root, collation,
offset == null ? null : convertExpression(offset),
fetch == null ? null : convertExpression(fetch)),
false);
@@ -784,7 +781,7 @@ public class SqlToRelConverter {
// If extra expressions were added to the project list for sorting,
// add another project to remove them.
if (orderExprList.size() > 0) {
- List<RexNode> exprs = new ArrayList<RexNode>();
+ final List<RexNode> exprs = new ArrayList<>();
final RelDataType rowType = bb.root.getRowType();
final int fieldCount =
rowType.getFieldCount() - orderExprList.size();
@@ -794,12 +791,11 @@ public class SqlToRelConverter {
bb.setRoot(
new LogicalProject(
cluster,
- cluster.traitSetOf(RelCollationImpl.PRESERVE),
+ cluster.traitSetOf(RelCollations.PRESERVE),
bb.root,
exprs,
cluster.getTypeFactory().createStructType(
- rowType.getFieldList().subList(0, fieldCount)),
- Project.Flags.BOXED),
+ rowType.getFieldList().subList(0, fieldCount))),
false);
}
}
@@ -1045,16 +1041,17 @@ public class SqlToRelConverter {
final int keyCount = leftKeys.size();
final List<Integer> args = ImmutableIntList.range(0, keyCount);
LogicalAggregate aggregate =
- new LogicalAggregate(cluster, seek, false, ImmutableBitSet.of(),
- null,
+ LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
ImmutableList.of(
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- ImmutableList.<Integer>of(), longType, null),
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- args, longType, null)));
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ ImmutableList.<Integer>of(), -1, longType, null),
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ args, -1, longType, null)));
LogicalJoin join =
- new LogicalJoin(cluster, bb.root, aggregate,
- rexBuilder.makeLiteral(true), JoinRelType.INNER,
+ LogicalJoin.create(bb.root,
+ aggregate,
+ rexBuilder.makeLiteral(true),
+ JoinRelType.INNER,
ImmutableSet.<String>of());
bb.setRoot(join, false);
}
@@ -1169,7 +1166,7 @@ public class SqlToRelConverter {
// on e.deptno = dt.deptno
final Join join = (Join) root;
final Project left = (Project) join.getLeft();
- final RelNode leftLeft = ((Join) left.getInput(0)).getLeft();
+ final RelNode leftLeft = ((Join) left.getInput()).getLeft();
final int leftLeftCount = leftLeft.getRowType().getFieldCount();
final RelDataType nullableBooleanType =
typeFactory.createTypeWithNullability(
@@ -1286,12 +1283,31 @@ public class SqlToRelConverter {
SqlNode selectExpr = selectList.get(0);
if (selectExpr instanceof SqlCall) {
SqlCall selectExprCall = (SqlCall) selectExpr;
- if (selectExprCall.getOperator()
- instanceof SqlAggFunction) {
+ if (Util.isSingleValue(selectExprCall)) {
+ return plan;
+ }
+ }
+
+ // If there is a limit with 0 or 1,
+ // it is ensured to produce a single value
+ if (select.getFetch() != null
+ && select.getFetch() instanceof SqlNumericLiteral) {
+ SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
+ if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
return plan;
}
}
}
+ } else if (query instanceof SqlCall) {
+ // If the query is (values ...),
+ // it is necessary to look into the operands to determine
+ // whether SingleValueAgg is necessary
+ SqlCall exprCall = (SqlCall) query;
+ if (exprCall.getOperator()
+ instanceof SqlValuesOperator
+ && Util.isSingleValue(exprCall)) {
+ return plan;
+ }
}
// If not, project SingleValueAgg
@@ -1313,7 +1329,7 @@ public class SqlToRelConverter {
final List<RexNode> leftKeys,
SqlNodeList valuesList,
boolean isNotIn) {
- List<RexNode> comparisons = new ArrayList<RexNode>();
+ final List<RexNode> comparisons = new ArrayList<>();
for (SqlNode rightVals : valuesList) {
RexNode rexComparison;
if (leftKeys.size() == 1) {
@@ -1321,7 +1337,8 @@ public class SqlToRelConverter {
rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS,
leftKeys.get(0),
- bb.convertExpression(rightVals));
+ rexBuilder.ensureType(leftKeys.get(0).getType(),
+ bb.convertExpression(rightVals), true));
} else {
assert rightVals instanceof SqlCall;
final SqlBasicCall call = (SqlBasicCall) rightVals;
@@ -1335,7 +1352,9 @@ public class SqlToRelConverter {
new Function<Pair<RexNode, SqlNode>, RexNode>() {
public RexNode apply(Pair<RexNode, SqlNode> pair) {
return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- pair.left, bb.convertExpression(pair.right));
+ pair.left,
+ rexBuilder.ensureType(pair.left.getType(),
+ bb.convertExpression(pair.right), true));
}
}),
false);
@@ -1368,8 +1387,8 @@ public class SqlToRelConverter {
* @return threshold, default 20
*/
protected int getInSubqueryThreshold() {
- // OVERRIDE POINT
- return Integer.MAX_VALUE; // was 20
+ /* OVERRIDE POINT */
+ return Integer.MAX_VALUE;
}
/**
@@ -1451,7 +1470,7 @@ public class SqlToRelConverter {
null);
}
- List<RelNode> unionInputs = new ArrayList<RelNode>();
+ final List<RelNode> unionInputs = new ArrayList<>();
for (SqlNode node : rows) {
SqlBasicCall call;
if (isRowConstructor(node)) {
@@ -1503,10 +1522,7 @@ public class SqlToRelConverter {
unionInputs.add(convertRowConstructor(bb, call));
}
LogicalValues values =
- new LogicalValues(
- cluster,
- rowType,
- tupleList.build());
+ LogicalValues.create(cluster, rowType, tupleList.build());
RelNode resultRel;
if (unionInputs.isEmpty()) {
resultRel = values;
@@ -1514,12 +1530,7 @@ public class SqlToRelConverter {
if (!values.getTuples().isEmpty()) {
unionInputs.add(values);
}
- LogicalUnion union =
- new LogicalUnion(
- cluster,
- unionInputs,
- true);
- resultRel = union;
+ resultRel = LogicalUnion.create(unionInputs, true);
}
leaves.add(resultRel);
return resultRel;
@@ -1710,8 +1721,7 @@ public class SqlToRelConverter {
public RexNode convertExpression(
SqlNode node,
Map<String, RexNode> nameToNodeMap) {
- final Map<String, RelDataType> nameToTypeMap =
- new HashMap<String, RelDataType>();
+ final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
}
@@ -1878,7 +1888,7 @@ public class SqlToRelConverter {
if (shouldConvertTableAccess) {
tableRel = toRel(table);
} else {
- tableRel = new LogicalTableScan(cluster, table);
+ tableRel = LogicalTableScan.create(cluster, table);
}
bb.setRoot(tableRel, true);
if (usedDataset[0]) {
@@ -2020,7 +2030,7 @@ public class SqlToRelConverter {
Set<RelColumnMapping> columnMappings =
getColumnMappings(operator);
LogicalTableFunctionScan callRel =
- new LogicalTableFunctionScan(
+ LogicalTableFunctionScan.create(
cluster,
inputs,
rexCall,
@@ -2161,12 +2171,8 @@ public class SqlToRelConverter {
ImmutableSet.copyOf(Util.skip(correlNames)));
rightRel = rightRel.accept(dedup);
}
- LogicalCorrelate corr = new LogicalCorrelate(
- rightRel.getCluster(),
- leftRel,
- rightRel,
- new CorrelationId(correlNames.get(0)),
- requiredColumns.build(),
+ LogicalCorrelate corr = LogicalCorrelate.create(leftRel, rightRel,
+ new CorrelationId(correlNames.get(0)), requiredColumns.build(),
SemiJoinType.of(joinType));
if (!joinCond.isAlwaysTrue()) {
return RelOptUtil.createFilter(corr, joinCond);
@@ -2175,8 +2181,8 @@ public class SqlToRelConverter {
}
}
- final List<RexNode> extraLeftExprs = new ArrayList<RexNode>();
- final List<RexNode> extraRightExprs = new ArrayList<RexNode>();
+ final List<RexNode> extraLeftExprs = new ArrayList<>();
+ final List<RexNode> extraRightExprs = new ArrayList<>();
final int leftCount = leftRel.getRowType().getFieldCount();
final int rightCount = rightRel.getRowType().getFieldCount();
if (!containsGet(joinCond)) {
@@ -2200,8 +2206,7 @@ public class SqlToRelConverter {
new RexInputRef(index, field.getType()),
field.getName());
} else {
- return Pair.<RexNode, String>of(
- extraLeftExprs.get(index - leftCount), null);
+ return Pair.of(extraLeftExprs.get(index - leftCount), null);
}
}
},
@@ -2248,7 +2253,7 @@ public class SqlToRelConverter {
+ rightCount + extraRightExprs.size(),
0, 0, leftCount,
leftCount, leftCount + extraLeftExprs.size(), rightCount);
- return RelOptUtil.project(join, mapping);
+ return RelOptUtil.createProject(join, mapping);
}
return join;
}
@@ -2287,8 +2292,8 @@ public class SqlToRelConverter {
case AND:
case OR:
case EQUALS:
- RexCall call = (RexCall) node;
- List<RexNode> list = new ArrayList<RexNode>();
+ final RexCall call = (RexCall) node;
+ final List<RexNode> list = new ArrayList<>();
List<RexNode> operands = Lists.newArrayList(call.getOperands());
for (int i = 0; i < operands.size(); i++) {
RexNode operand = operands.get(i);
@@ -2439,8 +2444,8 @@ public class SqlToRelConverter {
bb.setRoot(ImmutableList.of(leftRel, rightRel));
return bb.convertExpression(condition);
case USING:
- SqlNodeList list = (SqlNodeList) condition;
- List<String> nameList = new ArrayList<String>();
+ final SqlNodeList list = (SqlNodeList) condition;
+ final List<String> nameList = new ArrayList<>();
for (SqlNode columnName : list) {
final SqlIdentifier id = (SqlIdentifier) columnName;
String name = id.getSimple();
@@ -2547,17 +2552,16 @@ public class SqlToRelConverter {
SqlNodeList groupList,
SqlNode having,
List<SqlNode> orderExprList) {
- SqlNodeList aggList = new SqlNodeList(SqlParserPos.ZERO);
-
- for (SqlNode selectNode : selectList) {
- if (validator.isAggregate(selectNode)) {
- aggList.add(selectNode);
- }
+ // Find aggregate functions in SELECT and HAVING clause
+ final AggregateFinder aggregateFinder = new AggregateFinder();
+ selectList.accept(aggregateFinder);
+ if (having != null) {
+ having.accept(aggregateFinder);
}
// first replace the subqueries inside the aggregates
// because they will provide input rows to the aggregates.
- replaceSubqueries(bb, aggList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ replaceSubqueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
// If group-by clause is missing, pretend that it has zero elements.
if (groupList == null) {
@@ -2704,6 +2708,7 @@ public class SqlToRelConverter {
// (yet) appear in the validator type.
final SelectScope selectScope =
SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
+ assert selectScope != null;
final SqlValidatorNamespace selectNamespace =
validator.getNamespace(selectScope.getNode());
final List<String> names =
@@ -2769,13 +2774,8 @@ public class SqlToRelConverter {
protected RelNode createAggregate(Blackboard bb, boolean indicator,
ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls) {
- return new LogicalAggregate(
- cluster,
- bb.root,
- indicator,
- groupSet,
- groupSets,
- aggCalls);
+ return LogicalAggregate.create(
+ bb.root, indicator, groupSet, groupSets, aggCalls);
}
public RexDynamicParam convertDynamicParam(
@@ -2917,8 +2917,8 @@ public class SqlToRelConverter {
* @return Whether to trim unused fields
*/
public boolean isTrimUnusedFields() {
- // OVERRIDE POINT
- return false; // was `trimUnusedFields`
+ /* OVERRIDE POINT */
+ return false;
}
/**
@@ -2974,18 +2974,12 @@ public class SqlToRelConverter {
}
switch (call.getKind()) {
case UNION:
- return new LogicalUnion(
- cluster,
- ImmutableList.of(left, right),
- all);
+ return LogicalUnion.create(ImmutableList.of(left, right), all);
case INTERSECT:
// TODO: all
if (!all) {
- return new LogicalIntersect(
- cluster,
- ImmutableList.of(left, right),
- all);
+ return LogicalIntersect.create(ImmutableList.of(left, right), all);
} else {
throw Util.newInternal(
"set operator INTERSECT ALL not suported");
@@ -2994,10 +2988,7 @@ public class SqlToRelConverter {
case EXCEPT:
// TODO: all
if (!all) {
- return new LogicalMinus(
- cluster,
- ImmutableList.of(left, right),
- all);
+ return LogicalMinus.create(ImmutableList.of(left, right), all);
} else {
throw Util.newInternal(
"set operator EXCEPT ALL not suported");
@@ -3016,31 +3007,86 @@ public class SqlToRelConverter {
assert targetRowType != null;
RelNode sourceRel =
convertQueryRecursive(
- call.getSource(),
- false,
- targetRowType);
+ call.getSource(), false, targetRowType);
RelNode massagedRel = convertColumnList(call, sourceRel);
+ return createModify(targetTable, massagedRel);
+ }
+
+ /** Creates a relational expression to modify a table or modifiable view. */
+ private RelNode createModify(RelOptTable targetTable, RelNode source) {
final ModifiableTable modifiableTable =
targetTable.unwrap(ModifiableTable.class);
if (modifiableTable != null) {
- return modifiableTable.toModificationRel(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- LogicalTableModify.Operation.INSERT,
- null,
+ return modifiableTable.toModificationRel(cluster, targetTable,
+ catalogReader, source, LogicalTableModify.Operation.INSERT, null,
false);
}
- return new LogicalTableModify(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- LogicalTableModify.Operation.INSERT,
- null,
- false);
+ final ModifiableView modifiableView =
+ targetTable.unwrap(ModifiableView.class);
+ if (modifiableView != null) {
+ final Table delegateTable = modifiableView.getTable();
+ final RelDataType delegateRowType = delegateTable.getRowType(typeFactory);
+ final RelOptTable delegateRelOptTable =
+ RelOptTableImpl.create(null, delegateRowType, delegateTable,
+ modifiableView.getTablePath());
+ final RelNode newSource =
+ createSource(targetTable, source, modifiableView, delegateRowType);
+ return createModify(delegateRelOptTable, newSource);
+ }
+ return LogicalTableModify.create(targetTable, catalogReader, source,
+ LogicalTableModify.Operation.INSERT, null, false);
+ }
+
+ /** Wraps a relational expression in the projects and filters implied by
+ * a {@link ModifiableView}.
+ *
+ * <p>The input relational expression is suitable for inserting into the view,
+ * and the returned relational expression is suitable for inserting into its
+ * delegate table.
+ *
+ * <p>In principle, the delegate table of a view might be another modifiable
+ * view, and if so, the process can be repeated. */
+ private RelNode createSource(RelOptTable targetTable, RelNode source,
+ ModifiableView modifiableView, RelDataType delegateRowType) {
+ final ImmutableIntList mapping = modifiableView.getColumnMapping();
+ assert mapping.size() == targetTable.getRowType().getFieldCount();
+
+ // For columns represented in the mapping, the expression is just a field
+ // reference.
+ final Map<Integer, RexNode> projectMap = new HashMap<>();
+ final List<RexNode> filters = new ArrayList<>();
+ for (int i = 0; i < mapping.size(); i++) {
+ int target = mapping.get(i);
+ if (target >= 0) {
+ projectMap.put(target, RexInputRef.of(i, source.getRowType()));
+ }
+ }
+
+ // For columns that are not in the mapping, and have a constraint of the
+ // form "column = value", the expression is the literal "value".
+ //
+ // If a column has multiple constraints, the extra ones will become a
+ // filter.
+ final RexNode constraint =
+ modifiableView.getConstraint(rexBuilder, delegateRowType);
+ RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
+ final List<Pair<RexNode, String>> projects = new ArrayList<>();
+ for (RelDataTypeField field : delegateRowType.getFieldList()) {
+ RexNode node = projectMap.get(field.getIndex());
+ if (node == null) {
+ node = rexBuilder.makeNullLiteral(field.getType().getSqlTypeName());
+ }
+ projects.add(
+ Pair.of(rexBuilder.ensureType(field.getType(), node, false),
+ field.getName()));
+ }
+
+ source = RelOptUtil.createProject(source, projects, true);
+ if (filters.size() > 0) {
+ source = RelOptUtil.createFilter(source, filters);
+ }
+ return source;
}
private RelOptTable.ToRelContext createToRelContext() {
@@ -3088,8 +3134,8 @@ public class SqlToRelConverter {
RelDataType sourceRowType = sourceRel.getRowType();
final RexNode sourceRef =
rexBuilder.makeRangeReference(sourceRowType, 0, false);
- final List<String> targetColumnNames = new ArrayList<String>();
- final List<RexNode> columnExprs = new ArrayList<RexNode>();
+ final List<String> targetColumnNames = new ArrayList<>();
+ final List<RexNode> columnExprs = new ArrayList<>();
collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
final RelOptTable targetTable = getTargetTable(call);
@@ -3097,10 +3143,10 @@ public class SqlToRelConverter {
final List<RelDataTypeField> targetFields =
targetRowType.getFieldList();
final List<RexNode> sourceExps =
- new ArrayList<RexNode>(
+ new ArrayList<>(
Collections.<RexNode>nCopies(targetFields.size(), null));
final List<String> fieldNames =
- new ArrayList<String>(
+ new ArrayList<>(
Collections.<String>nCopies(targetFields.size(), null));
// Walk the name list and place the associated value in the
@@ -3181,21 +3227,15 @@ public class SqlToRelConverter {
private RelNode convertDelete(SqlDelete call) {
RelOptTable targetTable = getTargetTable(call);
RelNode sourceRel = convertSelect(call.getSourceSelect());
- return new LogicalTableModify(
- cluster,
- targetTable,
- catalogReader,
- sourceRel,
- LogicalTableModify.Operation.DELETE,
- null,
- false);
+ return LogicalTableModify.create(targetTable, catalogReader, sourceRel,
+ LogicalTableModify.Operation.DELETE, null, false);
}
private RelNode convertUpdate(SqlUpdate call) {
RelOptTable targetTable = getTargetTable(call);
// convert update column list from SqlIdentifier to String
- List<String> targetColumnNameList = new ArrayList<String>();
+ final List<String> targetColumnNameList = new ArrayList<>();
for (SqlNode node : call.getTargetColumnList()) {
SqlIdentifier id = (SqlIdentifier) node;
String name = id.getSimple();
@@ -3204,21 +3244,15 @@ public class SqlToRelConverter {
RelNode sourceRel = convertSelect(call.getSourceSelect());
- return new LogicalTableModify(
- cluster,
- targetTable,
- catalogReader,
- sourceRel,
- LogicalTableModify.Operation.UPDATE,
- targetColumnNameList,
- false);
+ return LogicalTableModify.create(targetTable, catalogReader, sourceRel,
+ LogicalTableModify.Operation.UPDATE, targetColumnNameList, false);
}
private RelNode convertMerge(SqlMerge call) {
RelOptTable targetTable = getTargetTable(call);
// convert update column list from SqlIdentifier to String
- List<String> targetColumnNameList = new ArrayList<String>();
+ final List<String> targetColumnNameList = new ArrayList<>();
SqlUpdate updateCall = call.getUpdateCall();
if (updateCall != null) {
for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
@@ -3267,7 +3301,7 @@ public class SqlToRelConverter {
LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0);
int nSourceFields = join.getLeft().getRowType().getFieldCount();
- List<RexNode> projects = new ArrayList<RexNode>();
+ final List<RexNode> projects = new ArrayList<>();
for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
if ((level2InsertExprs != null)
&& (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
@@ -3287,14 +3321,8 @@ public class SqlToRelConverter {
RelNode massagedRel =
RelOptUtil.createProject(join, projects, null, true);
- return new LogicalTableModify(
- cluster,
- targetTable,
- catalogReader,
- massagedRel,
- LogicalTableModify.Operation.MERGE,
- targetColumnNameList,
- false);
+ return LogicalTableModify.create(targetTable, catalogReader, massagedRel,
+ LogicalTableModify.Operation.MERGE, targetColumnNameList, false);
}
/**
@@ -3399,8 +3427,8 @@ public class SqlToRelConverter {
Blackboard bb) {
// NOTE: Wael 2/04/05: this implementation is not the most efficient in
// terms of planning since it generates XOs that can be reduced.
- List<Object> joinList = new ArrayList<Object>();
- List<SqlNode> lastList = new ArrayList<SqlNode>();
+ final List<Object> joinList = new ArrayList<>();
+ List<SqlNode> lastList = new ArrayList<>();
for (int i = 0; i < operands.size(); i++) {
SqlNode operand = operands.get(i);
if (!(operand instanceof SqlCall)) {
@@ -3447,7 +3475,7 @@ public class SqlToRelConverter {
if (lastList.size() > 0) {
joinList.add(lastList);
}
- lastList = new ArrayList<SqlNode>();
+ lastList = new ArrayList<>();
Collect collect =
new Collect(
cluster,
@@ -3465,8 +3493,8 @@ public class SqlToRelConverter {
Object o = joinList.get(i);
if (o instanceof List) {
List<SqlNode> projectList = (List<SqlNode>) o;
- final List<RexNode> selectList = new ArrayList<RexNode>();
- final List<String> fieldNameList = new ArrayList<String>();
+ final List<RexNode> selectList = new ArrayList<>();
+ final List<String> fieldNameList = new ArrayList<>();
for (int j = 0; j < projectList.size(); j++) {
SqlNode operand = projectList.get(j);
selectList.add(bb.convertExpression(operand));
@@ -3524,12 +3552,7 @@ public class SqlToRelConverter {
RexNode condition,
JoinRelType joinType,
Set<String> variablesStopped) {
- return new LogicalJoin(
- cluster,
- left,
- right,
- condition,
- joinType,
+ return LogicalJoin.create(left, right, condition, joinType,
variablesStopped);
}
@@ -3542,14 +3565,13 @@ public class SqlToRelConverter {
replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- List<String> fieldNames = new ArrayList<String>();
- List<RexNode> exprs = new ArrayList<RexNode>();
- Collection<String> aliases = new TreeSet<String>();
+ List<String> fieldNames = new ArrayList<>();
+ final List<RexNode> exprs = new ArrayList<>();
+ final Collection<String> aliases = new TreeSet<>();
// Project any system fields. (Must be done before regular select items,
// because offsets may be affected.)
- final List<SqlMonotonicity> columnMonotonicityList =
- new ArrayList<SqlMonotonicity>();
+ final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
extraSelectItems(
bb,
select,
@@ -3675,14 +3697,13 @@ public class SqlToRelConverter {
return;
}
- List<RelNode> unionRels = new ArrayList<RelNode>();
+ final List<RelNode> unionRels = new ArrayList<>();
for (SqlNode rowConstructor1 : values.getOperandList()) {
SqlCall rowConstructor = (SqlCall) rowConstructor1;
Blackboard tmpBb = createBlackboard(bb.scope, null);
replaceSubqueries(tmpBb, rowConstructor,
RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- List<Pair<RexNode, String>> exps =
- new ArrayList<Pair<RexNode, String>>();
+ final List<Pair<RexNode, String>> exps = new ArrayList<>();
for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
exps.add(
Pair.of(
@@ -3709,10 +3730,7 @@ public class SqlToRelConverter {
true);
} else {
bb.setRoot(
- new LogicalUnion(
- cluster,
- unionRels,
- true),
+ LogicalUnion.create(unionRels, true),
true);
}
@@ -3746,7 +3764,7 @@ public class SqlToRelConverter {
public RelNode root;
private List<RelNode> inputs;
private final Map<String, RexNode> mapCorrelateVariableToRexNode =
- new HashMap<String, RexNode>();
+ new HashMap<>();
List<RelNode> cursors;
@@ -3756,9 +3774,6 @@ public class SqlToRelConverter {
*/
private final Set<SubQuery> subqueryList = Sets.newLinkedHashSet();
- private final Map<SqlNode, SubQuery> subqueryMap =
- Util.asIndexMap(subqueryList, FN);
-
private boolean subqueryNeedsOuterJoin;
/**
@@ -3778,14 +3793,12 @@ public class SqlToRelConverter {
* "right" to the subquery.
*/
private final Map<RelNode, Map<Integer, Integer>>
- mapRootRelToFieldProjection =
- new HashMap<RelNode, Map<Integer, Integer>>();
+ mapRootRelToFieldProjection = new HashMap<>();
private final List<SqlMonotonicity> columnMonotonicities =
- new ArrayList<SqlMonotonicity>();
+ new ArrayList<>();
- private final List<RelDataTypeField> systemFieldList =
- new ArrayList<RelDataTypeField>();
+ private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
/**
* Creates a Blackboard.
@@ -3802,7 +3815,7 @@ public class SqlToRelConverter {
Map<String, RexNode> nameToNodeMap) {
this.scope = scope;
this.nameToNodeMap = nameToNodeMap;
- this.cursors = new ArrayList<RelNode>();
+ this.cursors = new ArrayList<>();
subqueryNeedsOuterJoin = false;
}
@@ -4078,9 +4091,24 @@ public class SqlToRelConverter {
}
void registerSubquery(SqlNode node, RelOptUtil.Logic logic) {
+ for (SubQuery subQuery : subqueryList) {
+ if (node.equalsDeep(subQuery.node, false)) {
+ return;
+ }
+ }
subqueryList.add(new SubQuery(node, logic));
}
+ SubQuery getSubquery(SqlNode expr) {
+ for (SubQuery subQuery : subqueryList) {
+ if (expr.equalsDeep(subQuery.node, false)) {
+ return subQuery;
+ }
+ }
+
+ return null;
+ }
+
ImmutableList<RelNode> retrieveCursors() {
try {
return ImmutableList.copyOf(cursors);
@@ -4113,26 +4141,28 @@ public class SqlToRelConverter {
return rex;
}
- boolean needTruthTest;
-
// Sub-queries and OVER expressions are not like ordinary
// expressions.
final SqlKind kind = expr.getKind();
final SubQuery subQuery;
switch (kind) {
case CURSOR:
+ case IN:
+ subQuery = getSubquery(expr);
+
+ assert subQuery != null;
+ rex = subQuery.expr;
+ assert rex != null : "rex != null";
+ return rex;
+
case SELECT:
case EXISTS:
case SCALAR_QUERY:
- subQuery = subqueryMap.get(expr);
+ subQuery = getSubquery(expr);
assert subQuery != null;
rex = subQuery.expr;
assert rex != null : "rex != null";
- if (kind == SqlKind.CURSOR) {
- // cursor reference is pre-baked
- return rex;
- }
if (((kind == SqlKind.SCALAR_QUERY)
|| (kind == SqlKind.EXISTS))
&& isConvertedSubq(rex)) {
@@ -4141,11 +4171,8 @@ public class SqlToRelConverter {
return rex;
}
- RexNode fieldAccess;
- needTruthTest = false;
-
// The indicator column is the last field of the subquery.
- fieldAccess =
+ RexNode fieldAccess =
rexBuilder.makeFieldAccess(
rex,
rex.getType().getFieldCount() - 1);
@@ -4153,13 +4180,8 @@ public class SqlToRelConverter {
// The indicator column will be nullable if it comes from
// the null-generating side of the join. For EXISTS, add an
// "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
- if (fieldAccess.getType().isNullable()) {
- if (kind == SqlKind.EXISTS) {
- needTruthTest = true;
- }
- }
-
- if (needTruthTest) {
+ if (fieldAccess.getType().isNullable()
+ && kind == SqlKind.EXISTS) {
fieldAccess =
rexBuilder.makeCall(
SqlStdOperatorTable.IS_NOT_NULL,
@@ -4167,12 +4189,6 @@ public class SqlToRelConverter {
}
return fieldAccess;
- case IN:
- subQuery = subqueryMap.get(expr);
- assert subQuery != null;
- assert subQuery.expr != null : "expr != null";
- return subQuery.expr;
-
case OVER:
return convertOver(this, expr);
@@ -4246,7 +4262,7 @@ public class SqlToRelConverter {
// implement SqlRexContext
public RexRangeRef getSubqueryExpr(SqlCall call) {
- final SubQuery subQuery = subqueryMap.get(call);
+ final SubQuery subQuery = getSubquery(call);
assert subQuery != null;
return (RexRangeRef) subQuery.expr;
}
@@ -4284,7 +4300,7 @@ public class SqlToRelConverter {
public RexNode visit(SqlCall call) {
if (agg != null) {
final SqlOperator op = call.getOperator();
- if (op.isAggregator()) {
+ if (op.isAggregator() || op.getKind() == SqlKind.FILTER) {
return agg.lookupAggregates(call);
}
}
@@ -4537,81 +4553,97 @@ public class SqlToRelConverter {
}
public Void visit(SqlCall call) {
- if (call.getOperator().isAggregator()) {
- assert bb.agg == this;
- List<Integer> args = new ArrayList<Integer>();
- List<RelDataType> argTypes =
- call.getOperator() instanceof SqlCountAggFunction
- ? new ArrayList<RelDataType>(call.getOperandList().size())
- : null;
- try {
- // switch out of agg mode
- bb.agg = null;
- for (SqlNode operand : call.getOperandList()) {
- RexNode convertedExpr;
-
- // special case for COUNT(*): delete the *
- if (operand instanceof SqlIdentifier) {
- SqlIdentifier id = (SqlIdentifier) operand;
- if (id.isStar() || isSimpleCount(call)) { // OVERRIDE POINT, was just `id.isStar()`
- assert call.operandCount() == 1;
- assert args.isEmpty();
- break;
- }
- }
- convertedExpr = bb.convertExpression(operand);
- assert convertedExpr != null;
- if (argTypes != null) {
- argTypes.add(convertedExpr.getType());
- }
- args.add(lookupOrCreateGroupExpr(convertedExpr));
- }
- } finally {
- // switch back into agg mode
- bb.agg = this;
- }
-
- final SqlAggFunction aggFunction =
- (SqlAggFunction) call.getOperator();
- RelDataType type = validator.deriveType(bb.scope, call);
- boolean distinct = false;
- SqlLiteral quantifier = call.getFunctionQuantifier();
- if ((null != quantifier)
- && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
- distinct = true;
- }
- final AggregateCall aggCall =
- new AggregateCall(
- aggFunction,
- distinct,
- args,
- type,
- nameMap.get(call.toString()));
- RexNode rex =
- rexBuilder.addAggCall(
- aggCall,
- groupExprs.size(),
- aggregatingSelectScope.indicator,
- aggCalls,
- aggCallMapping,
- argTypes);
- aggMapping.put(call, rex);
- } else if (call instanceof SqlSelect) {
+ switch (call.getKind()) {
+ case FILTER:
+ translateAgg((SqlCall) call.operand(0), call.operand(1), call);
+ return null;
+ case SELECT:
// rchen 2006-10-17:
// for now do not detect aggregates in subqueries.
return null;
- } else {
+ }
+ if (call.getOperator().isAggregator()) {
+ translateAgg(call, null, call);
+ return null;
+ }
+ for (SqlNode operand : call.getOperandList()) {
+ // Operands are occasionally null, e.g. switched CASE arg 0.
+ if (operand != null) {
+ operand.accept(this);
+ }
+ }
+ return null;
+ }
+
+ private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) {
+ assert bb.agg == this;
+ final List<Integer> args = new ArrayList<>();
+ int filterArg = -1;
+ final List<RelDataType> argTypes =
+ call.getOperator() instanceof SqlCountAggFunction
+ ? new ArrayList<RelDataType>(call.getOperandList().size())
+ : null;
+ try {
+ // switch out of agg mode
+ bb.agg = null;
for (SqlNode operand : call.getOperandList()) {
- // Operands are occasionally null, e.g. switched CASE arg 0.
- if (operand != null) {
- operand.accept(this);
+
+ // special case for COUNT(*): delete the *
+ if (operand instanceof SqlIdentifier) {
+ SqlIdentifier id = (SqlIdentifier) operand;
+ if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */
+ assert call.operandCount() == 1;
+ assert args.isEmpty();
+ break;
+ }
+ }
+ RexNode convertedExpr = bb.convertExpression(operand);
+ assert convertedExpr != null;
+ if (argTypes != null) {
+ argTypes.add(convertedExpr.getType());
}
+ args.add(lookupOrCreateGroupExpr(convertedExpr));
}
+
+ if (filter != null) {
+ RexNode convertedExpr = bb.convertExpression(filter);
+ assert convertedExpr != null;
+ filterArg = lookupOrCreateGroupExpr(convertedExpr);
+ }
+ } finally {
+ // switch back into agg mode
+ bb.agg = this;
}
- return null;
+
+ final SqlAggFunction aggFunction =
+ (SqlAggFunction) call.getOperator();
+ RelDataType type = validator.deriveType(bb.scope, call);
+ boolean distinct = false;
+ SqlLiteral quantifier = call.getFunctionQuantifier();
+ if ((null != quantifier)
+ && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
+ distinct = true;
+ }
+ final AggregateCall aggCall =
+ AggregateCall.create(
+ aggFunction,
+ distinct,
+ args,
+ filterArg,
+ type,
+ nameMap.get(outerCall.toString()));
+ RexNode rex =
+ rexBuilder.addAggCall(
+ aggCall,
+ groupExprs.size(),
+ aggregatingSelectScope.indicator,
+ aggCalls,
+ aggCallMapping,
+ argTypes);
+ aggMapping.put(outerCall, rex);
}
- // OVERRIDE POINT
+ /* OVERRIDE POINT */
private boolean isSimpleCount(SqlCall call) {
if (call.getOperator().isName("COUNT") && call.operandCount() == 1) {
final SqlNode parm = call.operand(0);
@@ -4622,7 +4654,7 @@ public class SqlToRelConverter {
}
return false;
}
-
+
private int lookupOrCreateGroupExpr(RexNode expr) {
for (int i = 0; i < convertedInputExprs.size(); i++) {
RexNode convertedInputExpr = convertedInputExprs.get(i);
@@ -4741,7 +4773,7 @@ public class SqlToRelConverter {
*/
private static class LookupContext {
private final List<Pair<RelNode, Integer>> relOffsetList =
- new ArrayList<Pair<RelNode, Integer>>();
+ new ArrayList<>();
/**
* Creates a LookupContext with multiple input relational expressions.
@@ -4848,7 +4880,7 @@ public class SqlToRelConverter {
// Replace original expression with CAST of not one
// of the supported types
if (histogramType != type) {
- exprs = new ArrayList<RexNode>(exprs);
+ exprs = new ArrayList<>(exprs);
exprs.set(
0,
reinterpretCast
@@ -4973,6 +5005,28 @@ public class SqlToRelConverter {
this.logic = logic;
}
}
+
+ /**
+ * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
+ */
+ private static class AggregateFinder extends SqlBasicVisitor<Void> {
+ final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
+
+ @Override public Void visit(SqlCall call) {
+ if (call.getOperator().isAggregator()) {
+ list.add(call);
+ return null;
+ }
+
+ // Don't traverse into sub-queries, even if they contain aggregate
+ // functions.
+ if (call instanceof SqlSelect) {
+ return null;
+ }
+
+ return call.getOperator().acceptCall(this, call);
+ }
+ }
}
// End SqlToRelConverter.java
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 4b73ad6..3d0007e 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -259,7 +259,7 @@ public class KylinClient implements IRemoteClient {
for (int i = 0; i < result.length; i++) {
ColumnMetaData meta = metas.get(i);
- row[i] = wrapObject(result[i], meta.type.type);
+ row[i] = wrapObject(result[i], meta.type.id);
}
data.add(row);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
index 030dbbb..f1a4939 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
@@ -30,6 +30,7 @@ import org.apache.calcite.avatica.AvaticaPreparedStatement;
import org.apache.calcite.avatica.AvaticaResultSet;
import org.apache.calcite.avatica.AvaticaResultSetMetaData;
import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta.Frame;
import org.apache.calcite.avatica.Meta.Signature;
import org.apache.calcite.avatica.Meta.StatementHandle;
import org.apache.calcite.avatica.UnregisteredDriver;
@@ -92,9 +93,9 @@ public class KylinJdbcFactory implements AvaticaFactory {
}
@Override
- public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Iterable<Object> iterable) throws SQLException {
+ public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException {
AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature);
- return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, iterable);
+ return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, firstFrame);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index 821fba5..433262f 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -31,6 +31,8 @@ import org.apache.calcite.avatica.AvaticaUtils;
import org.apache.calcite.avatica.ColumnMetaData;
import org.apache.calcite.avatica.MetaImpl;
+import com.google.common.collect.ImmutableList;
+
/**
* Implementation of Avatica interface
*/
@@ -48,26 +50,34 @@ public class KylinMeta extends MetaImpl {
// insert/update/delete go this path, ignorable for Kylin
@Override
- public Signature prepare(StatementHandle h, String sql, int maxRowCount) {
- return connection().mockPreparedSignature(sql);
+ public StatementHandle prepare(ConnectionHandle ch, String sql, int maxRowCount) {
+ StatementHandle result = super.createStatement(ch);
+ result.signature = connection().mockPreparedSignature(sql);
+ return result;
}
// mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute()
@Override
- public MetaResultSet prepareAndExecute(StatementHandle h, String sql, int maxRowCount, PrepareCallback callback) {
- final Signature signature;
+ public ExecuteResult prepareAndExecute(ConnectionHandle ch, String sql, int maxRowCount, PrepareCallback callback) {
+ final StatementHandle sh;
try {
synchronized (callback.getMonitor()) {
callback.clear();
- signature = prepare(h, sql, maxRowCount);
- callback.assign(signature, null);
+ sh = prepare(ch, sql, maxRowCount);
+ callback.assign(sh.signature, null, -1);
}
callback.execute();
- return new MetaResultSet(h.id, false, signature, null);
+ final MetaResultSet metaResultSet = MetaResultSet.create(ch.id, sh.id, false, sh.signature, null);
+ return new ExecuteResult(ImmutableList.of(metaResultSet));
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
+
+ @Override
+ public void closeStatement(StatementHandle h) {
+ // nothing to do
+ }
private KMetaProject getMetaProject() {
try {
@@ -164,9 +174,10 @@ public class KylinMeta extends MetaImpl {
CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames);
Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory);
- StatementHandle statementHandle = this.createStatement(null);
+ StatementHandle sh = this.createStatement(connection().handle);
+ Frame frame = new Frame(0, true, iterable);
- return new MetaResultSet(statementHandle.id, true, signature, iterable);
+ return MetaResultSet.create(connection().id, sh.id, true, signature, frame);
}
// ============================================================================
@@ -177,7 +188,7 @@ public class KylinMeta extends MetaImpl {
public static List<? extends NamedWithChildren> searchByPatterns(NamedWithChildren parent, Pat... patterns) {
assert patterns != null && patterns.length > 0;
-
+
List<? extends NamedWithChildren> children = findChildren(parent, patterns[0]);
if (patterns.length == 1) {
return children;
@@ -268,12 +279,12 @@ public class KylinMeta extends MetaImpl {
public List<KMetaTable> getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
return (List<KMetaTable>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern);
}
-
+
@SuppressWarnings("unchecked")
public List<KMetaColumn> getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
return (List<KMetaColumn>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern, columnNamePattern);
}
-
+
@Override
public String getName() {
return projectName;
@@ -293,7 +304,7 @@ public class KylinMeta extends MetaImpl {
this.tableCat = tableCatalog;
this.schemas = schemas;
}
-
+
@Override
public String getName() {
return tableCat;
@@ -344,4 +355,5 @@ public class KylinMeta extends MetaImpl {
return Collections.<NamedWithChildren> emptyList();
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
index 21cb162..b14865b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
@@ -6,7 +6,7 @@ import java.sql.NClob;
import java.sql.RowId;
import java.sql.SQLException;
import java.sql.SQLXML;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.avatica.AvaticaConnection;
@@ -19,82 +19,86 @@ public class KylinPreparedStatement extends AvaticaPreparedStatement {
protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
}
-
- protected List<Object> getParameterValues() {
- return Arrays.asList(slots);
+
+ protected List<Object> getParameterValues2() {
+ List<Object> values = new ArrayList<>(slots.length);
+ for (int i = 0; i < slots.length; i++) {
+ values.add(slots[i].value);
+ }
+ return values;
}
-
+
// ============================================================================
public void setRowId(int parameterIndex, RowId x) throws SQLException {
- getParameter(parameterIndex).setRowId(slots, parameterIndex, x);
+ getSite(parameterIndex).setRowId(x);
}
public void setNString(int parameterIndex, String value) throws SQLException {
- getParameter(parameterIndex).setNString(slots, parameterIndex, value);
+ getSite(parameterIndex).setNString(value);
}
public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
- getParameter(parameterIndex).setNCharacterStream(slots, parameterIndex, value, length);
+ getSite(parameterIndex).setNCharacterStream(value, length);
}
public void setNClob(int parameterIndex, NClob value) throws SQLException {
- getParameter(parameterIndex).setNClob(slots, parameterIndex, value);
+ getSite(parameterIndex).setNClob(value);
}
public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setClob(slots, parameterIndex, reader, length);
+ getSite(parameterIndex).setClob(reader, length);
}
public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
- getParameter(parameterIndex).setBlob(slots, parameterIndex, inputStream, length);
+ getSite(parameterIndex).setBlob(inputStream, length);
}
public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setNClob(slots, parameterIndex, reader, length);
+ getSite(parameterIndex).setNClob(reader, length);
}
public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
- getParameter(parameterIndex).setSQLXML(slots, parameterIndex, xmlObject);
+ getSite(parameterIndex).setSQLXML(xmlObject);
}
public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getParameter(parameterIndex).setAsciiStream(slots, parameterIndex, x, length);
+ getSite(parameterIndex).setAsciiStream(x, length);
}
public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getParameter(parameterIndex).setBinaryStream(slots, parameterIndex, x, length);
+ getSite(parameterIndex).setBinaryStream(x, length);
}
public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
- getParameter(parameterIndex).setCharacterStream(slots, parameterIndex, reader, length);
+ getSite(parameterIndex).setCharacterStream(reader, length);
}
public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
- getParameter(parameterIndex).setAsciiStream(slots, parameterIndex, x);
+ getSite(parameterIndex).setAsciiStream(x);
}
public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
- getParameter(parameterIndex).setBinaryStream(slots, parameterIndex, x);
+ getSite(parameterIndex).setBinaryStream(x);
}
public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setCharacterStream(slots, parameterIndex, reader);
+ getSite(parameterIndex).setCharacterStream(reader);
}
public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
- getParameter(parameterIndex).setNCharacterStream(slots, parameterIndex, value);
+ getSite(parameterIndex).setNCharacterStream(value);
}
public void setClob(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setClob(slots, parameterIndex, reader);
+ getSite(parameterIndex).setClob(reader);
}
public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
- getParameter(parameterIndex).setBlob(slots, parameterIndex, inputStream);
+ getSite(parameterIndex).setBlob(inputStream);
}
public void setNClob(int parameterIndex, Reader reader) throws SQLException {
- getParameter(parameterIndex).setNClob(slots, parameterIndex, reader);
+ getSite(parameterIndex).setNClob(reader);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 1229d10..3320415 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -27,21 +27,22 @@ import java.util.TimeZone;
import org.apache.calcite.avatica.AvaticaParameter;
import org.apache.calcite.avatica.AvaticaResultSet;
import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta.Frame;
import org.apache.calcite.avatica.Meta.Signature;
import org.apache.calcite.avatica.MetaImpl;
import org.apache.kylin.jdbc.IRemoteClient.QueryResult;
public class KylinResultSet extends AvaticaResultSet {
- public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Iterable<Object> iterable) {
- super(statement, signature, resultSetMetaData, timeZone, iterable);
+ public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) {
+ super(statement, signature, resultSetMetaData, timeZone, firstFrame);
}
@Override
protected AvaticaResultSet execute() throws SQLException {
// skip execution if result is already there (case of meta data lookup)
- if (this.iterable != null) {
+ if (this.firstFrame != null) {
return super.execute();
}
@@ -49,7 +50,7 @@ public class KylinResultSet extends AvaticaResultSet {
List<AvaticaParameter> params = signature.parameters;
List<Object> paramValues = null;
if (params != null && params.size() > 0) {
- paramValues = ((KylinPreparedStatement) statement).getParameterValues();
+ paramValues = ((KylinPreparedStatement) statement).getParameterValues2();
}
IRemoteClient client = ((KylinConnection) statement.connection).getRemoteClient();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c0a4be2..ed598fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,7 +84,7 @@
<spring.framework.version>3.1.2.RELEASE</spring.framework.version>
<!-- Calcite Version -->
- <calcite.version>1.0.0-incubating</calcite.version>
+ <calcite.version>1.3.0-incubating</calcite.version>
<!-- Metrics Codahale Version -->
<metrics.version>3.0.1</metrics.version>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 733e8fe..c538080 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -46,7 +46,9 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
private final Object[] current;
private Iterator<String[]> iterator;
+ @SuppressWarnings("unused")
private final static Logger logger = LoggerFactory.getLogger(LookupTableEnumerator.class);
+
public LookupTableEnumerator(OLAPContext olapContext) {
//TODO: assuming LookupTableEnumerator is handled by a cube
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
index 4c34f4e..3dcbbd7 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
@@ -39,7 +39,9 @@ public class OLAPFilterRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
LogicalFilter filter = call.rel(0);
- RelTraitSet traitSet = filter.getTraitSet().replace(OLAPRel.CONVENTION);
+ RelTraitSet origTraitSet = filter.getTraitSet();
+ RelTraitSet traitSet = origTraitSet.replace(OLAPRel.CONVENTION).simplify();
+
OLAPFilterRel olapFilter = new OLAPFilterRel(filter.getCluster(), traitSet, convert(filter.getInput(), traitSet), filter.getCondition());
call.transformTo(olapFilter);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
index cde934e..5eb1ff8 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
@@ -42,7 +42,10 @@ public class OLAPLimitRule extends RelOptRule {
if (sort.offset == null && sort.fetch == null) {
return;
}
- final RelTraitSet traitSet = sort.getTraitSet().replace(OLAPRel.CONVENTION);
+
+ RelTraitSet origTraitSet = sort.getTraitSet();
+ RelTraitSet traitSet = origTraitSet.replace(OLAPRel.CONVENTION).simplify();
+
RelNode input = sort.getInput();
if (!sort.getCollation().getFieldCollations().isEmpty()) {
// Create a sort with the same sort key, but no offset or fetch.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
index 4867162..bee992b 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
@@ -39,9 +39,11 @@ public class OLAPProjectRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
LogicalProject project = call.rel(0);
- RelTraitSet traitSet = project.getTraitSet().replace(OLAPRel.CONVENTION);
+ RelTraitSet origTraitSet = project.getTraitSet();
+ RelTraitSet traitSet = origTraitSet.replace(OLAPRel.CONVENTION).simplify();
+
OLAPProjectRel olapProj = new OLAPProjectRel(project.getCluster(), traitSet, //
- convert(project.getInput(), traitSet), project.getProjects(), project.getRowType(), project.getFlags());
+ convert(project.getInput(), traitSet), project.getProjects(), project.getRowType());
call.transformTo(olapProj);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 48c1075..cf985b7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -33,7 +33,6 @@ import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.type.RelDataType;
@@ -62,7 +61,6 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
/**
@@ -307,7 +305,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
RexProgram program = programBuilder.getProgram();
return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
- sole(inputs), program, ImmutableList.<RelCollation> of());
+ sole(inputs), program);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 8f80962..cda2cbc 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -33,7 +33,6 @@ import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.type.RelDataType;
@@ -53,7 +52,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
/**
*/
@@ -67,8 +65,8 @@ public class OLAPProjectRel extends Project implements OLAPRel {
private boolean afterJoin;
private boolean afterAggregate;
- public OLAPProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, List<RexNode> exps, RelDataType rowType, int flags) {
- super(cluster, traitSet, child, exps, rowType, flags);
+ public OLAPProjectRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, List<RexNode> exps, RelDataType rowType) {
+ super(cluster, traitSet, child, exps, rowType);
Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION);
Preconditions.checkArgument(child.getConvention() == OLAPRel.CONVENTION);
this.rewriteProjects = exps;
@@ -94,7 +92,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
@Override
public Project copy(RelTraitSet traitSet, RelNode child, List<RexNode> exps, RelDataType rowType) {
- return new OLAPProjectRel(getCluster(), traitSet, child, exps, rowType, this.flags);
+ return new OLAPProjectRel(getCluster(), traitSet, child, exps, rowType);
}
@Override
@@ -214,13 +212,13 @@ public class OLAPProjectRel extends Project implements OLAPRel {
RelNode inputOfFilter = inputs.get(0).getInput(0);
RexProgram program = RexProgram.create(inputOfFilter.getRowType(), this.rewriteProjects, filter.getCondition(), this.rowType, getCluster().getRexBuilder());
return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
- inputOfFilter, program, ImmutableList.<RelCollation> of());
+ inputOfFilter, program);
} else {
// keep project for table scan
EnumerableRel input = sole(inputs);
RexProgram program = RexProgram.create(input.getRowType(), this.rewriteProjects, null, this.rowType, getCluster().getRexBuilder());
return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
- input, program, ImmutableList.<RelCollation> of());
+ input, program);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2b92061/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index fc1fdf5..1f2edf9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -41,6 +41,8 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
+import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
+import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.rules.JoinCommuteRule;
@@ -130,7 +132,9 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
planner.removeRule(JoinPushThroughJoinRule.LEFT);
planner.removeRule(JoinPushThroughJoinRule.RIGHT);
- // for columns in having clause will enable table scan filter rule cause kylin does not depend on MPP
+ // keep tree structure like filter -> aggregation -> project -> join/table scan, implementOLAP() rely on this tree pattern
+ planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
+ planner.removeRule(AggregateProjectMergeRule.INSTANCE);
planner.removeRule(FilterProjectTransposeRule.INSTANCE);
// distinct count will be split into a separated query that is joined with the left query
planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
[22/50] [abbrv] incubator-kylin git commit: KYLIN-792 ,
add performance module
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/webapp/app/index.html b/webapp/app/index.html
index 6824498..f1087ed 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -30,20 +30,21 @@
<!-- ref:css css/styles.min.<%= buildNumber %>.css -->
<link rel="stylesheet" type="text/css" href="components/bootstrap/dist/css/bootstrap.min.css">
<link rel="stylesheet" type="text/css" href="components/font-awesome/css/font-awesome.css">
-
+ <link rel="stylesheet" type="text/css" href="css/AdminLTE-fonts.css">
+ <link rel="stylesheet" type="text/css" href="components/ng-grid/ng-grid.css">
<link rel="stylesheet" type="text/css" href="components/angular-tree-control/css/tree-control.css">
<link rel="stylesheet" type="text/css" href="components/angular-tree-control/css/tree-control-attribute.css">
+ <link rel="stylesheet" type="text/css" href="components/animate.css/animate.css">
+
<link rel="stylesheet" type="text/css" href="components/messenger/build/css/messenger.css">
<link rel="stylesheet" type="text/css" href="components/messenger/build/css/messenger-theme-ice.css">
- <link rel="stylesheet" type="text/css" href="components/ng-grid/ng-grid.css">
<link rel="stylesheet" type="text/css" href="components/chosen/chosen.css">
<link rel="stylesheet" type="text/css" href="components/angular-chosen-localytics/chosen-spinner.css">
- <link rel="stylesheet" type="text/css" href="components/animate.css/animate.css">
<link rel="stylesheet" type="text/css" href="components/nvd3/nv.d3.min.css">
-
- <link rel="stylesheet" type="text/css" href="css/AdminLTE-fonts.css">
<link rel="stylesheet" type="text/css" href="css/AdminLTE.css">
<link rel="stylesheet" type="text/css" href="components/bootstrap-sweetalert/lib/sweet-alert.css">
+ <link rel="stylesheet" type="text/css" href="components/angular-busy/dist/angular-busy.css">
+
<link rel="stylesheet/less" href="less/build.less">
<!-- endref -->
@@ -93,7 +94,7 @@
<script src="components/moment/moment.js"></script>
<script src="components/d3/d3.min.js"></script>
-<script src="components/nvd3/nv.d3.min.js"></script>
+<script src="components/nvd3/nv.d3.js"></script>
<script src="components/angularjs-nvd3-directives/dist/angularjs-nvd3-directives.js"></script>
<script src="components/bootstrap-sweetalert/lib/sweet-alert.js"></script>
<script src="components/angular-sweetalert/SweetAlert.js"></script>
@@ -101,6 +102,7 @@
<script src="components/angular-underscore/angular-underscore.js"></script>
<script src="components/jquery-ui/jquery-ui.min.js"></script>
<script src="components/angular-ui-sortable/sortable.js"></script>
+<script src="components/angular-busy/dist/angular-busy.js"></script>
<script src="js/app.js"></script>
<script src="js/config.js"></script>
@@ -127,6 +129,7 @@
<script src="js/services/tree.js"></script>
<script src="js/services/users.js"></script>
<script src="js/services/ngLoading.js"></script>
+<script src="js/services/dashboard.js"></script>
<script src="js/model/cubeConfig.js"></script>
<script src="js/model/jobConfig.js"></script>
@@ -157,6 +160,7 @@
<script src="js/controllers/cubeFilter.js"></script>
<script src="js/controllers/cubeRefresh.js"></script>
<script src="js/controllers/cubeAdvanceSetting.js"></script>
+<script src="js/controllers/dashboard.js"></script>
<!-- endref -->
<!-- ref:remove -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/js/app.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/app.js b/webapp/app/js/app.js
index 59d679e..ba288cb 100644
--- a/webapp/app/js/app.js
+++ b/webapp/app/js/app.js
@@ -17,4 +17,4 @@
*/
//Kylin Application Module
-KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable']);
+KylinApp = angular.module('kylin', ['ngRoute', 'ngResource', 'ngGrid', 'ui.bootstrap', 'ui.ace', 'base64', 'angularLocalStorage', 'localytics.directives', 'treeControl', 'nvd3ChartDirectives', 'ngLoadingRequest', 'oitozero.ngSweetAlert', 'ngCookies', 'angular-underscore', 'ngAnimate', 'ui.sortable',,'cgBusy']);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/js/controllers/dashboard.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/dashboard.js b/webapp/app/js/controllers/dashboard.js
new file mode 100644
index 0000000..40b7b8e
--- /dev/null
+++ b/webapp/app/js/controllers/dashboard.js
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+KylinApp.controller('DashBoardCtrl', function ($scope, DashBoardService, $log, $q) {
+
+
+ $scope.stastic = {
+ countUser: 0,
+ last30DayPercentile: 0,
+ cubeStorage: 0,
+ avgDayQuery: 0,
+ cubesCount: 0
+ }
+
+ $scope.eachDayPercentileData=[];
+
+
+ $scope.reduceCubeSourceTicks=false;
+
+ // each day percentile chart
+ $scope.cubeInfo = function () {
+ var cubeSourceRecords ={"key":"Cube Source Records","values":[],"sizes":[]};
+ $scope.cubeInfoPromise = DashBoardService.listCubes({},
+ function (data) {
+ if(data.length>30){
+ $scope.reduceCubeSourceTicks=true;
+ }
+ for (var i = 0; i < data.length; i++) {
+ cubeSourceRecords.values.push([data[i].name,parseInt(data[i].input_records_count)]);
+ cubeSourceRecords.sizes.push((data[i].size_kb));
+ $log.info(data[i]);
+ }
+ $scope.cubeUsageData = [cubeSourceRecords];
+
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+
+ $scope.cubeSourceYAxisTickFormat = function(){
+ return d3.format('0');
+ }
+
+ $scope.avgDayQuery = function () {
+ $scope.avgDayQueryPromise = DashBoardService.avgDayQuery({},
+ function (data) {
+ if (!isNaN(data[0][0])) {
+ $scope.stastic.avgDayQuery = Math.round(data[0][0]);
+ } else {
+ $log.info("No data available.");
+ }
+ $log.info("avg day query:" + data);
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+ $scope.totalQueryUser = function () {
+ $scope.queryUserPromise = DashBoardService.totalQueryUser({},
+ function (data) {
+ if (!isNaN(data[0][0])) {
+ $scope.stastic.userCount = data[0][0];
+ } else {
+ $log.info("No data available.");
+ }
+
+ $log.info("total query user:" + data);
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+ $scope.last30DayPercentile = function () {
+ $scope.last30DayPercentilePromise = DashBoardService.last30DayPercentile({},
+ function (data) {
+ if (!isNaN(data[0][0])) {
+ $scope.stastic.last30DayPercentile = Math.round(data[0][0] * 100) / 100;
+ } else {
+ $log.info("No data available.");
+ }
+ $log.info("last 30 Day 90th Percentile:" + data);
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+
+ //daily query num
+ $scope.dailyQueryData = [];
+
+
+ // last 30 days
+ $scope.dailyQueryCount = function () {
+ var queryCount ={"key":"Query Count","bar":true,"values":[]};
+ $scope.dailyQueryCountPromise = DashBoardService.dailyQueryCount({},
+ function (data) {
+ for (var i = 0; i < data.length; i++) {
+ $scope.dailyQueryData.push(parseInt(data[i][1]));
+ queryCount.values.push([new Date(data[i][0]).getTime()/1000,data[i][1]]);
+ }
+ console.log("daily query count");
+ $scope.eachDayPercentileData.push(queryCount);
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ };
+
+ $scope.eachDayPercentile = function () {
+ var percenTile90 ={"key":"90%-ile","values":[]},percenTile95 = {"key":"95%-ile","values":[]},queryCount ={"key":"Query Count","bar":true,"values":[]};;
+ $scope.eachDayPercentilePromise = DashBoardService.eachDayPercentile({},
+ function (data) {
+ for (var i = 0; i < data.length; i++) {
+ var _latency90 = data[i][1].split(",")[0].substr(1);
+ var _latency95 = data[i][1].split(",")[1].substr(0, data[i][1].length - 1);
+ var _querycount = data[i][2];
+ percenTile90.values.push([new Date(data[i][0]).getTime()/1000,Math.round(parseFloat(_latency90) * 100) / 100]);
+ percenTile95.values.push([new Date(data[i][0]).getTime()/1000,Math.round(parseFloat(_latency95) * 100) / 100]);
+ queryCount.values.push([new Date(data[i][0]).getTime()/1000,Math.round(parseInt(_querycount) * 100) / 100]);
+ }
+ $scope.eachDayPercentileData=[percenTile90,percenTile95,queryCount];
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+
+ $scope.dailyLatencyReportRightYaxis = function(){
+ return "Query Latency";
+ }
+
+ $scope.xAxisTickFormatFunction = function(){
+ return function(d){
+ return d3.time.format('%Y-%m-%d')(moment.unix(d).toDate());
+ }
+ };
+
+
+ $scope.legendColorFunction = function(){
+ return function(d){
+ return '#E01B5D';
+ }
+ };
+
+
+ $scope.reduceProjectPercentileTicks=false;
+
+ $scope.projectPercentile = function () {
+
+ var percenTile90 ={"key":"90%-ile","values":[]},percenTile95 = {"key":"95%-ile","values":[]};
+
+
+ $scope.projectPercentilePromise = DashBoardService.projectPercentile({},
+ function (data) {
+ if(data.length>30){
+ $scope.reduceProjectPercentileTicks=true;
+ }
+ for (var i = 0; i < data.length; i++) {
+
+ var _latency90 = data[i][1].split(",")[0].substr(1);
+ var _latency95 = data[i][1].split(",")[1].substr(0, data[i][1].length - 1);
+ percenTile90.values.push([data[i][0],Math.round(parseFloat(_latency90) * 100) / 100]);
+ percenTile95.values.push([data[i][0],Math.round(parseFloat(_latency95) * 100) / 100]);
+
+ }
+ $scope.eachProjectPercentileData=[percenTile90,percenTile95];
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+ $scope.projectPercentileToolTipContentFunction = function(key, x, y, e, graph) {
+ return '<table>'+'<tr>'+'<td>'+'<strong>Latency</strong>'+'</td>'+'<td>'+' : '+y+'(S)'+'</td>'+'</tr>'+'<tr>'+'<td>'+'<strong>Percentile</strong>'+'</td>'+'<td>'+' : '+key+'</td>'+'</tr>'+'<tr>'+'<td>'+'<strong>Project</strong>'+'</td>'+'<td>'+' : '+x+'</td>'+'</tr>'+'</table>';
+
+ }
+
+ $scope.dailyPercentileToolTip = function(key, x, y, e, graph) {
+ var suffix ='';
+ if(key.indexOf('ile')!=-1){
+ suffix = '(S)';
+ }
+ return '<table>'+'<tr>'+'<td>'+'<strong>'+key+'</strong>'+'</td>'+'<td>'+' : '+y+suffix+'</td>'+'</tr>'+'<tr>'+'<td>'+'<strong>Date</strong>'+'</td>'+'<td>'+' : '+x+'</td>'+'</tr>'+'</table>';
+
+ }
+ $scope.cubeToolTipContentFunction = function(key, x, y, e, graph) {
+ return '<table>'+'<tr>'+'<td>'+'<strong>Source Records</strong>'+'</td>'+'<td>'+' : '+parseInt(y)+'</td>'+'</tr>'+'<tr>'+'<td>'+'<strong>Source Size</strong>'+'</td>'+'<td>'+' : '+$scope.dataSize(e.series.sizes[e.pointIndex]*1024)+'</td>'+'</tr>'+'<tr>'+'<td>'+'<strong>Cube Name</strong>'+'</td>'+'<td>'+' : '+x+'</td>'+'</tr>'+'</table>';
+ }
+
+ $scope.commonToolTipContentFunction = function(key, x, y, e, graph) {
+ return '<p>' + x +':'+y+ '</p>';
+ }
+
+ $scope.cubesStorage = function () {
+ $scope.cubesStoragePromise = DashBoardService.cubesStorage({},
+ function (cubes) {
+ $scope.stastic.cubesCount = cubes.length;
+ var _cubeStorage = $scope.getTotalSize(cubes);
+ $scope.stastic.cubeStorage = _cubeStorage;
+ }, function (result) {
+ $log.error(result);
+ }).$promise;
+ }();
+
+ $scope.getTotalSize = function (cubes) {
+ var size = 0;
+ if (!cubes) {
+ return 0;
+ }
+ else {
+ for (var i = 0; i < cubes.length; i++) {
+ size += cubes[i].size_kb;
+ }
+ return $scope.dataSize(size * 1024);
+ }
+ };
+
+});
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/js/services/dashboard.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/dashboard.js b/webapp/app/js/services/dashboard.js
new file mode 100644
index 0000000..f5b1454
--- /dev/null
+++ b/webapp/app/js/services/dashboard.js
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+KylinApp.factory('DashBoardService', ['$resource', function ($resource, config) {
+ return $resource(Config.service.url + 'performance/:action', {}, {
+// avgCubeLatency: {method: 'GET', params: {action: 'avgCubeLatency'}, isArray: true},
+ avgDayQuery: {method: 'GET', params: {action: 'avgDayQuery'}, isArray: true},
+ dailyQueryCount: {method: 'GET', params: {action: 'dailyQueryCount'}, isArray: true},
+ eachDayPercentile: {method: 'GET', params: {action: 'eachDayPercentile'}, isArray: true}, // last 30 day 90,95 percentile for each day
+ projectPercentile: {method: 'GET', params: {action: 'projectPercentile'}, isArray: true}, // last 30 day 90,95 percentile for each project
+ cubesStorage: {method: 'GET', params: {action: 'cubesStorage'}, isArray: true},
+ last30DayPercentile: {method: 'GET', params: {action: 'last30DayPercentile'}, isArray: true},//90th percentile of last 30 day
+ totalQueryUser: {method: 'GET', params: {action: 'totalQueryUser'}, isArray: true},
+ listCubes: {method: 'GET', params: {action: 'listCubes'}, isArray: true}
+ });
+}])
+;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/js/services/users.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/users.js b/webapp/app/js/services/users.js
index d05ca94..891ad89 100644
--- a/webapp/app/js/services/users.js
+++ b/webapp/app/js/services/users.js
@@ -49,9 +49,16 @@ KylinApp.service('UserService', function ($http, $q) {
var homePage = "/login";
if (curUser.userDetails && curUser.userDetails.authorities) {
- angular.forEach(curUser.userDetails.authorities, function (authority, index) {
- homePage = (!!roles[authority.authority]) ? roles[authority.authority] : homePage;
- });
+ var authorities = curUser.userDetails.authorities;
+ for(var i=0;i<authorities.length;i++){
+ if(authorities[i].authority==="ROLE_ADMIN"){
+ homePage = roles[authorities[i].authority];
+ break;
+ }else{
+ homePage = roles[authorities[i].authority]
+ }
+ }
+
}
return homePage;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/less/app.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 162c929..4edf83c 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -605,3 +605,20 @@ ul.messenger .messenger-message-inner ,.ngCellText{
max-height: 300px;
overflow-y: auto;
}
+
+.dashboard_wrapper{
+ min-height: 100%;
+ background-color: #ecf0f5;
+ z-index: 800;
+}
+
+
+.size-h1{font-size:36px}
+.size-h2{font-size:30px}
+.size-h3{font-size:24px}
+.size-h4{font-size:18px}
+
+.text-muted {
+ color: #777;
+}
+.tick line {display: none;}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/partials/admin/admin.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/admin/admin.html b/webapp/app/partials/admin/admin.html
index c17aba4..c7b23f0 100644
--- a/webapp/app/partials/admin/admin.html
+++ b/webapp/app/partials/admin/admin.html
@@ -71,6 +71,9 @@
<a class="label-lg label-yellow arrowed-right"style="font-size:18px;" tooltip="Cluster Resource Monitoring" href="{{config.reference_links['hadoop'].link}}" >
Hadoop Monitor
</a>
+ <a href="dashboard">
+ Dashboard
+ </a>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/partials/dashboard.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/dashboard.html b/webapp/app/partials/dashboard.html
new file mode 100644
index 0000000..acccfe4
--- /dev/null
+++ b/webapp/app/partials/dashboard.html
@@ -0,0 +1,228 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<div class="wrapper">
+ <aside class="main-sidebar" style="padding-top:0px !important;">
+ <!-- sidebar: style can be found in sidebar.less -->
+ <section class="sidebar" style="height: auto;">
+
+ </section>
+ <!-- /.sidebar -->
+ </aside>
+
+ <div class="content-wrapper" style="min-height: 800px;">
+
+ <section class="content-header">
+ <h1>
+ Dashboard
+ <small></small>
+ </h1>
+ </section>
+
+ <section class="content">
+ <!-- Info boxes -->
+ <div class="row">
+ <div class="col-md-3 col-sm-6 col-xs-12">
+ <div class="info-box">
+ <span class="info-box-icon bg-aqua"><i class="fa fa-users"></i></span>
+
+ <div class="info-box-content" cg-busy="{promise:queryUserPromise,message:'Loading...'}">
+ <span class="info-box-number size-h2 text-muted">{{stastic.userCount}}</span>
+ <span class="info-box-text size-h4 text-muted">Users</span>
+ </div>
+ <!-- /.info-box-content -->
+ </div>
+ <!-- /.info-box -->
+ </div>
+ <!-- /.col -->
+ <div class="col-md-3 col-sm-6 col-xs-12">
+ <div class="info-box">
+ <span class="info-box-icon bg-red"><i class="fa fa-eye"></i></span>
+
+ <div class="info-box-content" cg-busy="{promise:last30DayPercentilePromise,message:'Loading...'}">
+ <span class="info-box-number size-h2 text-muted">{{stastic.last30DayPercentile}} S</span>
+ <span class="info-box-text size-h4 text-muted">90%-ile Latency (last 30 days)</span>
+ </div>
+ <!-- /.info-box-content -->
+ </div>
+ <!-- /.info-box -->
+ </div>
+ <!-- /.col -->
+
+ <div class="col-md-3 col-sm-6 col-xs-12">
+ <div class="info-box">
+ <span class="info-box-icon bg-green"><i class="fa fa-database"></i></span>
+
+ <div class="info-box-content" cg-busy="{promise:cubesStoragePromise,message:'Loading...'}">
+ <span class="info-box-number size-h2 text-muted">{{stastic.cubeStorage}}</span>
+ <span class="info-box-text size-h4 text-muted">Storage</span>
+ </div>
+ <!-- /.info-box-content -->
+ </div>
+ <!-- /.info-box -->
+ </div>
+ <!-- /.col -->
+ <div class="col-md-3 col-sm-6 col-xs-12">
+ <div class="info-box">
+ <span class="info-box-icon bg-yellow"><i class="fa fa-cubes"></i></span>
+
+ <div class="info-box-content" cg-busy="{promise:cubesStoragePromise,message:'Loading...'}">
+ <span class="info-box-number size-h2 text-muted">{{stastic.cubesCount}}</span>
+ <span class="info-box-text size-h4 text-muted">Cubes</span>
+ </div>
+ <!-- /.info-box-content -->
+ </div>
+ <!-- /.info-box -->
+ </div>
+ <!-- /.col -->
+ </div>
+ <!-- /.row -->
+
+ <div class="row" ng-controller="DashBoardCtrl">
+ <div class="col-md-12">
+ <div class="box">
+ <div class="box-header with-border">
+ <h3 class="box-title">Query Latency (last 30 days)</h3>
+ </div>
+ <!-- /.box-header -->
+ <div class="box-body">
+
+
+ <div class="row">
+ <div class="col-md-12">
+ <p class="text-center">
+ <strong></strong>
+ </p>
+
+
+ <!-- Sales Chart Canvas -->
+ <!--<div cg-busy="{promise:queryCountDailyPercentilePromise,message:'Loading...'}">-->
+ <div cg-busy="[eachDayPercentilePromise]">
+
+ <nvd3-line-plus-bar-chart
+ data="eachDayPercentileData"
+ objectequality="true"
+ margin="{left:100,top:10,bottom:20,right:100}"
+ showXAxis="true"
+ showYAxis="true"
+ y1AxisLabel="Query Count"
+ y2AxisLabel="Query Latency(seconds)"
+ forceY="[0]"
+ height="400"
+ xaxisrotatelabels="-45"
+ tooltips="true"
+ useInteractiveGuideline="true"
+ showLegend="true"
+ legendWidth="200"
+ legendHeight="100"
+ xAxisTickFormat="xAxisTickFormatFunction()"
+ interactive="true"
+ tooltipcontent="dailyPercentileToolTip"
+ >
+ <svg></svg>
+ </nvd3-line-plus-bar-chart>
+ </div>
+ <!-- /.chart-responsive -->
+ </div>
+ <!-- /.col -->
+ </div>
+ <!-- /.row -->
+
+ <div class="row">
+ <div class="col-md-6">
+ <div class="box">
+ <div class="box-header with-border">
+ <h3 class="box-title">Cube Source</h3>
+ </div>
+ <div class="box-body">
+ <!-- Sales Chart Canvas -->
+ <div cg-busy="{promise:cubeInfoPromise}">
+ <nvd3-multi-bar-chart
+ margin="{left:100,top:10,bottom:100,right:60}"
+ data="cubeUsageData"
+ objectequality="true"
+ height="350"
+ showXAxis="true"
+ reducexticks="{{reduceCubeSourceTicks}}"
+ xaxisrotatelabels="-45"
+ showYAxis="true"
+ forceY="[0]"
+ xAxisLabel="Cube Name"
+ yAxisLabel="Cube Source Records"
+ showLegend="true"
+ tooltips="true"
+ legendWidth="200"
+ legendHeight="100"
+ tooltips="true"
+ yAxisTickFormat="cubeSourceYAxisTickFormat()"
+ tooltipcontent="cubeToolTipContentFunction"
+ >
+ <svg></svg>
+ </nvd3-multi-bar-chart>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="col-md-6">
+ <div class="box">
+ <div class="box-header with-border">
+ <h3 class="box-title">Project Latency (last 30 days)</h3>
+ </div>
+ <div class="box-body">
+ <!-- Sales Chart Canvas -->
+ <div cg-busy="{promise:projectPercentilePromise}">
+ <nvd3-multi-bar-chart
+ margin="{left:100,top:10,bottom:100,right:60}"
+ data="eachProjectPercentileData"
+ objectequality="true"
+ height="350"
+ showXAxis="true"
+ xAxisLabel="Project Name"
+ reducexticks="{{reduceProjectPercentileTicks}}"
+ xaxisrotatelabels="-45"
+ showYAxis="true"
+ yAxisLabel="Query Latency(seconds)"
+ showLegend="true"
+ legendWidth="200"
+ legendHeight="100"
+ tooltips="true"
+ tooltipcontent="projectPercentileToolTipContentFunction"
+ >
+ <svg></svg>
+ </nvd3-multi-bar-chart>
+
+ </div>
+ </div>
+ </div>
+
+
+ </div>
+ </div>
+
+
+ </div>
+ <!-- ./box-body -->
+ </div>
+ <!-- /.box -->
+ </div>
+ <!-- /.col -->
+ </div>
+ <!-- /.row -->
+ </section>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/partials/header.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/header.html b/webapp/app/partials/header.html
index eb94664..74e03a0 100644
--- a/webapp/app/partials/header.html
+++ b/webapp/app/partials/header.html
@@ -26,7 +26,7 @@
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
- <a class="navbar-brand" style="padding: 2px 10px 0px 0px"><img src="image/logo.png" height="40px" width="40px"/><small> Kylin </small></a>
+ <a class="navbar-brand" style="padding: 2px 10px 0px 0px;"><img src="image/logo.png" height="40px" width="40px"/><small> Kylin </small></a>
</div>
<div class="navbar-collapse collapse" collapse="isCollapsed">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/routes.json
----------------------------------------------------------------------
diff --git a/webapp/app/routes.json b/webapp/app/routes.json
index 9f13c70..6401528 100644
--- a/webapp/app/routes.json
+++ b/webapp/app/routes.json
@@ -1,5 +1,12 @@
[
{
+ "url": "/dashboard",
+ "params": {
+ "templateUrl": "partials/dashboard.html",
+ "controller": "DashBoardCtrl"
+ }
+ },
+ {
"url": "/cubes",
"params": {
"templateUrl": "partials/cubes/cubes.html",
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/bower.json
----------------------------------------------------------------------
diff --git a/webapp/bower.json b/webapp/bower.json
index 298cba3..f9d99fb 100755
--- a/webapp/bower.json
+++ b/webapp/bower.json
@@ -19,7 +19,7 @@
"moment": "2.5.1",
"d3": "3.4.4",
"nvd3": "1.1.15-beta",
- "angularjs-nvd3-directives": "0.0.5-beta",
+ "angularjs-nvd3-directives": "0.0.7",
"angular-sweetalert": "~1.0.3",
"bootstrap-sweetalert": "~0.4.3",
"angular-underscore": "~0.5.0",
@@ -27,8 +27,8 @@
"underscore": "~1.7.0",
"fuelux": "~3.5.1",
"angular-animate": "1.2",
- "angular-cookies":"1.2"
-
+ "angular-cookies": "1.2",
+ "angular-busy": "~4.0"
},
"devDependencies": {
"less.js": "~1.4.0",
@@ -39,9 +39,9 @@
"nvd3": "1.1.15-beta",
"d3": "3.4.4",
"moment": "2.4.0",
+ "angular-busy": "~4.0",
"angular-resource": "1.2.15",
- "angularLocalStorage": "0.1.7",
"angular-cookies": "~1.2.0-rc.2",
- "angular-animate": "1.2"
+ "angular-animate": "~1.2"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/grunt.json
----------------------------------------------------------------------
diff --git a/webapp/grunt.json b/webapp/grunt.json
index 9a4499f..fa272c9 100755
--- a/webapp/grunt.json
+++ b/webapp/grunt.json
@@ -40,6 +40,7 @@
"app/components/angular-underscore/angular-underscore.js",
"app/components/jquery-ui/jquery-ui.min.js",
"app/components/angular-ui-sortable/sortable.js",
+ "app/components/angular-busy/dist/angular-busy.js",
"tmp/js/scripts.js"
],
"dest": "tmp/js/scripts.min.js"
@@ -60,6 +61,7 @@
"app/components/nvd3/nv.d3.min.css",
"app/css/AdminLTE.css",
"app/components/bootstrap-sweetalert/lib/sweet-alert.css",
+ "app/components/angular-busy/dist/angular-busy.css",
"tmp/css/styles.css"
],
"dest": "tmp/css/styles.min.css"
[47/50] [abbrv] incubator-kylin git commit: KYLIN-880
Posted by li...@apache.org.
KYLIN-880
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/340a175d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/340a175d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/340a175d
Branch: refs/heads/0.7
Commit: 340a175d4b5a65276534261c9ea7b53407ba9086
Parents: f2b9206
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Jul 7 17:30:49 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Jul 7 17:30:49 2015 +0800
----------------------------------------------------------------------
common/src/main/java/org/apache/kylin/common/KylinConfig.java | 6 +++++-
conf/kylin.properties | 4 ++--
examples/test_case_data/localmeta/kylin.properties | 4 ++--
examples/test_case_data/localmeta_v1/kylin.properties | 4 ++--
examples/test_case_data/minicluster/b-kylin/kylin.properties | 4 ++--
.../test_case_data/minicluster/b-kylin/meta/kylin.properties | 4 ++--
examples/test_case_data/minicluster/kylin.properties | 4 ++--
examples/test_case_data/sandbox-hdp21/kylin.properties | 4 ++--
examples/test_case_data/sandbox/kylin.properties | 4 ++--
9 files changed, 21 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 5ae18a6..aa2f224 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -279,7 +279,11 @@ public class KylinConfig {
}
public String getHdfsWorkingDirectory() {
- return getRequired(KYLIN_HDFS_WORKING_DIR);
+ String root = getRequired(KYLIN_HDFS_WORKING_DIR);
+ if (!root.endsWith("/")) {
+ root += "/";
+ }
+ return root + getMetadataUrlPrefix();
}
public String getKylinJobLogDir() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index b1ebccc..aa232bb 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index bf51be7..ff9ece4 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/localmeta_v1/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta_v1/kylin.properties b/examples/test_case_data/localmeta_v1/kylin.properties
index bf51be7..ff9ece4 100644
--- a/examples/test_case_data/localmeta_v1/kylin.properties
+++ b/examples/test_case_data/localmeta_v1/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/minicluster/b-kylin/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties
index e2fe45c..38587f2 100644
--- a/examples/test_case_data/minicluster/b-kylin/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata@hbase:sandbox.hortonworks.com:2181:/hbase-unse
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
index 6e2e951..5d890de 100644
--- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata@hbase:sandbox.hortonworks.com:2181:/hbase-unse
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index aa53158..e81b8d5 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata@hbase:minicluster:/hbase-unsecure
# The storage for final cube file in hbase
kylin.storage.url=hbase:minicluster:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties
index 04fa829..bb1719e 100644
--- a/examples/test_case_data/sandbox-hdp21/kylin.properties
+++ b/examples/test_case_data/sandbox-hdp21/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/340a175d/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 8f73f64..caacb6e 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -26,8 +26,8 @@ kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Temp folder in hdfs
-kylin.hdfs.working.dir=/tmp
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
kylin.job.mapreduce.default.reduce.input.mb=500
[09/50] [abbrv] incubator-kylin git commit: minor,
DefaultScheduler to run next step immediate
Posted by li...@apache.org.
minor, DefaultScheduler to run next step immediate
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/613106c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/613106c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/613106c8
Branch: refs/heads/0.7
Commit: 613106c8b8201b288657d04a2e2c4fb446b3325a
Parents: de7b13b
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 25 14:02:48 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 25 14:04:38 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/impl/threadpool/DefaultScheduler.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/613106c8/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index b486a3e..6483035 100644
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -61,6 +61,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
private ExecutableManager executableManager;
+ private FetcherRunner fetcher;
private ScheduledExecutorService fetcherPool;
private ExecutorService jobPool;
private DefaultContext context;
@@ -80,7 +81,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
private class FetcherRunner implements Runnable {
@Override
- public void run() {
+ synchronized public void run() {
// logger.debug("Job Fetcher is running...");
Map<String, Executable> runningJobs = context.getRunningJobs();
if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
@@ -130,6 +131,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
public void run() {
try {
executable.execute(context);
+ // trigger the next step asap
+ fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
} catch (ExecuteException e) {
logger.error("ExecuteException job:" + executable.getId(), e);
} catch (Exception e) {
@@ -231,7 +234,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
}
});
- fetcherPool.scheduleAtFixedRate(new FetcherRunner(), 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
+ fetcher = new FetcherRunner();
+ fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
hasStarted = true;
}
[43/50] [abbrv] incubator-kylin git commit: KYLIN-867 refactor Hybrid
Posted by li...@apache.org.
KYLIN-867 refactor Hybrid
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4f742cda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4f742cda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4f742cda
Branch: refs/heads/0.7
Commit: 4f742cda2f996b7b744f98e0f8515ba74f036299
Parents: 9491e92
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jul 3 16:43:42 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 6 16:19:17 2015 +0800
----------------------------------------------------------------------
.../common/util/AbstractKylinTestCase.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 5 -
.../hybrid/test_kylin_hybrid_inner_join.json | 15 +-
.../hybrid/test_kylin_hybrid_left_join.json | 15 +-
.../localmeta/project/default.json | 10 ++
.../apache/kylin/invertedindex/IIInstance.java | 5 -
.../metadata/project/RealizationEntry.java | 22 ++-
.../metadata/realization/IRealization.java | 2 -
.../AdjustForWeeklyMatchedRealization.java | 4 +-
.../RoutingRules/RealizationPriorityRule.java | 4 +-
.../apache/kylin/query/test/IIQueryTest.java | 12 +-
.../apache/kylin/query/test/KylinQueryTest.java | 6 +
.../apache/kylin/rest/service/BasicService.java | 5 +
.../apache/kylin/rest/service/CacheService.java | 4 +
.../apache/kylin/rest/service/CubeService.java | 6 +-
.../kylin/storage/hybrid/HybridInstance.java | 161 +++++++++++--------
.../kylin/storage/hybrid/HybridManager.java | 25 ++-
.../storage/hybrid/HybridStorageEngine.java | 62 +++----
.../kylin/storage/hybrid/HybridManagerTest.java | 9 +-
19 files changed, 207 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index a4a6dc4..d337501 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -36,7 +36,7 @@ public abstract class AbstractKylinTestCase {
public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.metadata.project.ProjectManager" };
+ public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" };
public abstract void createTestMetadata() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 898925a..88f110f 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -377,11 +377,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
}
@Override
- public String getModelName() {
- return this.getDescriptor().getModelName();
- }
-
- @Override
public List<TblColRef> getAllDimensions() {
return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
index 5582841..e38e55b 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
@@ -1,14 +1,13 @@
{
"uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_inner_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_empty"
+ }
+ ],
+ "cost": 40,
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
index 37f59d6..48cebe6 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
@@ -1,14 +1,13 @@
{
"uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_left_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_left_join_empty"
+ }
+ ],
+ "cost": 40,
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 853a10a..639f3be 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -26,6 +26,16 @@
"name": "test_kylin_ii",
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii"
+ },
+ {
+ "name": "test_kylin_hybrid_inner_join",
+ "type": "HYBRID",
+ "realization": "test_kylin_hybrid_inner_join"
+ },
+ {
+ "name": "test_kylin_hybrid_left_join",
+ "type": "HYBRID",
+ "realization": "test_kylin_hybrid_left_join"
}
]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index 9c46fbb..89ed059 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -338,11 +338,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
}
@Override
- public String getModelName() {
- return this.getDescriptor().getModelName();
- }
-
- @Override
public List<TblColRef> getAllDimensions() {
return getDescriptor().listAllDimensions();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
index cfbe6a3..73c899b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
@@ -49,7 +49,27 @@ public class RealizationEntry {
public void setRealization(String realization) {
this.realization = realization;
}
-
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RealizationEntry entry = (RealizationEntry) o;
+
+ if (realization != null ? !realization.equalsIgnoreCase(entry.realization) : entry.realization != null) return false;
+ if (type != entry.type) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = type != null ? type.hashCode() : 0;
+ result = 31 * result + (realization != null ? realization.hashCode() : 0);
+ return result;
+ }
+
@Override
public String toString() {
return "" + type.name() + "." + realization;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 43f5e9a..1f3837d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -69,6 +69,4 @@ public interface IRealization {
public long getDateRangeEnd();
- public String getModelName();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
index 3402556..8872a7d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
@@ -49,8 +49,8 @@ public class AdjustForWeeklyMatchedRealization extends RoutingRule {
if (first instanceof HybridInstance) {
HybridInstance hybrid = (HybridInstance) first;
- if (hybrid.getHistoryRealizationInstance() instanceof CubeInstance)
- first = hybrid.getHistoryRealizationInstance();
+ if (hybrid.getRealizations()[0] instanceof CubeInstance)
+ first = hybrid.getRealizations()[0];
}
if (first instanceof CubeInstance) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
index 7425d62..72995d6 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
@@ -33,9 +33,9 @@ public class RealizationPriorityRule extends RoutingRule {
static Map<RealizationType, Integer> priorities = Maps.newHashMap();
static {
- priorities.put(RealizationType.HYBRID, 2);
priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.INVERTED_INDEX, 0);
+ priorities.put(RealizationType.HYBRID, 1);
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
}
public static void setPriorities(Map<RealizationType, Integer> priorities) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index b58e439..864e3c1 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -39,9 +39,9 @@ public class IIQueryTest extends KylinQueryTest {
KylinQueryTest.setUp();//invoke super class
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 0);
- priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.HYBRID, 1);
+ priorities.put(RealizationType.INVERTED_INDEX, 1);
+ priorities.put(RealizationType.CUBE, 2);
+ priorities.put(RealizationType.HYBRID, 2);
RealizationPriorityRule.setPriorities(priorities);
}
@@ -51,9 +51,9 @@ public class IIQueryTest extends KylinQueryTest {
KylinQueryTest.tearDown();//invoke super class
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.HYBRID, 0);
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
+ priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.HYBRID, 1);
RealizationPriorityRule.setPriorities(priorities);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 8c3901c..415783a 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -28,6 +28,7 @@ import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.query.enumerator.OLAPQuery;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.junit.AfterClass;
@@ -105,6 +106,7 @@ public class KylinQueryTest extends KylinTestBase {
protected static void preferCubeOf(String joinType) {
CubeManager cubeManager = CubeManager.getInstance(config);
+ HybridManager hybridManager = HybridManager.getInstance(config);
boolean cubesBuiltInBatch = cubeManager.getCube("test_kylin_cube_with_slr_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_empty") != null && cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty") != null;
@@ -114,11 +116,15 @@ public class KylinQueryTest extends KylinTestBase {
}
if (joinType.equals("inner")) {
+ hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(10);
+ hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(100);
cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(20);
cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(10);
cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(100);
cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty").setCost(90);
} else if (joinType.equals("left") || joinType.equals("default")) {
+ hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(20);
+ hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(100);
cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(100);
cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(90);
cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(20);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 764dd7b..d124cf0 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -45,6 +45,7 @@ import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
import org.apache.calcite.jdbc.Driver;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
@@ -174,6 +175,10 @@ public abstract class BasicService {
return IIManager.getInstance(getConfig());
}
+ public final HybridManager getHybridManager() {
+ return HybridManager.getInstance(getConfig());
+ }
+
protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables()).filter(new Predicate<AbstractExecutable>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 9ec1621..f68fd91 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -28,6 +28,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.springframework.stereotype.Component;
import java.io.IOException;
@@ -45,6 +46,7 @@ public class CacheService extends BasicService {
switch (cacheType) {
case CUBE:
getCubeManager().loadCubeCache(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cacheKey);
cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
break;
case CUBE_DESC:
@@ -55,6 +57,7 @@ public class CacheService extends BasicService {
break;
case INVERTED_INDEX:
getIIManager().loadIICache(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey);
cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
break;
case INVERTED_INDEX_DESC:
@@ -76,6 +79,7 @@ public class CacheService extends BasicService {
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ HybridManager.clearCache();
RealizationRegistry.clearCache();
ProjectManager.clearCache();
BasicService.resetOLAPDataSources();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 71e9f19..2c7c628 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -207,9 +207,9 @@ public class CubeService extends BasicService {
if (project == null) {
return false;
}
- for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
- if (projectDataModel.getType() == RealizationType.CUBE) {
- CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
+ for (RealizationEntry projectRealization : project.getRealizationEntries()) {
+ if (projectRealization != null && projectRealization.getType() == RealizationType.CUBE) {
+ CubeInstance cube = getCubeManager().getCube(projectRealization.getRealization());
assert cube != null;
if (cube.equals(target)) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 1694719..68984cd 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -3,11 +3,10 @@ package org.apache.kylin.storage.hybrid;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -16,6 +15,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.SQLDigest;
+import java.util.LinkedHashSet;
import java.util.List;
/**
@@ -30,40 +30,90 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
@JsonProperty("name")
private String name;
- @JsonProperty("historyRealization")
- private RealizationEntry historyRealization;
+ @JsonProperty("realizations")
+ private List<RealizationEntry> realizationEntries;
- @JsonProperty("realTimeRealization")
- private RealizationEntry realTimeRealization;
+ @JsonProperty("cost")
+ private int cost = 50;
- private IRealization historyRealizationInstance;
- private IRealization realTimeRealizationInstance;
+ private IRealization[] realizations = null;
+ private List<TblColRef> allDimensions = null;
+ private List<TblColRef> allColumns = null;
+ private List<MeasureDesc> allMeasures = null;
+ private long dateRangeStart;
+ private long dateRangeEnd;
+ private boolean isReady = false;
private String projectName;
- public void init() {
- RealizationRegistry registry = RealizationRegistry.getInstance(config);
- historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization());
- realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization());
+ private boolean initiated = false;
- if (historyRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + historyRealization.getType() + "'" + " with name '" + historyRealization.getRealization() + "' in '" + name + "'");
- }
+ public List<RealizationEntry> getRealizationEntries() {
+ return realizationEntries;
+ }
- if (realTimeRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + realTimeRealization.getType() + "'" + " with name '" + realTimeRealization.getRealization() + "' in '" + name + "'");
- }
+ private void init() {
+ if (initiated == true)
+ return;
+
+ synchronized (this) {
+ if (initiated == true)
+ return;
+
+ if (realizationEntries == null || realizationEntries.size() == 0)
+ throw new IllegalArgumentException();
+
+ RealizationRegistry registry = RealizationRegistry.getInstance(config);
+ realizations = new IRealization[realizationEntries.size()];
+ for (int i = 0; i < realizationEntries.size(); i++) {
+ IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization());
+ if (realization == null)
+ throw new IllegalArgumentException("Realization '" + realizationEntries.get(i) + "' not loaded.");
+ realizations[i] = realization;
+ }
+
+ LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<TblColRef> dimensions = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>();
+ dateRangeStart = 0;
+ dateRangeEnd = Long.MAX_VALUE;
+ for (IRealization realization : realizations) {
+ if (realization.isReady() == false)
+ continue;
+
+ columns.addAll(realization.getAllColumns());
+ dimensions.addAll(realization.getAllDimensions());
+ measures.addAll(realization.getMeasures());
+
+ if (realization.isReady())
+ isReady = true;
+ if (dateRangeStart == 0 || realization.getDateRangeStart() < dateRangeStart)
+ dateRangeStart = realization.getDateRangeStart();
+
+ if (dateRangeStart == Long.MAX_VALUE || realization.getDateRangeEnd() > dateRangeEnd)
+ dateRangeEnd = realization.getDateRangeEnd();
+ }
+
+ allDimensions = Lists.newArrayList(dimensions);
+ allColumns = Lists.newArrayList(columns);
+ allMeasures = Lists.newArrayList(measures);
+
+ initiated = true;
+ }
}
@Override
public boolean isCapable(SQLDigest digest) {
- return getHistoryRealizationInstance().isCapable(digest) && getRealTimeRealizationInstance().isCapable(digest);
+ for (IRealization realization : getRealizations()) {
+ if (realization.isCapable(digest))
+ return true;
+ }
+ return false;
}
@Override
public int getCost(SQLDigest digest) {
- return historyRealizationInstance.getCost(digest);
- //return Math.min(historyRealizationInstance.getCost(digest), realTimeRealizationInstance.getCost(digest));
+ return cost;
}
@Override
@@ -73,22 +123,24 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
@Override
public String getFactTable() {
- return getHistoryRealizationInstance().getFactTable();
+ return getRealizations()[0].getFactTable();
}
@Override
public List<TblColRef> getAllColumns() {
- return getHistoryRealizationInstance().getAllColumns();
+ init();
+ return allColumns;
}
@Override
public List<MeasureDesc> getMeasures() {
- return getHistoryRealizationInstance().getMeasures();
+ init();
+ return allMeasures;
}
@Override
public boolean isReady() {
- return historyRealizationInstance.isReady() || realTimeRealizationInstance.isReady();
+ return isReady;
}
@Override
@@ -128,61 +180,32 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
this.config = config;
}
- public RealizationEntry getHistoryRealization() {
- return historyRealization;
- }
-
- public RealizationEntry getRealTimeRealization() {
- return realTimeRealization;
- }
-
- public IRealization getHistoryRealizationInstance() {
- if (historyRealizationInstance == null) {
- this.init();
- }
- return historyRealizationInstance;
- }
-
- public IRealization getRealTimeRealizationInstance() {
- if (realTimeRealizationInstance == null) {
- this.init();
- }
- return realTimeRealizationInstance;
- }
-
@Override
public long getDateRangeStart() {
- return Math.min(getHistoryRealizationInstance().getDateRangeStart(), getRealTimeRealizationInstance().getDateRangeStart());
+ return dateRangeStart;
}
@Override
public long getDateRangeEnd() {
- return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1;
+ return dateRangeEnd;
}
-
-
- public DataModelDesc getDataModelDesc(){
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModel();
- }
-
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModel();
+ @Override
+ public List<TblColRef> getAllDimensions() {
+ init();
+ return allDimensions;
}
- @Override
- public String getModelName() {
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName();
- }
-
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModelName();
+ public IRealization[] getRealizations() {
+ init();
+ return realizations;
}
- @Override
- public List<TblColRef> getAllDimensions(){
-
- return this.getHistoryRealizationInstance().getAllDimensions();
+ public static String concatResourcePath(String hybridName) {
+ return ResourceStore.HYBRID_RESOURCE_ROOT + "/" + hybridName + ".json";
}
+ public void setCost(int cost) {
+ this.cost = cost;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index df3fae3..8373cf1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -81,6 +81,21 @@ public class HybridManager implements IRealizationProvider {
logger.debug("Loaded " + paths.size() + " Hybrid(s)");
}
+ public void reloadHybridInstanceByChild(RealizationType type, String realizationName) throws IOException {
+ for (HybridInstance hybridInstance : hybridMap.values()) {
+ boolean includes = false;
+ for (IRealization realization : hybridInstance.getRealizations()) {
+ if (realization.getType() == type && realization.getName().equalsIgnoreCase(realizationName)) {
+ includes = true;
+ break;
+ }
+ }
+
+ if (includes == true)
+ loadHybridInstance(HybridInstance.concatResourcePath(hybridInstance.getName()));
+ }
+ }
+
private synchronized HybridInstance loadHybridInstance(String path) throws IOException {
ResourceStore store = getStore();
@@ -89,14 +104,13 @@ public class HybridManager implements IRealizationProvider {
hybridInstance = store.getResource(path, HybridInstance.class, HYBRID_SERIALIZER);
hybridInstance.setConfig(config);
+ if (hybridInstance.getRealizationEntries() == null || hybridInstance.getRealizationEntries().size() == 0) {
+ throw new IllegalStateException("HybridInstance must have realization entries, " + path);
+ }
+
if (StringUtils.isBlank(hybridInstance.getName()))
throw new IllegalStateException("HybridInstance name must not be blank, at " + path);
- if (hybridInstance.getHistoryRealization() == null || hybridInstance.getRealTimeRealization() == null) {
-
- throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path);
- }
-
final String name = hybridInstance.getName();
hybridMap.putLocal(name, hybridInstance);
@@ -124,4 +138,5 @@ public class HybridManager implements IRealizationProvider {
private ResourceStore getStore() {
return ResourceStore.getStore(this.config);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 7ad5ed0..11ad211 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,69 +1,43 @@
package org.apache.kylin.storage.hybrid;
-import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import com.google.common.collect.Ranges;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.realization.SQLDigestUtil;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import javax.annotation.Nullable;
+import java.util.List;
/**
*/
public class HybridStorageEngine implements IStorageEngine {
- private HybridInstance hybridInstance;
- private IStorageEngine historicalStorageEngine;
- private IStorageEngine realtimeStorageEngine;
+ private IRealization[] realizations;
+ private IStorageEngine[] storageEngines;
public HybridStorageEngine(HybridInstance hybridInstance) {
- this.hybridInstance = hybridInstance;
- this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance());
- this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance());
+ this.realizations = hybridInstance.getRealizations();
+ storageEngines = new IStorageEngine[realizations.length];
+ for (int i = 0; i < realizations.length; i++) {
+ storageEngines[i] = StorageEngineFactory.getStorageEngine(realizations[i]);
+ }
}
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) {
-
- // search the historic realization
- ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest);
-
- String modelName = hybridInstance.getModelName();
- MetadataManager metaMgr = getMetadataManager();
- DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
-
- // if the model isn't partitioned, only query the history
- if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
- return historicalDataIterator;
-
- TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
-
- ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
- Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
- new Function<Void, ITupleIterator>() {
- @Nullable
- @Override
- public ITupleIterator apply(@Nullable Void input) {
- return realtimeStorageEngine.search(context, sqlDigest);
- }
- });
-
- // combine history and real-time tuple iterator
- return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
+ List<ITupleIterator> tupleIterators = Lists.newArrayList();
+ for (int i = 0; i < realizations.length; i++) {
+ if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest)) {
+ ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest);
+ tupleIterators.add(dataIterator);
+ }
+ }
+ // combine tuple iterator
+ return new CompoundTupleIterator(tupleIterators);
}
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f742cda/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 3ad7ca6..8a1bcf0 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -26,15 +26,12 @@ public class HybridManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
- HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
- cube.init();
- System.out.println(JsonUtil.writeValueAsIndentString(cube));
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
+ System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));
- IRealization history = cube.getHistoryRealizationInstance();
- IRealization realTime = cube.getRealTimeRealizationInstance();
+ IRealization history = hybridInstance.getRealizations()[0];
Assert.assertTrue(history instanceof CubeInstance);
- Assert.assertTrue(realTime instanceof IIInstance);
}
[17/50] [abbrv] incubator-kylin git commit: KYLIN-839 Optimize
Snapshot table memory usage
Posted by li...@apache.org.
KYLIN-839 Optimize Snapshot table memory usage
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3ce0233d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3ce0233d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3ce0233d
Branch: refs/heads/0.7
Commit: 3ce0233d4def856c348c7c31d4a8b8b0e6d2b710
Parents: 7fdcd31
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 30 15:09:56 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 30 15:09:56 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/dict/lookup/SnapshotTable.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ce0233d/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 28db5b4..fb1feb1 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -80,7 +80,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
}
for (String cell : row) {
- b.addValue(cell);
+ if (cell != null)
+ b.addValue(cell);
}
}
@@ -231,7 +232,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
rows.add(row);
for (int j = 0; j < n; j++) {
row[j] = in.readUTF();
- b.addValue(row[j]);
+ if (row[j] != null)
+ b.addValue(row[j]);
}
}
this.dict = b.build(0);
[37/50] [abbrv] incubator-kylin git commit: KYLIN-780 JDBC upgrade,
test pass
Posted by li...@apache.org.
KYLIN-780 JDBC upgrade, test pass
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b28e9f4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b28e9f4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b28e9f4a
Branch: refs/heads/0.7
Commit: b28e9f4a9216e6405bf2e15a992cfd3287117622
Parents: f4be152
Author: Yang Li <li...@apache.org>
Authored: Tue Jun 30 20:36:14 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 6 13:48:13 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/jdbc/Driver.java | 287 ++++---
.../org/apache/kylin/jdbc/IRemoteClient.java | 56 ++
.../java/org/apache/kylin/jdbc/KylinClient.java | 373 +++++++++
.../org/apache/kylin/jdbc/KylinConnection.java | 128 +++
.../apache/kylin/jdbc/KylinConnectionImpl.java | 158 ----
.../org/apache/kylin/jdbc/KylinEnumerator.java | 87 --
.../apache/kylin/jdbc/KylinJdbc40Factory.java | 30 -
.../apache/kylin/jdbc/KylinJdbc41Factory.java | 127 ---
.../org/apache/kylin/jdbc/KylinJdbcFactory.java | 108 +++
.../java/org/apache/kylin/jdbc/KylinMeta.java | 341 ++++++++
.../org/apache/kylin/jdbc/KylinMetaImpl.java | 821 -------------------
.../org/apache/kylin/jdbc/KylinPrepare.java | 101 ---
.../org/apache/kylin/jdbc/KylinPrepareImpl.java | 47 --
.../kylin/jdbc/KylinPrepareStatementImpl.java | 170 ----
.../kylin/jdbc/KylinPreparedStatement.java | 100 +++
.../org/apache/kylin/jdbc/KylinResultSet.java | 117 ++-
.../org/apache/kylin/jdbc/KylinStatement.java | 31 +
.../apache/kylin/jdbc/KylinStatementImpl.java | 54 --
.../kylin/jdbc/json/PreparedQueryRequest.java | 33 +
.../apache/kylin/jdbc/json/QueryRequest.java | 40 +
.../apache/kylin/jdbc/json/SQLResponseStub.java | 321 ++++++++
.../kylin/jdbc/json/StatementParameter.java | 47 ++
.../apache/kylin/jdbc/json/TableMetaStub.java | 345 ++++++++
.../kylin/jdbc/stub/ConnectionException.java | 55 --
.../org/apache/kylin/jdbc/stub/DataSet.java | 55 --
.../org/apache/kylin/jdbc/stub/KylinClient.java | 374 ---------
.../kylin/jdbc/stub/KylinColumnMetaData.java | 38 -
.../apache/kylin/jdbc/stub/RemoteClient.java | 59 --
.../apache/kylin/jdbc/stub/SQLResponseStub.java | 323 --------
.../apache/kylin/jdbc/stub/TableMetaStub.java | 347 --------
.../util/DefaultSslProtocolSocketFactory.java | 2 -
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 183 -----
.../java/org/apache/kylin/jdbc/DriverTest.java | 339 ++++----
.../java/org/apache/kylin/jdbc/DummyClient.java | 165 ++--
.../java/org/apache/kylin/jdbc/DummyDriver.java | 64 +-
.../apache/kylin/jdbc/DummyJdbc41Factory.java | 35 -
.../org/apache/kylin/jdbc/DummyJdbcFactory.java | 39 +
.../java/org/apache/kylin/query/QueryCli.java | 4 +-
.../kylin/rest/controller/QueryController.java | 27 +-
.../apache/kylin/rest/service/BasicService.java | 3 +-
.../org/apache/kylin/rest/util/QueryUtil.java | 22 +-
41 files changed, 2491 insertions(+), 3565 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
index cee4de4..4cc0445 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
@@ -1,147 +1,140 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.DriverVersion;
-import org.apache.calcite.avatica.Handler;
-import org.apache.calcite.avatica.HandlerImpl;
-import org.apache.calcite.avatica.UnregisteredDriver;
-
-import org.apache.kylin.jdbc.stub.RemoteClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.jdbc.stub.ConnectionException;
-
-/**
- * <p>
- * Kylin JDBC Driver based on optiq avatica and kylin restful api.<br>
- * Supported versions:
- * </p>
- * <ul>
- * <li>jdbc 4.0</li>
- * <li>jdbc 4.1</li>
- * </ul>
- *
- * <p>
- * Supported Statements:
- * </p>
- * <ul>
- * <li>{@link KylinStatementImpl}</li>
- * <li>{@link KylinPrepareStatementImpl}</li>
- * </ul>
- *
- * <p>
- * Supported properties:
- * <ul>
- * <li>user: username</li>
- * <li>password: password</li>
- * <li>ssl: true/false</li>
- * </ul>
- * </p>
- *
- * <p>
- * Driver init code sample:<br>
- *
- * <pre>
- * Driver driver = (Driver) Class.forName("org.apache.kylin.kylin.jdbc.Driver").newInstance();
- * Properties info = new Properties();
- * info.put("user", "user");
- * info.put("password", "password");
- * info.put("ssl", true);
- * Connection conn = driver.connect("jdbc:kylin://{domain}/{project}", info);
- * </pre>
- *
- * </p>
- *
- * @author xduo
- *
- */
-public class Driver extends UnregisteredDriver {
- private static final Logger logger = LoggerFactory.getLogger(Driver.class);
-
- public static final String CONNECT_STRING_PREFIX = "jdbc:kylin:";
- static {
- try {
- DriverManager.registerDriver(new Driver());
- } catch (SQLException e) {
- throw new RuntimeException("Error occurred while registering JDBC driver " + Driver.class.getName() + ": " + e.toString());
- }
- }
-
- @Override
- protected DriverVersion createDriverVersion() {
- return DriverVersion.load(Driver.class, "org-apache-kylin-jdbc.properties", "Kylin JDBC Driver", "unknown version", "Kylin", "unknown version");
- }
-
- @Override
- protected String getFactoryClassName(JdbcVersion jdbcVersion) {
- switch (jdbcVersion) {
- case JDBC_30:
- throw new UnsupportedOperationException();
- case JDBC_40:
- return "org.apache.kylin.jdbc.KylinJdbc40Factory";
- case JDBC_41:
- default:
- return "org.apache.kylin.jdbc.KylinJdbc41Factory";
- }
- }
-
- @Override
- protected Handler createHandler() {
- return new HandlerImpl() {
- @Override
- public void onConnectionInit(AvaticaConnection connection_) throws SQLException {
- KylinConnectionImpl kylinConn = (KylinConnectionImpl) connection_;
- RemoteClient runner = ((KylinJdbc41Factory) factory).newRemoteClient(kylinConn);
- try {
- runner.connect();
- kylinConn.setMetaProject(runner.getMetadata(kylinConn.getProject()));
- logger.debug("Connection inited.");
- } catch (ConnectionException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new SQLException(e.getLocalizedMessage());
- }
- }
-
- public void onConnectionClose(AvaticaConnection connection) {
- logger.debug("Connection closed.");
- }
-
- public void onStatementExecute(AvaticaStatement statement, ResultSink resultSink) {
- logger.debug("statement executed.");
- }
-
- public void onStatementClose(AvaticaStatement statement) {
- logger.debug("statement closed.");
- }
- };
- }
-
- @Override
- protected String getConnectStringPrefix() {
- return CONNECT_STRING_PREFIX;
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.DriverVersion;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.avatica.UnregisteredDriver;
+
+/**
+ * <p>
+ * Kylin JDBC Driver based on Calcite Avatica and Kylin restful API.<br>
+ * Supported versions:
+ * </p>
+ * <ul>
+ * <li>jdbc 4.0</li>
+ * <li>jdbc 4.1</li>
+ * </ul>
+ *
+ * <p>
+ * Supported Statements:
+ * </p>
+ * <ul>
+ * <li>{@link KylinStatementImpl}</li>
+ * <li>{@link KylinPrepareStatementImpl}</li>
+ * </ul>
+ *
+ * <p>
+ * Supported properties:
+ * <ul>
+ * <li>user: username</li>
+ * <li>password: password</li>
+ * <li>ssl: true/false</li>
+ * </ul>
+ * </p>
+ *
+ * <p>
+ * Driver init code sample:<br>
+ *
+ * <pre>
+ * Driver driver = (Driver) Class.forName("org.apache.kylin.kylin.jdbc.Driver").newInstance();
+ * Properties info = new Properties();
+ * info.put("user", "user");
+ * info.put("password", "password");
+ * info.put("ssl", true);
+ * Connection conn = driver.connect("jdbc:kylin://{domain}/{project}", info);
+ * </pre>
+ *
+ * </p>
+ */
+public class Driver extends UnregisteredDriver {
+
+ public static final String CONNECT_STRING_PREFIX = "jdbc:kylin:";
+ static {
+ try {
+ DriverManager.registerDriver(new Driver());
+ } catch (SQLException e) {
+ throw new RuntimeException("Error occurred while registering JDBC driver " + Driver.class.getName() + ": " + e.toString());
+ }
+ }
+
+ @Override
+ protected String getConnectStringPrefix() {
+ return CONNECT_STRING_PREFIX;
+ }
+
+ @Override
+ protected DriverVersion createDriverVersion() {
+ return DriverVersion.load(Driver.class, "kylin-jdbc.properties", "Kylin JDBC Driver", "unknown version", "Kylin", "unknown version");
+ }
+
+ @Override
+ protected String getFactoryClassName(JdbcVersion jdbcVersion) {
+ switch (jdbcVersion) {
+ case JDBC_30:
+ throw new UnsupportedOperationException();
+ case JDBC_40:
+ return KylinJdbcFactory.Version40.class.getName();
+ case JDBC_41:
+ default:
+ return KylinJdbcFactory.Version41.class.getName();
+ }
+ }
+
+ @Override
+ public Meta createMeta(AvaticaConnection connection) {
+ return new KylinMeta((KylinConnection) connection);
+ }
+
+// @Override
+// protected Handler createHandler() {
+// return new HandlerImpl() {
+// @Override
+// public void onConnectionInit(AvaticaConnection connection) throws SQLException {
+// KylinConnection conn = (KylinConnection) connection;
+// RemoteClient runner = ((KylinJdbcFactory) factory).newRemoteClient(conn);
+// try {
+// runner.connect();
+// conn.setMetaProject(runner.getMetadata(conn.getProject()));
+// logger.debug("Connection inited.");
+// } catch (ConnectionException e) {
+// logger.error(e.getLocalizedMessage(), e);
+// throw new SQLException(e.getLocalizedMessage());
+// }
+// }
+//
+// public void onConnectionClose(AvaticaConnection connection) {
+// logger.debug("Connection closed.");
+// }
+//
+// public void onStatementExecute(AvaticaStatement statement, ResultSink resultSink) {
+// logger.debug("statement executed.");
+// }
+//
+// public void onStatementClose(AvaticaStatement statement) {
+// logger.debug("statement closed.");
+// }
+// };
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
new file mode 100644
index 0000000..b6a13e5
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
+
+public interface IRemoteClient extends Closeable {
+
+ public static class QueryResult {
+ public final List<ColumnMetaData> columnMeta;
+ public final Iterable<Object> iterable;
+
+ public QueryResult(List<ColumnMetaData> columnMeta, Iterable<Object> iterable) {
+ this.columnMeta = columnMeta;
+ this.iterable = iterable;
+ }
+ }
+
+ /**
+ * Connect to Kylin restful service. IOException will be thrown if authentication failed.
+ */
+ public void connect() throws IOException;
+
+ /**
+ * Retrieve meta data of given project.
+ */
+ public KMetaProject retrieveMetaData(String project) throws IOException;
+
+ /**
+ * Execute query remotely and get back result.
+ */
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
new file mode 100644
index 0000000..4b73ad6
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -0,0 +1,373 @@
+package org.apache.kylin.jdbc;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.xml.bind.DatatypeConverter;
+
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.calcite.avatica.ColumnMetaData.ScalarType;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpMethodBase;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.httpclient.protocol.Protocol;
+import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
+import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog;
+import org.apache.kylin.jdbc.KylinMeta.KMetaColumn;
+import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
+import org.apache.kylin.jdbc.KylinMeta.KMetaSchema;
+import org.apache.kylin.jdbc.KylinMeta.KMetaTable;
+import org.apache.kylin.jdbc.json.PreparedQueryRequest;
+import org.apache.kylin.jdbc.json.QueryRequest;
+import org.apache.kylin.jdbc.json.SQLResponseStub;
+import org.apache.kylin.jdbc.json.StatementParameter;
+import org.apache.kylin.jdbc.json.TableMetaStub;
+import org.apache.kylin.jdbc.json.TableMetaStub.ColumnMetaStub;
+import org.apache.kylin.jdbc.util.DefaultSslProtocolSocketFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class KylinClient implements IRemoteClient {
+
+ private static final Logger logger = LoggerFactory.getLogger(KylinClient.class);
+
+ private final KylinConnection conn;
+ private final Properties connProps;
+ private final HttpClient httpClient;
+ private final ObjectMapper jsonMapper;
+
+ public KylinClient(KylinConnection conn) {
+ this.conn = conn;
+ this.connProps = conn.getConnectionProperties();
+ this.httpClient = new HttpClient();
+ this.jsonMapper = new ObjectMapper();
+
+ // trust all certificates
+ if (isSSL()) {
+ Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new DefaultSslProtocolSocketFactory(), 443));
+ }
+ }
+
+ private boolean isSSL() {
+ return Boolean.parseBoolean(connProps.getProperty("ssl", "false"));
+ }
+
+ private String baseUrl() {
+ return (isSSL() ? "https://" : "http://") + conn.getBaseUrl();
+ }
+
+ private void addHttpHeaders(HttpMethodBase method) {
+ method.addRequestHeader("Accept", "application/json, text/plain, */*");
+ method.addRequestHeader("Content-Type", "application/json");
+
+ String username = connProps.getProperty("user");
+ String password = connProps.getProperty("password");
+ String basicAuth = DatatypeConverter.printBase64Binary((username + ":" + password).getBytes());
+ method.addRequestHeader("Authorization", "Basic " + basicAuth);
+ }
+
+ @Override
+ public void connect() throws IOException {
+ PostMethod post = new PostMethod(baseUrl() + "/kylin/api/user/authentication");
+ addHttpHeaders(post);
+ StringRequestEntity requestEntity = new StringRequestEntity("{}", "application/json", "UTF-8");
+ post.setRequestEntity(requestEntity);
+
+ httpClient.executeMethod(post);
+
+ if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
+ throw asIOException(post);
+ }
+ }
+
+ @Override
+ public KMetaProject retrieveMetaData(String project) throws IOException {
+ assert conn.getProject().equals(project);
+
+ String url = baseUrl() + "/kylin/api/tables_and_columns?project=" + project;
+ GetMethod get = new GetMethod(url);
+ addHttpHeaders(get);
+
+ httpClient.executeMethod(get);
+
+ if (get.getStatusCode() != 200 && get.getStatusCode() != 201) {
+ throw asIOException(get);
+ }
+
+ List<TableMetaStub> tableMetaStubs = jsonMapper.readValue(get.getResponseBodyAsStream(), new TypeReference<List<TableMetaStub>>() {
+ });
+
+ List<KMetaTable> tables = convertMetaTables(tableMetaStubs);
+ List<KMetaSchema> schemas = convertMetaSchemas(tables);
+ List<KMetaCatalog> catalogs = convertMetaCatalogs(schemas);
+ return new KMetaProject(project, catalogs);
+ }
+
+ private List<KMetaCatalog> convertMetaCatalogs(List<KMetaSchema> schemas) {
+ Map<String, List<KMetaSchema>> catalogMap = new LinkedHashMap<String, List<KMetaSchema>>();
+ for (KMetaSchema schema : schemas) {
+ List<KMetaSchema> list = catalogMap.get(schema.tableCatalog);
+ if (list == null) {
+ list = new ArrayList<KMetaSchema>();
+ catalogMap.put(schema.tableCatalog, list);
+ }
+ list.add(schema);
+ }
+
+ List<KMetaCatalog> result = new ArrayList<KMetaCatalog>();
+ for (List<KMetaSchema> catSchemas : catalogMap.values()) {
+ String catalog = catSchemas.get(0).tableCatalog;
+ result.add(new KMetaCatalog(catalog, catSchemas));
+ }
+ return result;
+ }
+
+ private List<KMetaSchema> convertMetaSchemas(List<KMetaTable> tables) {
+ Map<String, List<KMetaTable>> schemaMap = new LinkedHashMap<String, List<KMetaTable>>();
+ for (KMetaTable table : tables) {
+ String key = table.tableCat + "!!" + table.tableSchem;
+ List<KMetaTable> list = schemaMap.get(key);
+ if (list == null) {
+ list = new ArrayList<KMetaTable>();
+ schemaMap.put(key, list);
+ }
+ list.add(table);
+ }
+
+ List<KMetaSchema> result = new ArrayList<KMetaSchema>();
+ for (List<KMetaTable> schemaTables : schemaMap.values()) {
+ String catalog = schemaTables.get(0).tableCat;
+ String schema = schemaTables.get(0).tableSchem;
+ result.add(new KMetaSchema(catalog, schema, schemaTables));
+ }
+ return result;
+ }
+
+ private List<KMetaTable> convertMetaTables(List<TableMetaStub> tableMetaStubs) {
+ List<KMetaTable> result = new ArrayList<KMetaTable>(tableMetaStubs.size());
+ for (TableMetaStub tableStub : tableMetaStubs) {
+ result.add(convertMetaTable(tableStub));
+ }
+ return result;
+ }
+
+ private KMetaTable convertMetaTable(TableMetaStub tableStub) {
+ List<KMetaColumn> columns = new ArrayList<KMetaColumn>(tableStub.getColumns().size());
+ for (ColumnMetaStub columnStub : tableStub.getColumns()) {
+ columns.add(convertMetaColumn(columnStub));
+ }
+ return new KMetaTable(tableStub.getTABLE_CAT(), tableStub.getTABLE_SCHEM(), tableStub.getTABLE_NAME(), tableStub.getTABLE_TYPE(), columns);
+ }
+
+ private KMetaColumn convertMetaColumn(ColumnMetaStub columnStub) {
+ return new KMetaColumn(columnStub.getTABLE_CAT(), columnStub.getTABLE_SCHEM(), columnStub.getTABLE_NAME(), columnStub.getCOLUMN_NAME(), columnStub.getDATA_TYPE(), columnStub.getTYPE_NAME(), columnStub.getCOLUMN_SIZE(), columnStub.getDECIMAL_DIGITS(), columnStub.getNUM_PREC_RADIX(), columnStub.getNULLABLE(), columnStub.getCHAR_OCTET_LENGTH(), columnStub.getORDINAL_POSITION(), columnStub.getIS_NULLABLE());
+ }
+
+ @Override
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
+
+ SQLResponseStub queryResp = executeKylinQuery(sql, convertParameters(params, paramValues));
+ if (queryResp.getIsException())
+ throw new IOException(queryResp.getExceptionMessage());
+
+ List<ColumnMetaData> metas = convertColumnMeta(queryResp);
+ List<Object> data = convertResultData(queryResp, metas);
+
+ return new QueryResult(metas, data);
+ }
+
+ private List<StatementParameter> convertParameters(List<AvaticaParameter> params, List<Object> paramValues) {
+ if (params == null || params.isEmpty())
+ return null;
+
+ assert params.size() == paramValues.size();
+
+ List<StatementParameter> result = new ArrayList<StatementParameter>();
+ for (Object v : paramValues) {
+ result.add(new StatementParameter(v.getClass().getCanonicalName(), String.valueOf(v)));
+ }
+ return result;
+ }
+
+ private SQLResponseStub executeKylinQuery(String sql, List<StatementParameter> params) throws IOException {
+ String url = baseUrl() + "/kylin/api/query";
+ String project = conn.getProject();
+
+ QueryRequest request = null;
+ if (null != params) {
+ request = new PreparedQueryRequest();
+ ((PreparedQueryRequest) request).setParams(params);
+ url += "/prestate"; // means prepared statement..
+ } else {
+ request = new QueryRequest();
+ }
+ request.setSql(sql);
+ request.setProject(project);
+
+ PostMethod post = new PostMethod(url);
+ addHttpHeaders(post);
+
+ String postBody = jsonMapper.writeValueAsString(request);
+ logger.debug("Post body:\n " + postBody);
+ StringRequestEntity requestEntity = new StringRequestEntity(postBody, "application/json", "UTF-8");
+ post.setRequestEntity(requestEntity);
+
+ httpClient.executeMethod(post);
+
+ if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
+ throw asIOException(post);
+ }
+
+ return jsonMapper.readValue(post.getResponseBodyAsStream(), SQLResponseStub.class);
+ }
+
+ private List<ColumnMetaData> convertColumnMeta(SQLResponseStub queryResp) {
+ List<ColumnMetaData> metas = new ArrayList<ColumnMetaData>();
+ for (int i = 0; i < queryResp.getColumnMetas().size(); i++) {
+ SQLResponseStub.ColumnMetaStub scm = queryResp.getColumnMetas().get(i);
+ ScalarType type = ColumnMetaData.scalar(scm.getColumnType(), scm.getColumnTypeName(), Rep.of(convertType(scm.getColumnType())));
+
+ ColumnMetaData meta = new ColumnMetaData(i, scm.isAutoIncrement(), scm.isCaseSensitive(), scm.isSearchable(), scm.isCurrency(), scm.getIsNullable(), scm.isSigned(), scm.getDisplaySize(), scm.getLabel(), scm.getName(), scm.getSchemaName(), scm.getPrecision(), scm.getScale(), scm.getTableName(), scm.getSchemaName(), type, scm.isReadOnly(), scm.isWritable(), scm.isWritable(), null);
+
+ metas.add(meta);
+ }
+
+ return metas;
+ }
+
+ private List<Object> convertResultData(SQLResponseStub queryResp, List<ColumnMetaData> metas) {
+ List<String[]> stringResults = queryResp.getResults();
+ List<Object> data = new ArrayList<Object>(stringResults.size());
+ for (String[] result : stringResults) {
+ Object[] row = new Object[result.length];
+
+ for (int i = 0; i < result.length; i++) {
+ ColumnMetaData meta = metas.get(i);
+ row[i] = wrapObject(result[i], meta.type.type);
+ }
+
+ data.add(row);
+ }
+ return (List<Object>) data;
+ }
+
+ private IOException asIOException(HttpMethodBase method) throws IOException {
+ return new IOException(method + " failed, error code " + method.getStatusCode() + " and response: " + method.getResponseBodyAsString());
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static Class convertType(int sqlType) {
+ Class result = Object.class;
+
+ switch (sqlType) {
+ case Types.CHAR:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ result = String.class;
+ break;
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ result = BigDecimal.class;
+ break;
+ case Types.BIT:
+ result = Boolean.class;
+ break;
+ case Types.TINYINT:
+ result = Byte.class;
+ break;
+ case Types.SMALLINT:
+ result = Short.class;
+ break;
+ case Types.INTEGER:
+ result = Integer.class;
+ break;
+ case Types.BIGINT:
+ result = Long.class;
+ break;
+ case Types.REAL:
+ case Types.FLOAT:
+ case Types.DOUBLE:
+ result = Double.class;
+ break;
+ case Types.BINARY:
+ case Types.VARBINARY:
+ case Types.LONGVARBINARY:
+ result = Byte[].class;
+ break;
+ case Types.DATE:
+ result = Date.class;
+ break;
+ case Types.TIME:
+ result = Time.class;
+ break;
+ case Types.TIMESTAMP:
+ result = Timestamp.class;
+ break;
+ }
+
+ return result;
+ }
+
+ public static Object wrapObject(String value, int sqlType) {
+ if (null == value) {
+ return null;
+ }
+
+ switch (sqlType) {
+ case Types.CHAR:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ return value;
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ return new BigDecimal(value);
+ case Types.BIT:
+ return Boolean.parseBoolean(value);
+ case Types.TINYINT:
+ return Byte.valueOf(value);
+ case Types.SMALLINT:
+ return Short.valueOf(value);
+ case Types.INTEGER:
+ return Integer.parseInt(value);
+ case Types.BIGINT:
+ return Long.parseLong(value);
+ case Types.FLOAT:
+ return Float.parseFloat(value);
+ case Types.REAL:
+ case Types.DOUBLE:
+ return Double.parseDouble(value);
+ case Types.BINARY:
+ case Types.VARBINARY:
+ case Types.LONGVARBINARY:
+ return value.getBytes();
+ case Types.DATE:
+ return Date.valueOf(value);
+ case Types.TIME:
+ return Time.valueOf(value);
+ case Types.TIMESTAMP:
+ return Timestamp.valueOf(value);
+ }
+
+ return value;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
new file mode 100644
index 0000000..86b2ff8
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.avatica.Meta.CursorFactory;
+import org.apache.calcite.avatica.Meta.Signature;
+import org.apache.calcite.avatica.UnregisteredDriver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KylinConnection extends AvaticaConnection {
+
+ private static final Logger logger = LoggerFactory.getLogger(KylinConnection.class);
+
+ private final String baseUrl;
+ private final String project;
+ private final IRemoteClient remoteClient;
+
+ protected KylinConnection(UnregisteredDriver driver, KylinJdbcFactory factory, String url, Properties info) throws SQLException {
+ super(driver, factory, url, info);
+
+ String odbcUrl = url;
+ odbcUrl = odbcUrl.replace(Driver.CONNECT_STRING_PREFIX + "//", "");
+ String[] temps = odbcUrl.split("/");
+
+ assert temps.length == 2;
+
+ this.baseUrl = temps[0];
+ this.project = temps[1];
+
+ logger.debug("Kylin base url " + this.baseUrl + ", project name " + this.project);
+
+ this.remoteClient = factory.newRemoteClient(this);
+
+ try {
+ this.remoteClient.connect();
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
+ String getBaseUrl() {
+ return baseUrl;
+ }
+
+ String getProject() {
+ return project;
+ }
+
+ Properties getConnectionProperties() {
+ return info;
+ }
+
+ @Override
+ public AvaticaStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ return super.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ Meta.Signature sig = mockPreparedSignature(sql);
+ return factory().newPreparedStatement(this, null, sig, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ // TODO add restful API to prepare SQL, get back expected ResultSetMetaData
+ Signature mockPreparedSignature(String sql) {
+ List<AvaticaParameter> params = new ArrayList<AvaticaParameter>();
+ int startIndex = 0;
+ while (sql.indexOf("?", startIndex) >= 0) {
+ AvaticaParameter param = new AvaticaParameter(false, 0, 0, 0, null, null, null);
+ params.add(param);
+ startIndex = sql.indexOf("?", startIndex) + 1;
+ }
+
+ ArrayList<ColumnMetaData> columns = new ArrayList<ColumnMetaData>();
+ Map<String, Object> internalParams = Collections.<String, Object>emptyMap();
+
+ return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY);
+ }
+
+ private KylinJdbcFactory factory() {
+ return (KylinJdbcFactory) factory;
+ }
+
+ public IRemoteClient getRemoteClient() {
+ return remoteClient;
+ }
+
+ @Override
+ public void close() throws SQLException {
+ super.close();
+ try {
+ remoteClient.close();
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
deleted file mode 100644
index 718ef7d..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import javax.xml.bind.DatatypeConverter;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaFactory;
-import org.apache.calcite.avatica.AvaticaPreparedStatement;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.Meta;
-import org.apache.calcite.avatica.UnregisteredDriver;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.jdbc.KylinPrepare.PrepareResult;
-
-/**
- * Kylin connection implementation
- *
- * @author xduo
- *
- */
-public abstract class KylinConnectionImpl extends AvaticaConnection {
- private static final Logger logger = LoggerFactory.getLogger(KylinConnectionImpl.class);
-
- private final String baseUrl;
- private final String project;
- private KylinMetaImpl.MetaProject metaProject;
- public final List<AvaticaStatement> statements;
- static final Trojan TROJAN = createTrojan();
-
- protected KylinConnectionImpl(UnregisteredDriver driver, AvaticaFactory factory, String url, Properties info) {
- super(driver, factory, url, info);
-
- String odbcUrl = url;
- odbcUrl = odbcUrl.replace(Driver.CONNECT_STRING_PREFIX + "//", "");
- String[] temps = odbcUrl.split("/");
-
- assert temps.length == 2;
-
- this.baseUrl = temps[0];
- this.project = temps[1];
-
- logger.debug("Kylin base url " + this.baseUrl + ", project name " + this.project);
-
- statements = new ArrayList<AvaticaStatement>();
- }
-
- @Override
- protected Meta createMeta() {
- return new KylinMetaImpl(this, (KylinJdbc41Factory) factory);
- }
-
- @Override
- public AvaticaStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- AvaticaStatement statement = super.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
- statements.add(statement);
-
- return statement;
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql) throws SQLException {
- PrepareResult pr = new KylinPrepareImpl().prepare(sql);
- AvaticaPreparedStatement statement = ((KylinJdbc41Factory) factory).newPreparedStatement(this, pr, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, this.getHoldability());
- statements.add(statement);
-
- return statement;
- }
-
- // ~ kylin specified implements
-
- public String getBasicAuthHeader() {
- String username = this.info.getProperty("user");
- String password = this.info.getProperty("password");
-
- return DatatypeConverter.printBase64Binary((username + ":" + password).getBytes());
- }
-
- public String getConnectUrl() {
- boolean isSsl = Boolean.parseBoolean((this.info.getProperty("ssl", "false")));
- return (isSsl ? "https://" : "http://") + this.baseUrl + "/kylin/api/user/authentication";
- }
-
- public String getMetaProjectUrl(String project) {
- assert project != null;
- boolean isSsl = Boolean.parseBoolean((this.info.getProperty("ssl", "false")));
- return (isSsl ? "https://" : "http://") + this.baseUrl + "/kylin/api/tables_and_columns?project=" + project;
- }
-
- public String getQueryUrl() {
- boolean isSsl = Boolean.parseBoolean((this.info.getProperty("ssl", "false")));
- return (isSsl ? "https://" : "http://") + this.baseUrl + "/kylin/api/query";
- }
-
- public String getProject() {
- return this.project;
- }
-
- public Meta getMeta() {
- return this.meta;
- }
-
- public AvaticaFactory getFactory() {
- return this.factory;
- }
-
- public UnregisteredDriver getDriver() {
- return this.driver;
- }
-
- public KylinMetaImpl.MetaProject getMetaProject() {
- return metaProject;
- }
-
- public void setMetaProject(KylinMetaImpl.MetaProject metaProject) {
- this.metaProject = metaProject;
- }
-
- @Override
- public void close() throws SQLException {
- super.close();
-
- this.metaProject = null;
- this.statements.clear();
- }
-
- @Override
- public String toString() {
- return "KylinConnectionImpl [baseUrl=" + baseUrl + ", project=" + project + ", metaProject=" + metaProject + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
deleted file mode 100644
index 2502d4b..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.util.Collection;
-import java.util.Iterator;
-
-import org.apache.calcite.linq4j.Enumerator;
-
-/**
- * Query results enumerator
- *
- * @author xduo
- *
- */
-public class KylinEnumerator<E> implements Enumerator<E> {
-
- /**
- * current row
- */
- private E current;
-
- /**
- * data collection
- */
- private Collection<E> dataCollection;
-
- /**
- * result iterator
- */
- private Iterator<E> cursor;
-
- public KylinEnumerator(Collection<E> dataCollection) {
- this.dataCollection = dataCollection;
- this.cursor = this.dataCollection.iterator();
-
- if (null == this.cursor) {
- throw new RuntimeException("Cursor can't be null");
- }
- }
-
- @Override
- public E current() {
- return current;
- }
-
- @Override
- public boolean moveNext() {
- if (!cursor.hasNext()) {
- this.reset();
-
- return false;
- }
-
- current = cursor.next();
-
- return true;
- }
-
- @Override
- public void reset() {
- this.cursor = this.dataCollection.iterator();
- }
-
- @Override
- public void close() {
- this.cursor = null;
- this.dataCollection = null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc40Factory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc40Factory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc40Factory.java
deleted file mode 100644
index 235348b..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc40Factory.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-/**
- * @author xduo
- *
- */
-public class KylinJdbc40Factory extends KylinJdbc41Factory {
-
- public KylinJdbc40Factory() {
- super(4, 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
deleted file mode 100644
index 1336f96..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Properties;
-import java.util.TimeZone;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaDatabaseMetaData;
-import org.apache.calcite.avatica.AvaticaFactory;
-import org.apache.calcite.avatica.AvaticaPrepareResult;
-import org.apache.calcite.avatica.AvaticaPreparedStatement;
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaResultSetMetaData;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.UnregisteredDriver;
-
-import org.apache.kylin.jdbc.stub.KylinClient;
-import org.apache.kylin.jdbc.stub.RemoteClient;
-
-/**
- * Kylin JDBC factory.
- *
- * @author xduo
- *
- */
-public class KylinJdbc41Factory implements AvaticaFactory {
- private final int major;
- private final int minor;
-
- /** Creates a JDBC factory. */
- public KylinJdbc41Factory() {
- this(4, 1);
- }
-
- /** Creates a JDBC factory with given major/minor version number. */
- protected KylinJdbc41Factory(int major, int minor) {
- this.major = major;
- this.minor = minor;
- }
-
- public int getJdbcMajorVersion() {
- return major;
- }
-
- public int getJdbcMinorVersion() {
- return minor;
- }
-
- public AvaticaConnection newConnection(UnregisteredDriver driver, AvaticaFactory factory, String url, Properties info) {
- return new KylinJdbc41Connection(driver, factory, url, info);
- }
-
- public AvaticaDatabaseMetaData newDatabaseMetaData(AvaticaConnection connection) {
- return new AvaticaJdbc41DatabaseMetaData(connection);
- }
-
- public AvaticaStatement newStatement(AvaticaConnection connection, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
- return new KylinJdbc41Statement(connection, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- public AvaticaPreparedStatement newPreparedStatement(AvaticaConnection connection, AvaticaPrepareResult prepareResult, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- return new KylinJdbc41PreparedStatement(connection, prepareResult, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- public AvaticaResultSet newResultSet(AvaticaStatement statement, AvaticaPrepareResult prepareResult, TimeZone timeZone) {
- final ResultSetMetaData metaData = newResultSetMetaData(statement, prepareResult.getColumnList());
- return new KylinResultSet(statement, prepareResult, metaData, timeZone);
- }
-
- public AvaticaResultSetMetaData newResultSetMetaData(AvaticaStatement statement, List<ColumnMetaData> columnMetaDataList) {
- return new AvaticaResultSetMetaData(statement, null, columnMetaDataList);
- }
-
- // ~ kylin sepcified
- public RemoteClient newRemoteClient(KylinConnectionImpl connection) {
- return new KylinClient(connection);
- }
-
- /** Implementation of Connection for JDBC 4.1. */
- private static class KylinJdbc41Connection extends KylinConnectionImpl {
- KylinJdbc41Connection(UnregisteredDriver driver, AvaticaFactory factory, String url, Properties info) {
- super(driver, (KylinJdbc41Factory) factory, url, info);
- }
- }
-
- /** Implementation of Statement for JDBC 4.1. */
- public static class KylinJdbc41Statement extends KylinStatementImpl {
- public KylinJdbc41Statement(AvaticaConnection connection, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
- super(connection, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
- }
-
- /** Implementation of PreparedStatement for JDBC 4.1. */
- public static class KylinJdbc41PreparedStatement extends KylinPrepareStatementImpl {
- KylinJdbc41PreparedStatement(AvaticaConnection connection, AvaticaPrepareResult sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- super(connection, sql, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
- }
-
- /** Implementation of DatabaseMetaData for JDBC 4.1. */
- private static class AvaticaJdbc41DatabaseMetaData extends AvaticaDatabaseMetaData {
- AvaticaJdbc41DatabaseMetaData(AvaticaConnection connection) {
- super(connection);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
new file mode 100644
index 0000000..030dbbb
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaDatabaseMetaData;
+import org.apache.calcite.avatica.AvaticaFactory;
+import org.apache.calcite.avatica.AvaticaPreparedStatement;
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaResultSetMetaData;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta.Signature;
+import org.apache.calcite.avatica.Meta.StatementHandle;
+import org.apache.calcite.avatica.UnregisteredDriver;
+
+/**
+ * Kylin JDBC factory.
+ */
+public class KylinJdbcFactory implements AvaticaFactory {
+
+ public static class Version40 extends KylinJdbcFactory {
+ public Version40() {
+ super(4, 0);
+ }
+ }
+
+ public static class Version41 extends KylinJdbcFactory {
+ public Version41() {
+ super(4, 1);
+ }
+ }
+
+ final int major;
+ final int minor;
+
+ /** Creates a JDBC factory with given major/minor version number. */
+ protected KylinJdbcFactory(int major, int minor) {
+ this.major = major;
+ this.minor = minor;
+ }
+
+ @Override
+ public int getJdbcMajorVersion() {
+ return major;
+ }
+
+ @Override
+ public int getJdbcMinorVersion() {
+ return minor;
+ }
+
+ @Override
+ public AvaticaConnection newConnection(UnregisteredDriver driver, AvaticaFactory factory, String url, Properties info) throws SQLException {
+ return new KylinConnection(driver, (KylinJdbcFactory) factory, url, info);
+ }
+
+ @Override
+ public AvaticaDatabaseMetaData newDatabaseMetaData(AvaticaConnection connection) {
+ return new AvaticaDatabaseMetaData(connection) {
+ };
+ }
+
+ @Override
+ public AvaticaStatement newStatement(AvaticaConnection connection, StatementHandle h, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ return new KylinStatement((KylinConnection) connection, h, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public AvaticaPreparedStatement newPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ return new KylinPreparedStatement((KylinConnection) connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Iterable<Object> iterable) throws SQLException {
+ AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature);
+ return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, iterable);
+ }
+
+ @Override
+ public ResultSetMetaData newResultSetMetaData(AvaticaStatement statement, Signature signature) throws SQLException {
+ return new AvaticaResultSetMetaData(statement, null, signature);
+ }
+
+ public IRemoteClient newRemoteClient(KylinConnection conn) {
+ return new KylinClient(conn);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
new file mode 100644
index 0000000..72d7fb1
--- /dev/null
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.calcite.avatica.AvaticaUtils;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.MetaImpl;
+
+/**
+ * Implementation of Avatica interface
+ */
+public class KylinMeta extends MetaImpl {
+
+ private KMetaProject metaProject;
+
+ public KylinMeta(KylinConnection connection) {
+ super(connection);
+ }
+
+ private KylinConnection connection() {
+ return (KylinConnection) connection;
+ }
+
+ // insert/update/delete go this path, ignorable for Kylin
+ @Override
+ public Signature prepare(StatementHandle h, String sql, int maxRowCount) {
+ return connection().mockPreparedSignature(sql);
+ }
+
+ // mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute()
+ @Override
+ public MetaResultSet prepareAndExecute(StatementHandle h, String sql, int maxRowCount, PrepareCallback callback) {
+ final Signature signature;
+ try {
+ synchronized (callback.getMonitor()) {
+ callback.clear();
+ signature = prepare(h, sql, maxRowCount);
+ callback.assign(signature, null);
+ }
+ callback.execute();
+ return new MetaResultSet(h.id, false, signature, null);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private KMetaProject getMetaProject() {
+ try {
+ if (metaProject == null) {
+ KylinConnection conn = connection();
+ metaProject = conn.getRemoteClient().retrieveMetaData(conn.getProject());
+ }
+ return metaProject;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public MetaResultSet getTableTypes() {
+ return createResultSet(metaTableTypes, MetaTableType.class, "TABLE_TYPE");
+ }
+
+ @Override
+ public MetaResultSet getCatalogs() {
+ List<KMetaCatalog> catalogs = getMetaProject().catalogs;
+ return createResultSet(catalogs, MetaCatalog.class, "TABLE_CATALOG");
+ }
+
+ @Override
+ public MetaResultSet getSchemas(String catalog, Pat schemaPattern) {
+ List<KMetaSchema> schemas = getMetaProject().getSchemas(catalog, schemaPattern);
+ return createResultSet(schemas, MetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG");
+ }
+
+ @Override
+ public MetaResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
+ List<KMetaTable> tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList);
+ return createResultSet(tables, MetaTable.class, //
+ "TABLE_CAT", //
+ "TABLE_SCHEM", //
+ "TABLE_NAME", //
+ "TABLE_TYPE", //
+ "REMARKS", //
+ "TYPE_CAT", //
+ "TYPE_SCHEM", //
+ "TYPE_NAME", //
+ "SELF_REFERENCING_COL_NAME", //
+ "REF_GENERATION");
+ }
+
+ @Override
+ public MetaResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
+ List<KMetaColumn> columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern);
+ return createResultSet(columns, MetaColumn.class, //
+ "TABLE_CAT", //
+ "TABLE_SCHEM", //
+ "TABLE_NAME", //
+ "COLUMN_NAME", //
+ "DATA_TYPE", //
+ "TYPE_NAME", //
+ "COLUMN_SIZE", //
+ "BUFFER_LENGTH", //
+ "DECIMAL_DIGITS", //
+ "NUM_PREC_RADIX", //
+ "NULLABLE", //
+ "REMARKS", //
+ "COLUMN_DEF", //
+ "SQL_DATA_TYPE", //
+ "SQL_DATETIME_SUB", //
+ "CHAR_OCTET_LENGTH", //
+ "ORDINAL_POSITION", //
+ "IS_NULLABLE", //
+ "SCOPE_CATALOG", //
+ "SCOPE_TABLE", //
+ "SOURCE_DATA_TYPE", //
+ "IS_AUTOINCREMENT", //
+ "IS_GENERATEDCOLUMN");
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private MetaResultSet createResultSet(List iterable, Class clazz, String... names) {
+ final List<ColumnMetaData> columns = new ArrayList<ColumnMetaData>();
+ final List<Field> fields = new ArrayList<Field>();
+ final List<String> fieldNames = new ArrayList<String>();
+ for (String name : names) {
+ final int index = fields.size();
+ final String fieldName = AvaticaUtils.toCamelCase(name);
+ final Field field;
+ try {
+ field = clazz.getField(fieldName);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
+ }
+ columns.add(columnMetaData(name, index, field.getType()));
+ fields.add(field);
+ fieldNames.add(fieldName);
+ }
+
+ CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames);
+ Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory);
+ StatementHandle statementHandle = this.createStatement(null);
+
+ return new MetaResultSet(statementHandle.id, true, signature, iterable);
+ }
+
+ // ============================================================================
+
+ public static interface NamedWithChildren extends Named {
+ List<? extends NamedWithChildren> getChildren();
+ }
+
+ public static List<? extends NamedWithChildren> searchByPatterns(NamedWithChildren parent, Pat... patterns) {
+ assert patterns != null && patterns.length > 0;
+
+ List<? extends NamedWithChildren> children = findChildren(parent, patterns[0]);
+ if (patterns.length == 1) {
+ return children;
+ } else {
+ List<NamedWithChildren> result = new ArrayList<NamedWithChildren>();
+ Pat[] subPatterns = Arrays.copyOfRange(patterns, 1, patterns.length);
+ for (NamedWithChildren c : children) {
+ result.addAll(searchByPatterns(c, subPatterns));
+ }
+ return result;
+ }
+ }
+
+ private static List<? extends NamedWithChildren> findChildren(NamedWithChildren parent, Pat pattern) {
+ if (null == pattern.s || pattern.s.equals("%")) {
+ return parent.getChildren();
+ }
+
+ List<NamedWithChildren> result = new ArrayList<NamedWithChildren>();
+ Pattern regex = likeToRegex(pattern);
+
+ for (NamedWithChildren c : parent.getChildren()) {
+ if (regex.matcher(c.getName()).matches()) {
+ result.add(c);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Converts a LIKE-style pattern (where '%' represents a wild-card,
+ * escaped using '\') to a Java regex.
+ */
+ private static Pattern likeToRegex(Pat pattern) {
+ StringBuilder buf = new StringBuilder("^");
+ char[] charArray = pattern.s.toCharArray();
+ int slash = -2;
+ for (int i = 0; i < charArray.length; i++) {
+ char c = charArray[i];
+ if (slash == i - 1) {
+ buf.append('[').append(c).append(']');
+ } else {
+ switch (c) {
+ case '\\':
+ slash = i;
+ break;
+ case '%':
+ buf.append(".*");
+ break;
+ case '[':
+ buf.append("\\[");
+ break;
+ case ']':
+ buf.append("\\]");
+ break;
+ default:
+ buf.append('[').append(c).append(']');
+ }
+ }
+ }
+ buf.append("$");
+
+ return Pattern.compile(buf.toString());
+ }
+
+ // ============================================================================
+
+ private static final List<MetaTableType> metaTableTypes = new ArrayList<MetaTableType>();
+ static {
+ metaTableTypes.add(new MetaTableType("TABLE"));
+ }
+
+ public static class KMetaProject implements NamedWithChildren {
+ public final String projectName;
+ public final List<KMetaCatalog> catalogs;
+
+ public KMetaProject(String projectName, List<KMetaCatalog> catalogs) {
+ this.projectName = projectName;
+ this.catalogs = catalogs;
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<KMetaSchema> getSchemas(String catalog, Pat schemaPattern) {
+ return (List<KMetaSchema>) searchByPatterns(this, Pat.of(catalog), schemaPattern);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<KMetaTable> getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
+ return (List<KMetaTable>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<KMetaColumn> getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
+ return (List<KMetaColumn>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern, columnNamePattern);
+ }
+
+ @Override
+ public String getName() {
+ return projectName;
+ }
+
+ @Override
+ public List<? extends NamedWithChildren> getChildren() {
+ return catalogs;
+ }
+ }
+
+ public static class KMetaCatalog extends MetaCatalog implements NamedWithChildren {
+ public final List<KMetaSchema> schemas;
+
+ public KMetaCatalog(String tableCatalog, List<KMetaSchema> schemas) {
+ super(tableCatalog);
+ this.schemas = schemas;
+ }
+
+ @Override
+ public List<? extends NamedWithChildren> getChildren() {
+ return schemas;
+ }
+ }
+
+ public static class KMetaSchema extends MetaSchema implements NamedWithChildren {
+ public final List<KMetaTable> tables;
+
+ public KMetaSchema(String tableCatalog, String tableSchem, List<KMetaTable> tables) {
+ super(tableCatalog, tableSchem);
+ this.tables = tables;
+ }
+
+ @Override
+ public List<? extends NamedWithChildren> getChildren() {
+ return tables;
+ }
+ }
+
+ public static class KMetaTable extends MetaTable implements NamedWithChildren {
+ public final List<KMetaColumn> columns;
+
+ public KMetaTable(String tableCat, String tableSchem, String tableName, String tableType, List<KMetaColumn> columns) {
+ super(tableCat, tableSchem, tableName, tableType);
+ this.columns = columns;
+ }
+
+ @Override
+ public List<? extends NamedWithChildren> getChildren() {
+ return columns;
+ }
+ }
+
+ public static class KMetaColumn extends MetaColumn implements NamedWithChildren {
+
+ public KMetaColumn(String tableCat, String tableSchem, String tableName, String columnName, int dataType, String typeName, int columnSize, Integer decimalDigits, int numPrecRadix, int nullable, int charOctetLength, int ordinalPosition, String isNullable) {
+ super(tableCat, tableSchem, tableName, columnName, dataType, typeName, columnSize, decimalDigits, numPrecRadix, nullable, charOctetLength, ordinalPosition, isNullable);
+ }
+
+ @Override
+ public List<NamedWithChildren> getChildren() {
+ return Collections.<NamedWithChildren> emptyList();
+ }
+ }
+}
[39/50] [abbrv] incubator-kylin git commit: KYLIN-780 add a test case
for where not(A=B)
Posted by li...@apache.org.
KYLIN-780 add a test case for where not(A=B)
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f364ffcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f364ffcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f364ffcc
Branch: refs/heads/0.7
Commit: f364ffcc6919ca186de326f81858c05ea87eb9a5
Parents: ce77dd2
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Jul 6 14:01:50 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jul 6 14:01:50 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query08.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f364ffcc/query/src/test/resources/query/sql/query08.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query08.sql b/query/src/test/resources/query/sql/query08.sql
index 2f64c7e..7a31541 100644
--- a/query/src/test/resources/query/sql/query08.sql
+++ b/query/src/test/resources/query/sql/query08.sql
@@ -18,5 +18,5 @@
select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV
, count(*) as TRANS_CNT from test_kylin_fact
- where test_kylin_fact.lstg_format_name='FP-GTC'
+ where not (test_kylin_fact.lstg_format_name='FP-GTC')
group by test_kylin_fact.lstg_format_name
[23/50] [abbrv] incubator-kylin git commit: KYLIN-792 ,
add performance module
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/webapp/app/css/AdminLTE.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index ac467b8..e91795e 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -1,116 +1,143 @@
+@import url(https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,600,700,300italic,400italic,600italic);
/*!
- * AdminLTE v1.2
- * Author: AlmsaeedStudio.com
+ * AdminLTE v2.1.0
+ * Author: Almsaeed Studio
+ * Website: Almsaeed Studio <http://almsaeedstudio.com>
* License: Open source - MIT
* Please visit http://opensource.org/licenses/MIT for more information
!*/
/*
- Core: General style
-----------------------------
-*/
+ * Core: General Layout Style
+ * -------------------------
+ */
html,
body {
- font-family: 'Source Sans Pro', sans-serif;
- -webkit-font-smoothing: antialiased;
min-height: 100%;
}
-a {
- color: #3c8dbc;
+.layout-boxed html,
+.layout-boxed body {
+ height: 100%;
}
-a:hover,
-a:active,
-a:focus {
- outline: none;
- text-decoration: none;
- color: #72afd2;
+body {
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+ font-family: 'Source Sans Pro', 'Helvetica Neue', Helvetica, Arial, sans-serif;
+ font-weight: 400;
+ overflow-x: hidden;
+ overflow-y: auto;
}
-/* Layouts */
+/* Layout */
.wrapper {
min-height: 100%;
+ position: relative;
+ overflow: hidden!important;
}
.wrapper:before,
.wrapper:after {
- display: table;
content: " ";
+ display: table;
}
.wrapper:after {
clear: both;
}
-/* Header */
-body > .header {
- position: absolute;
- top: 0;
- left: 0;
- right: 0;
- z-index: 1030;
-}
-/* Define 2 column template */
-.right-side,
-.left-side {
+.layout-boxed .wrapper {
+ max-width: 1250px;
+ margin: 0 auto;
min-height: 100%;
- display: block;
+ box-shadow: 0 0 8px rgba(0, 0, 0, 0.5);
+ position: relative;
}
-/*right side - contins main content*/
-.right-side {
- background-color: #f9f9f9;
- margin-left: 220px;
+.layout-boxed {
+ background: url('../img/boxed-bg.jpg') repeat fixed;
}
-/*left side - contains sidebar*/
-.left-side {
- position: absolute;
- width: 220px;
- top: 0;
+/*
+ * Content Wrapper - contains the main content
+ * ```.right-side has been deprecated as of v2.0.0 in favor of .content-wrapper ```
+ */
+.content-wrapper,
+.right-side,
+.main-footer {
+ -webkit-transition: -webkit-transform 0.3s ease-in-out, margin 0.3s ease-in-out;
+ -moz-transition: -moz-transform 0.3s ease-in-out, margin 0.3s ease-in-out;
+ -o-transition: -o-transform 0.3s ease-in-out, margin 0.3s ease-in-out;
+ transition: transform 0.3s ease-in-out, margin 0.3s ease-in-out;
+ margin-left: 230px;
+ z-index: 820;
+}
+.layout-top-nav .content-wrapper,
+.layout-top-nav .right-side,
+.layout-top-nav .main-footer {
+ margin-left: 0;
}
-@media screen and (min-width: 992px) {
- .left-side {
- top: 50px;
- }
- /*Right side strech mode*/
- .right-side.strech {
+@media (max-width: 767px) {
+ .content-wrapper,
+ .right-side,
+ .main-footer {
margin-left: 0;
}
- .right-side.strech > .content-header {
- margin-top: 0px;
- }
- /* Left side collapse */
- .left-side.collapse-left {
- left: -220px;
- }
}
-/*Give content full width on xs screens*/
-@media screen and (max-width: 992px) {
- .right-side {
+@media (min-width: 768px) {
+ .sidebar-collapse .content-wrapper,
+ .sidebar-collapse .right-side,
+ .sidebar-collapse .main-footer {
margin-left: 0;
}
}
-/*
- By default the layout is not fixed but if you add the class .fixed to the body element
- the sidebar and the navbar will automatically become poisitioned fixed
-*/
-body.fixed > .header,
-body.fixed .left-side,
-body.fixed .navbar {
+@media (max-width: 767px) {
+ .sidebar-open .content-wrapper,
+ .sidebar-open .right-side,
+ .sidebar-open .main-footer {
+ -webkit-transform: translate(230px, 0);
+ -ms-transform: translate(230px, 0);
+ -o-transform: translate(230px, 0);
+ transform: translate(230px, 0);
+ }
+}
+.content-wrapper,
+.right-side {
+ min-height: 100%;
+ background-color: #ecf0f5;
+ z-index: 800;
+}
+.main-footer {
+ background: #fff;
+ padding: 15px;
+ color: #444;
+ border-top: 1px solid #d2d6de;
+}
+/* Fixed layout */
+.fixed .main-header,
+.fixed .main-sidebar,
+.fixed .left-side {
position: fixed;
}
-body.fixed > .header {
+.fixed .main-header {
top: 0;
right: 0;
left: 0;
}
-body.fixed .navbar {
- left: 0;
- right: 0;
+.fixed .content-wrapper,
+.fixed .right-side {
+ padding-top: 50px;
+}
+@media (max-width: 767px) {
+ .fixed .content-wrapper,
+ .fixed .right-side {
+ padding-top: 100px;
+ }
}
-body.fixed .wrapper {
- margin-top: 50px;
+.fixed.layout-boxed .wrapper {
+ max-width: 100%;
}
/* Content */
.content {
- padding: 20px 15px;
- background: #f9f9f9;
- overflow: auto;
+ min-height: 250px;
+ padding: 15px;
+ margin-right: auto;
+ margin-left: auto;
+ padding-left: 15px;
+ padding-right: 15px;
}
-/* Utility */
/* H1 - H6 font */
h1,
h2,
@@ -126,607 +153,747 @@ h6,
.h6 {
font-family: 'Source Sans Pro', sans-serif;
}
-/* All images should be responsive */
-img {
- max-width: 100% !important;
-}
-.sort-highlight {
- background: #f4f4f4;
- border: 1px dashed #ddd;
- margin-bottom: 10px;
-}
-/* 10px padding and margins */
-.pad {
- padding: 10px;
-}
-.margin {
- margin: 10px;
-}
-/* Display inline */
-.inline {
- display: inline;
- width: auto;
-}
-/* Background colors */
-.bg-red,
-.bg-yellow,
-.bg-aqua,
-.bg-blue,
-.bg-light-blue,
-.bg-green,
-.bg-navy,
-.bg-teal,
-.bg-olive,
-.bg-lime,
-.bg-orange,
-.bg-fuchsia,
-.bg-purple,
-.bg-maroon,
-.bg-black {
- color: #f9f9f9 !important;
-}
-.bg-gray {
- background-color: #eaeaec !important;
+/* General Links */
+a {
+ color: #3c8dbc;
}
-.bg-black {
- background-color: #222222 !important;
+a:hover,
+a:active,
+a:focus {
+ outline: none;
+ text-decoration: none;
+ color: #72afd2;
}
-.bg-red {
- background-color: #f56954 !important;
+/* Page Header */
+.page-header {
+ margin: 10px 0 20px 0;
+ font-size: 22px;
}
-.bg-yellow {
- background-color: #f39c12 !important;
+.page-header > small {
+ color: #666;
+ display: block;
+ margin-top: 5px;
}
-.bg-aqua {
- background-color: #00c0ef !important;
+/*
+ * Component: Main Header
+ * ----------------------
+ */
+.main-header {
+ position: relative;
+ max-height: 100px;
+ z-index: 1030;
}
-.bg-blue {
- background-color: #0073b7 !important;
+.main-header > .navbar {
+ -webkit-transition: margin-left 0.3s ease-in-out;
+ -o-transition: margin-left 0.3s ease-in-out;
+ transition: margin-left 0.3s ease-in-out;
+ margin-bottom: 0;
+ margin-left: 230px;
+ border: none;
+ min-height: 50px;
+ border-radius: 0;
}
-.bg-light-blue {
- background-color: #3c8dbc !important;
+.layout-top-nav .main-header > .navbar {
+ margin-left: 0!important;
}
-.bg-green {
- background-color: #00a65a !important;
+.main-header #navbar-search-input {
+ background: rgba(255, 255, 255, 0.2);
+ border-color: transparent;
}
-.bg-navy {
- background-color: #001f3f !important;
+.main-header #navbar-search-input:focus,
+.main-header #navbar-search-input:active {
+ border-color: rgba(0, 0, 0, 0.1) !important;
+ background: rgba(255, 255, 255, 0.9);
}
-.bg-teal {
- background-color: #39cccc !important;
+.main-header #navbar-search-input::-moz-placeholder {
+ color: #ccc;
+ opacity: 1;
}
-.bg-olive {
- background-color: #3d9970 !important;
+.main-header #navbar-search-input:-ms-input-placeholder {
+ color: #ccc;
}
-.bg-lime {
- background-color: #01ff70 !important;
+.main-header #navbar-search-input::-webkit-input-placeholder {
+ color: #ccc;
}
-.bg-orange {
- background-color: #ff851b !important;
+.main-header .navbar-custom-menu,
+.main-header .navbar-right {
+ float: right;
}
-.bg-fuchsia {
- background-color: #f012be !important;
+@media (max-width: 991px) {
+ .main-header .navbar-custom-menu a,
+ .main-header .navbar-right a {
+ color: inherit;
+ background: transparent;
+ }
}
-.bg-purple {
- background-color: #932ab6 !important;
+@media (max-width: 767px) {
+ .main-header .navbar-right {
+ float: none;
+ }
+ .navbar-collapse .main-header .navbar-right {
+ margin: 7.5px -15px;
+ }
+ .main-header .navbar-right > li {
+ color: inherit;
+ border: 0;
+ }
}
-.bg-maroon {
- background-color: #85144b !important;
+.main-header .sidebar-toggle {
+ float: left;
+ background-color: transparent;
+ background-image: none;
+ padding: 15px 15px;
+ font-family: fontAwesome;
}
-/* Text colors */
-.text-red {
- color: #f56954 !important;
+.main-header .sidebar-toggle:before {
+ content: "\f0c9";
}
-.text-yellow {
- color: #f39c12 !important;
+.main-header .sidebar-toggle:hover {
+ color: #fff;
}
-.text-aqua {
- color: #00c0ef !important;
+.main-header .sidebar-toggle:focus,
+.main-header .sidebar-toggle:active {
+ background: transparent;
}
-.text-blue {
- color: #0073b7 !important;
+.main-header .sidebar-toggle .icon-bar {
+ display: none;
}
-.text-black {
- color: #222222 !important;
+.main-header .navbar .nav > li.user > a > .fa,
+.main-header .navbar .nav > li.user > a > .glyphicon,
+.main-header .navbar .nav > li.user > a > .ion {
+ margin-right: 5px;
}
-.text-light-blue {
- color: #3c8dbc !important;
+.main-header .navbar .nav > li > a > .label {
+ position: absolute;
+ top: 9px;
+ right: 7px;
+ text-align: center;
+ font-size: 9px;
+ padding: 2px 3px;
+ line-height: .9;
}
-.text-green {
- color: #00a65a !important;
+.main-header .logo {
+ -webkit-transition: width 0.3s ease-in-out;
+ -o-transition: width 0.3s ease-in-out;
+ transition: width 0.3s ease-in-out;
+ display: block;
+ float: left;
+ height: 50px;
+ font-size: 20px;
+ line-height: 50px;
+ text-align: center;
+ width: 230px;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ padding: 0 15px;
+ font-weight: 300;
+ overflow: hidden;
}
-.text-navy {
- color: #001f3f !important;
+.main-header .logo .logo-lg {
+ display: block;
}
-.text-teal {
- color: #39cccc !important;
+.main-header .logo .logo-mini {
+ display: none;
}
-.text-olive {
- color: #3d9970 !important;
+.main-header .navbar-brand {
+ color: #fff;
}
-.text-lime {
- color: #01ff70 !important;
+.content-header {
+ position: relative;
+ padding: 15px 15px 0 15px;
}
-.text-orange {
- color: #ff851b !important;
+.content-header > h1 {
+ margin: 0;
+ font-size: 24px;
}
-.text-fuchsia {
- color: #f012be !important;
+.content-header > h1 > small {
+ font-size: 15px;
+ display: inline-block;
+ padding-left: 4px;
+ font-weight: 300;
}
-.text-purple {
- color: #932ab6 !important;
+.content-header > .breadcrumb {
+ float: right;
+ background: transparent;
+ margin-top: 0px;
+ margin-bottom: 0;
+ font-size: 12px;
+ padding: 7px 5px;
+ position: absolute;
+ top: 15px;
+ right: 10px;
+ border-radius: 2px;
}
-.text-maroon {
- color: #85144b !important;
+.content-header > .breadcrumb > li > a {
+ color: #444;
+ text-decoration: none;
+ display: inline-block;
}
-/*Hide elements by display none only*/
-.hide {
- display: none !important;
+.content-header > .breadcrumb > li > a > .fa,
+.content-header > .breadcrumb > li > a > .glyphicon,
+.content-header > .breadcrumb > li > a > .ion {
+ margin-right: 5px;
}
-/* Remove borders */
-.no-border {
- border: 0px !important;
+.content-header > .breadcrumb > li + li:before {
+ content: '>\00a0';
}
-/* Remove padding */
-.no-padding {
- padding: 0px !important;
+@media (max-width: 991px) {
+ .content-header > .breadcrumb {
+ position: relative;
+ margin-top: 5px;
+ top: 0;
+ right: 0;
+ float: none;
+ background: #d2d6de;
+ padding-left: 10px;
+ }
+ .content-header > .breadcrumb li:before {
+ color: #97a0b3;
+ }
}
-/* Remove margins */
-.no-margin {
- margin: 0px !important;
+.navbar-toggle {
+ color: #fff;
+ border: 0;
+ margin: 0;
+ padding: 15px 15px;
}
-/* Remove box shadow */
-.no-shadow {
- box-shadow: none!important;
+@media (max-width: 991px) {
+ .navbar-custom-menu .navbar-nav > li {
+ float: left;
+ }
+ .navbar-custom-menu .navbar-nav {
+ margin: 0;
+ float: left;
+ }
+ .navbar-custom-menu .navbar-nav > li > a {
+ padding-top: 15px;
+ padding-bottom: 15px;
+ line-height: 20px;
+ }
}
-/* Don't display when printing */
-@media print {
- .no-print {
- display: none;
+@media (max-width: 767px) {
+ .main-header {
+ position: relative;
}
- .left-side,
- .header,
- .content-header {
- display: none;
+ .main-header .logo,
+ .main-header .navbar {
+ width: 100%;
+ float: none;
+ position: relative!important;
}
- .right-side {
+ .main-header .navbar {
margin: 0;
}
+ .main-header .navbar-custom-menu {
+ float: right;
+ }
+ .main-sidebar,
+ .left-side {
+ padding-top: 100px!important;
+ }
}
-/* Remove border radius */
-.flat {
- -webkit-border-radius: 0 !important;
- -moz-border-radius: 0 !important;
- border-radius: 0 !important;
-}
-/* Change the color of the striped tables */
-.table-striped > tbody > tr:nth-child(odd) > td,
-.table-striped > tbody > tr:nth-child(odd) > th {
- background-color: #f3f4f5;
-}
-.table.no-border,
-.table.no-border td,
-.table.no-border th {
- border: 0;
-}
-/* .text-center in tables */
-table.text-center,
-table.text-center td,
-table.text-center th {
- text-align: center;
-}
-.table.align th {
- text-align: left;
+@media (max-width: 991px) {
+ .navbar-collapse.pull-left {
+ float: none!important;
+ }
+ .navbar-collapse.pull-left + .navbar-custom-menu {
+ display: block;
+ position: absolute;
+ top: 0;
+ right: 40px;
+ }
}
-.table.align td {
- text-align: right;
+/*
+ * Component: Sidebar
+ * ------------------
+ */
+.main-sidebar,
+.left-side {
+ position: absolute;
+ top: 0;
+ left: 0;
+ padding-top: 50px;
+ min-height: 100%;
+ width: 230px;
+ z-index: 810;
+ -webkit-transition: -webkit-transform 0.3s ease-in-out, width 0.3s ease-in-out;
+ -moz-transition: -moz-transform 0.3s ease-in-out, width 0.3s ease-in-out;
+ -o-transition: -o-transform 0.3s ease-in-out, width 0.3s ease-in-out;
+ transition: transform 0.3s ease-in-out, width 0.3s ease-in-out;
}
-.text-bold,
-.text-bold.table td,
-.text-bold.table th {
- font-weight: 700;
+@media (max-width: 767px) {
+ .main-sidebar,
+ .left-side {
+ -webkit-transform: translate(-230px, 0);
+ -ms-transform: translate(-230px, 0);
+ -o-transform: translate(-230px, 0);
+ transform: translate(-230px, 0);
+ }
}
-.border-radius-none {
- -webkit-border-radius: 0 !important;
- -moz-border-radius: 0 !important;
- border-radius: 0 !important;
+@media (min-width: 768px) {
+ .sidebar-collapse .main-sidebar,
+ .sidebar-collapse .left-side {
+ -webkit-transform: translate(-230px, 0);
+ -ms-transform: translate(-230px, 0);
+ -o-transform: translate(-230px, 0);
+ transform: translate(-230px, 0);
+ }
}
-/* _fix for sparkline tooltip */
-.jqstooltip {
- padding: 5px!important;
- width: auto!important;
- height: auto!important;
+@media (max-width: 767px) {
+ .sidebar-open .main-sidebar,
+ .sidebar-open .left-side {
+ -webkit-transform: translate(0, 0);
+ -ms-transform: translate(0, 0);
+ -o-transform: translate(0, 0);
+ transform: translate(0, 0);
+ }
}
-/*
-Gradient Background colors
-*/
-.bg-teal-gradient {
- background: #39cccc !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #39cccc), color-stop(1, #7adddd)) !important;
- background: -ms-linear-gradient(bottom, #39cccc, #7adddd) !important;
- background: -moz-linear-gradient(center bottom, #39cccc 0%, #7adddd 100%) !important;
- background: -o-linear-gradient(#7adddd, #39cccc) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#7adddd', endColorstr='#39cccc', GradientType=0) !important;
- color: #fff;
+.sidebar {
+ padding-bottom: 10px;
}
-.bg-light-blue-gradient {
- background: #3c8dbc !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #3c8dbc), color-stop(1, #67a8ce)) !important;
- background: -ms-linear-gradient(bottom, #3c8dbc, #67a8ce) !important;
- background: -moz-linear-gradient(center bottom, #3c8dbc 0%, #67a8ce 100%) !important;
- background: -o-linear-gradient(#67a8ce, #3c8dbc) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#67a8ce', endColorstr='#3c8dbc', GradientType=0) !important;
- color: #fff;
+.sidebar-form input:focus {
+ border-color: transparent!important;
}
-.bg-blue-gradient {
- background: #0073b7 !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #0073b7), color-stop(1, #0089db)) !important;
- background: -ms-linear-gradient(bottom, #0073b7, #0089db) !important;
- background: -moz-linear-gradient(center bottom, #0073b7 0%, #0089db 100%) !important;
- background: -o-linear-gradient(#0089db, #0073b7) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#0089db', endColorstr='#0073b7', GradientType=0) !important;
- color: #fff;
+.user-panel {
+ position: relative;
+ width: 100%;
+ padding: 10px;
+ overflow: hidden;
}
-.bg-aqua-gradient {
- background: #00c0ef !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #00c0ef), color-stop(1, #14d1ff)) !important;
- background: -ms-linear-gradient(bottom, #00c0ef, #14d1ff) !important;
- background: -moz-linear-gradient(center bottom, #00c0ef 0%, #14d1ff 100%) !important;
- background: -o-linear-gradient(#14d1ff, #00c0ef) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#14d1ff', endColorstr='#00c0ef', GradientType=0) !important;
- color: #fff;
+.user-panel:before,
+.user-panel:after {
+ content: " ";
+ display: table;
}
-.bg-yellow-gradient {
- background: #f39c12 !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #f39c12), color-stop(1, #f7bc60)) !important;
- background: -ms-linear-gradient(bottom, #f39c12, #f7bc60) !important;
- background: -moz-linear-gradient(center bottom, #f39c12 0%, #f7bc60 100%) !important;
- background: -o-linear-gradient(#f7bc60, #f39c12) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#f7bc60', endColorstr='#f39c12', GradientType=0) !important;
- color: #fff;
+.user-panel:after {
+ clear: both;
}
-.bg-purple-gradient {
- background: #932ab6 !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #932ab6), color-stop(1, #b959d9)) !important;
- background: -ms-linear-gradient(bottom, #932ab6, #b959d9) !important;
- background: -moz-linear-gradient(center bottom, #932ab6 0%, #b959d9 100%) !important;
- background: -o-linear-gradient(#b959d9, #932ab6) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b959d9', endColorstr='#932ab6', GradientType=0) !important;
- color: #fff;
+.user-panel > .image > img {
+ width: 100%;
+ max-width: 45px;
+ height: auto;
}
-.bg-green-gradient {
- background: #00a65a !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #00a65a), color-stop(1, #00ca6d)) !important;
- background: -ms-linear-gradient(bottom, #00a65a, #00ca6d) !important;
- background: -moz-linear-gradient(center bottom, #00a65a 0%, #00ca6d 100%) !important;
- background: -o-linear-gradient(#00ca6d, #00a65a) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#00ca6d', endColorstr='#00a65a', GradientType=0) !important;
- color: #fff;
+.user-panel > .info {
+ padding: 5px 5px 5px 15px;
+ line-height: 1;
+ position: absolute;
+ left: 55px;
}
-.bg-red-gradient {
- background: #f56954 !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #f56954), color-stop(1, #f89384)) !important;
- background: -ms-linear-gradient(bottom, #f56954, #f89384) !important;
- background: -moz-linear-gradient(center bottom, #f56954 0%, #f89384 100%) !important;
- background: -o-linear-gradient(#f89384, #f56954) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#f89384', endColorstr='#f56954', GradientType=0) !important;
- color: #fff;
+.user-panel > .info > p {
+ font-weight: 600;
+ margin-bottom: 9px;
}
-.bg-black-gradient {
- background: #222222 !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #222222), color-stop(1, #3c3c3c)) !important;
- background: -ms-linear-gradient(bottom, #222222, #3c3c3c) !important;
- background: -moz-linear-gradient(center bottom, #222222 0%, #3c3c3c 100%) !important;
- background: -o-linear-gradient(#3c3c3c, #222222) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#3c3c3c', endColorstr='#222222', GradientType=0) !important;
- color: #fff;
+.user-panel > .info > a {
+ text-decoration: none;
+ padding-right: 5px;
+ margin-top: 3px;
+ font-size: 11px;
}
-.bg-maroon-gradient {
- background: #85144b !important;
- background: -webkit-gradient(linear, left bottom, left top, color-stop(0, #85144b), color-stop(1, #b11b64)) !important;
- background: -ms-linear-gradient(bottom, #85144b, #b11b64) !important;
- background: -moz-linear-gradient(center bottom, #85144b 0%, #b11b64 100%) !important;
- background: -o-linear-gradient(#b11b64, #85144b) !important;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b11b64', endColorstr='#85144b', GradientType=0) !important;
- color: #fff;
+.user-panel > .info > a > .fa,
+.user-panel > .info > a > .ion,
+.user-panel > .info > a > .glyphicon {
+ margin-right: 3px;
}
-.connectedSortable {
- min-height: 100px;
+.sidebar-menu {
+ list-style: none;
+ margin: 0;
+ padding: 0;
}
-/*---------------------------------------------------
- LESS Elements 0.9
- ---------------------------------------------------
- A set of useful LESS mixins
- More info at: http://lesselements.com
- ---------------------------------------------------*/
-/*
- Components: navbar, logo and content header
--------------------------------------------------
-*/
-body > .header {
+.sidebar-menu > li {
position: relative;
- max-height: 100px;
- z-index: 1030;
-}
-body > .header .navbar {
- height: 50px;
- margin-bottom: 0;
- margin-left: 220px;
-}
-body > .header .navbar .sidebar-toggle {
- float: left;
- padding: 9px 5px;
- margin-top: 8px;
- margin-right: 0;
- margin-bottom: 8px;
- margin-left: 5px;
- background-color: transparent;
- background-image: none;
- border: 1px solid transparent;
- -webkit-border-radius: 0 !important;
- -moz-border-radius: 0 !important;
- border-radius: 0 !important;
-}
-body > .header .navbar .sidebar-toggle:hover .icon-bar {
- background: #f6f6f6;
+ margin: 0;
+ padding: 0;
}
-body > .header .navbar .sidebar-toggle .icon-bar {
+.sidebar-menu > li > a {
+ padding: 12px 5px 12px 15px;
display: block;
- width: 22px;
- height: 2px;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
-}
-body > .header .navbar .sidebar-toggle .icon-bar + .icon-bar {
- margin-top: 4px;
}
-body > .header .navbar .nav > li.user > a {
- font-weight: bold;
+.sidebar-menu > li > a > .fa,
+.sidebar-menu > li > a > .glyphicon,
+.sidebar-menu > li > a > .ion {
+ width: 20px;
}
-body > .header .navbar .nav > li.user > a > .fa,
-body > .header .navbar .nav > li.user > a > .glyphicon,
-body > .header .navbar .nav > li.user > a > .ion {
+.sidebar-menu > li .label,
+.sidebar-menu > li .badge {
+ margin-top: 3px;
margin-right: 5px;
}
-body > .header .navbar .nav > li > a > .label {
- -webkit-border-radius: 50%;
- -moz-border-radius: 50%;
- border-radius: 50%;
- position: absolute;
- top: 7px;
- right: 2px;
- font-size: 10px;
- font-weight: normal;
- width: 15px;
- height: 15px;
- line-height: 1.0em;
- text-align: center;
- padding: 2px;
-}
-body > .header .navbar .nav > li > a:hover > .label {
- top: 3px;
-}
-body > .header .logo {
- float: left;
- font-size: 20px;
- line-height: 50px;
- text-align: center;
- padding: 0 10px;
- width: 220px;
- font-family: 'Kaushan Script', cursive;
- font-weight: 500;
- height: 50px;
- display: block;
+.sidebar-menu li.header {
+ padding: 10px 25px 10px 15px;
+ font-size: 12px;
}
-body > .header .logo .icon {
+.sidebar-menu li > a > .fa-angle-left {
+ width: auto;
+ height: auto;
+ padding: 0;
margin-right: 10px;
+ margin-top: 3px;
}
-.right-side > .content-header {
- position: relative;
- padding: 15px 15px 10px 20px;
+.sidebar-menu li.active > a > .fa-angle-left {
+ -webkit-transform: rotate(-90deg);
+ -ms-transform: rotate(-90deg);
+ -o-transform: rotate(-90deg);
+ transform: rotate(-90deg);
+}
+.sidebar-menu li.active > .treeview-menu {
+ display: block;
}
-.right-side > .content-header > h1 {
+.sidebar-menu .treeview-menu {
+ display: none;
+ list-style: none;
+ padding: 0;
margin: 0;
- font-size: 24px;
+ padding-left: 5px;
}
-.right-side > .content-header > h1 > small {
- font-size: 15px;
- display: inline-block;
- padding-left: 4px;
- font-weight: 300;
+.sidebar-menu .treeview-menu .treeview-menu {
+ padding-left: 20px;
}
-.right-side > .content-header > .breadcrumb {
- float: right;
- background: transparent;
- margin-top: 0px;
- margin-bottom: 0;
- font-size: 12px;
- padding: 7px 5px;
- position: absolute;
- top: 15px;
- right: 10px;
- -webkit-border-radius: 2px;
- -moz-border-radius: 2px;
- border-radius: 2px;
+.sidebar-menu .treeview-menu > li {
+ margin: 0;
}
-.right-side > .content-header > .breadcrumb > li > a {
- color: #444;
- text-decoration: none;
+.sidebar-menu .treeview-menu > li > a {
+ padding: 5px 5px 5px 15px;
+ display: block;
+ font-size: 14px;
}
-.right-side > .content-header > .breadcrumb > li > a > .fa,
-.right-side > .content-header > .breadcrumb > li > a > .glyphicon,
-.right-side > .content-header > .breadcrumb > li > a > .ion {
- margin-right: 5px;
+.sidebar-menu .treeview-menu > li > a > .fa,
+.sidebar-menu .treeview-menu > li > a > .glyphicon,
+.sidebar-menu .treeview-menu > li > a > .ion {
+ width: 20px;
}
-.right-side > .content-header > .breadcrumb > li + li:before {
- content: '>\00a0';
+.sidebar-menu .treeview-menu > li > a > .fa-angle-left,
+.sidebar-menu .treeview-menu > li > a > .fa-angle-down {
+ width: auto;
}
-@media screen and (max-width: 767px) {
- .right-side > .content-header > .breadcrumb {
+/*
+ * Component: Sidebar Mini
+ */
+@media (min-width: 768px) {
+ .sidebar-mini.sidebar-collapse .content-wrapper,
+ .sidebar-mini.sidebar-collapse .right-side,
+ .sidebar-mini.sidebar-collapse .main-footer {
+ margin-left: 50px!important;
+ z-index: 840;
+ }
+ .sidebar-mini.sidebar-collapse .main-sidebar {
+ -webkit-transform: translate(0, 0);
+ -ms-transform: translate(0, 0);
+ -o-transform: translate(0, 0);
+ transform: translate(0, 0);
+ width: 50px!important;
+ z-index: 850;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li {
position: relative;
- margin-top: 5px;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > a {
+ margin-right: 0;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > a > span {
+ border-top-right-radius: 4px;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li:not(.treeview) > a > span {
+ border-bottom-right-radius: 4px;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > .treeview-menu {
+ padding-top: 5px;
+ padding-bottom: 5px;
+ border-bottom-right-radius: 4px;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li:hover > a > span:not(.pull-right),
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li:hover > .treeview-menu {
+ display: block!important;
+ position: absolute;
+ width: 180px;
+ left: 50px;
+ }
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li:hover > a > span {
top: 0;
- right: 0;
- float: none;
- background: #efefef;
+ margin-left: -3px;
+ padding: 12px 5px 12px 20px;
+ background-color: inherit;
}
-}
-
-@media screen and (max-width: 560px) {
- body > .header {
- position: relative;
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li:hover > .treeview-menu {
+ top: 44px;
+ margin-left: 0;
}
- body > .header .logo,
- body > .header .navbar {
- width: 100%;
- float: none;
- position: relative!important;
+ .sidebar-mini.sidebar-collapse .main-sidebar .user-panel > .info,
+ .sidebar-mini.sidebar-collapse .sidebar-form,
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > a > span,
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > .treeview-menu,
+ .sidebar-mini.sidebar-collapse .sidebar-menu > li > a > .pull-right,
+ .sidebar-mini.sidebar-collapse .sidebar-menu li.header {
+ display: none!important;
}
- body > .header .navbar {
- margin: 0;
+ .sidebar-mini.sidebar-collapse .main-header .logo {
+ width: 50px;
}
- body.fixed > .header {
- position: fixed;
+ .sidebar-mini.sidebar-collapse .main-header .logo > .logo-mini {
+ display: block;
+ margin-left: -15px;
+ margin-right: -15px;
+ font-size: 18px;
}
- body.fixed > .wrapper,
- body.fixed .sidebar-offcanvas {
- margin-top: 100px!important;
+ .sidebar-mini.sidebar-collapse .main-header .logo > .logo-lg {
+ display: none;
+ }
+ .sidebar-mini.sidebar-collapse .main-header .navbar {
+ margin-left: 50px;
}
}
-/*
- Component: Sidebar
---------------------------
-*/
-.sidebar {
- margin-bottom: 5px;
-}
-.sidebar .sidebar-form input:focus {
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
- box-shadow: none;
- border-color: transparent!important;
+.sidebar-menu,
+.main-sidebar .user-panel,
+.sidebar-menu > li.header {
+ white-space: nowrap!important;
+ overflow: hidden;
}
-.sidebar .sidebar-menu {
- list-style: none;
- margin: 0;
- padding: 0;
+.sidebar-menu:hover {
+ overflow: visible;
}
-.sidebar .sidebar-menu > li {
- margin: 0;
- padding: 0;
+.sidebar-form,
+.sidebar-menu > li.header {
+ overflow: hidden;
+ text-overflow: clip;
}
-.sidebar .sidebar-menu > li > a {
- padding: 12px 5px 12px 15px;
- display: block;
+.sidebar-menu li > a {
+ position: relative;
}
-.sidebar .sidebar-menu > li > a > .fa,
-.sidebar .sidebar-menu > li > a > .glyphicon,
-.sidebar .sidebar-menu > li > a > .ion {
- width: 20px;
+.sidebar-menu li > a > .pull-right {
+ position: absolute;
+ top: 50%;
+ right: 10px;
+ margin-top: -7px;
}
-.sidebar .sidebar-menu .treeview-menu {
- display: none;
- list-style: none;
- padding: 0;
- margin: 0;
+/*
+ * Component: Control sidebar. By default, this is the right sidebar.
+ */
+.control-sidebar-bg {
+ position: fixed;
+ z-index: 1000;
+ bottom: 0;
}
-.sidebar .sidebar-menu .treeview-menu > li {
- margin: 0;
+.control-sidebar-bg,
+.control-sidebar {
+ top: 0;
+ right: -230px;
+ width: 230px;
+ -webkit-transition: right 0.3s ease-in-out;
+ -o-transition: right 0.3s ease-in-out;
+ transition: right 0.3s ease-in-out;
}
-.sidebar .sidebar-menu .treeview-menu > li > a {
- padding: 5px 5px 5px 15px;
+.control-sidebar {
+ position: absolute;
+ padding-top: 50px;
+ z-index: 1010;
+}
+@media (max-width: 768px) {
+ .control-sidebar {
+ padding-top: 100px;
+ }
+}
+.control-sidebar > .tab-content {
+ padding: 10px 15px;
+}
+.control-sidebar.control-sidebar-open,
+.control-sidebar.control-sidebar-open + .control-sidebar-bg {
+ right: 0;
+}
+.control-sidebar-open .control-sidebar-bg,
+.control-sidebar-open .control-sidebar {
+ right: 0;
+}
+@media (min-width: 768px) {
+ .control-sidebar-open .content-wrapper,
+ .control-sidebar-open .right-side,
+ .control-sidebar-open .main-footer {
+ margin-right: 230px;
+ }
+}
+.control-sidebar-tabs > li:first-of-type > a {
+ margin-left: 1px;
+}
+.control-sidebar-tabs > li:first-of-type > a,
+.control-sidebar-tabs > li:first-of-type > a:hover {
+ border-left-width: 0!important;
+}
+.control-sidebar-tabs > li > a {
+ border-radius: 0 !important;
+}
+.control-sidebar-tabs > li > a,
+.control-sidebar-tabs > li > a:hover {
+ border-top: none;
+ border-right: none;
+ border-left: 1px solid transparent!important;
+ border-bottom: 1px solid transparent!important;
+}
+.control-sidebar-tabs > li > a .icon {
+ font-size: 16px;
+}
+.control-sidebar-tabs > li.active > a,
+.control-sidebar-tabs > li.active > a:hover,
+.control-sidebar-tabs > li.active > a:focus,
+.control-sidebar-tabs > li.active > a:active {
+ border-top: none!important;
+ border-right: none!important;
+ border-bottom: none!important;
+}
+@media (max-width: 768px) {
+ .control-sidebar-tabs {
+ display: table;
+ }
+ .control-sidebar-tabs > li {
+ display: table-cell !important;
+ }
+}
+.control-sidebar-heading {
+ font-weight: 400;
+ font-size: 16px;
+ padding: 10px 0;
+ margin-bottom: 10px;
+}
+.control-sidebar-subheading {
display: block;
+ font-weight: 400;
font-size: 14px;
- margin: 0px 0px;
}
-.sidebar .sidebar-menu .treeview-menu > li > a > .fa,
-.sidebar .sidebar-menu .treeview-menu > li > a > .glyphicon,
-.sidebar .sidebar-menu .treeview-menu > li > a > .ion {
- width: 20px;
+.control-sidebar-menu {
+ list-style: none;
+ padding: 0;
+ margin: 0 -15px;
}
-.user-panel {
- padding: 10px;
+.control-sidebar-menu > li > a {
+ display: block;
+ padding: 10px 15px;
}
-.user-panel:before,
-.user-panel:after {
- display: table;
+.control-sidebar-menu > li > a:before,
+.control-sidebar-menu > li > a:after {
content: " ";
+ display: table;
}
-.user-panel:after {
+.control-sidebar-menu > li > a:after {
clear: both;
}
-.user-panel > .image > img {
- width: 45px;
- height: 45px;
-}
-.user-panel > .info {
- font-weight: 600;
- padding: 5px 5px 5px 15px;
- font-size: 14px;
- line-height: 1;
+.control-sidebar-menu > li > a > .control-sidebar-subheading {
+ margin-top: 0;
}
-.user-panel > .info > p {
- margin-bottom: 9px;
+.control-sidebar-menu .menu-icon {
+ float: left;
+ width: 35px;
+ height: 35px;
+ border-radius: 50%;
+ text-align: center;
+ line-height: 35px;
}
-.user-panel > .info > a {
- text-decoration: none;
- padding-right: 5px;
+.control-sidebar-menu .menu-info {
+ margin-left: 45px;
margin-top: 3px;
+}
+.control-sidebar-menu .menu-info > .control-sidebar-subheading {
+ margin: 0;
+}
+.control-sidebar-menu .menu-info > p {
+ margin: 0;
font-size: 11px;
- font-weight: normal;
}
-.user-panel > .info > a > .fa,
-.user-panel > .info > a > .ion,
-.user-panel > .info > a > .glyphicon {
- margin-right: 3px;
+.control-sidebar-menu .progress {
+ margin: 0;
}
-/*
- * Off Canvas
- * --------------------------------------------------
- * Gives us the push menu effect
- */
-@media screen and (max-width: 992px) {
- .relative {
- position: relative;
- }
- .row-offcanvas-right .sidebar-offcanvas {
- right: -220px;
- }
- .row-offcanvas-left .sidebar-offcanvas {
- left: -220px;
- }
- .row-offcanvas-right.active {
- right: 220px;
- }
- .row-offcanvas-left.active {
- left: 220px;
- }
- .sidebar-offcanvas {
- left: 0;
- }
- body.fixed .sidebar-offcanvas {
- margin-top: 50px;
- left: -220px;
- }
- body.fixed .row-offcanvas-left.active .navbar {
- left: 220px !important;
- right: 0;
- }
- body.fixed .row-offcanvas-left.active .sidebar-offcanvas {
- left: 0px;
- }
+.control-sidebar-dark {
+ color: #b8c7ce;
+}
+.control-sidebar-dark,
+.control-sidebar-dark + .control-sidebar-bg {
+ background: #222d32;
+}
+.control-sidebar-dark .control-sidebar-tabs {
+ border-bottom: #1c2529;
+}
+.control-sidebar-dark .control-sidebar-tabs > li > a {
+ background: #181f23;
+ color: #b8c7ce;
+}
+.control-sidebar-dark .control-sidebar-tabs > li > a,
+.control-sidebar-dark .control-sidebar-tabs > li > a:hover {
+ border-left-color: #141a1d !important;
+ border-bottom-color: #141a1d !important;
+}
+.control-sidebar-dark .control-sidebar-tabs > li > a:hover,
+.control-sidebar-dark .control-sidebar-tabs > li > a:focus,
+.control-sidebar-dark .control-sidebar-tabs > li > a:active {
+ background: #1c2529;
+}
+.control-sidebar-dark .control-sidebar-tabs > li.active > a,
+.control-sidebar-dark .control-sidebar-tabs > li.active > a:hover,
+.control-sidebar-dark .control-sidebar-tabs > li.active > a:focus,
+.control-sidebar-dark .control-sidebar-tabs > li.active > a:active {
+ background: #222d32;
+ color: #fff;
+}
+.control-sidebar-dark .control-sidebar-heading,
+.control-sidebar-dark .control-sidebar-subheading {
+ color: #fff;
+}
+.control-sidebar-dark .control-sidebar-menu > li > a:hover {
+ background: #1e282c;
+}
+.control-sidebar-dark .control-sidebar-menu > li > a .menu-info > p {
+ color: #b8c7ce;
+}
+.control-sidebar-light {
+ color: #5e5e5e;
+}
+.control-sidebar-light,
+.control-sidebar-light + .control-sidebar-bg {
+ background: #f9fafc;
+ border-left: 1px solid #d2d6de;
+}
+.control-sidebar-light .control-sidebar-tabs {
+ border-bottom: #d2d6de;
+}
+.control-sidebar-light .control-sidebar-tabs > li > a {
+ background: #e8ecf4;
+ color: #444444;
+}
+.control-sidebar-light .control-sidebar-tabs > li > a,
+.control-sidebar-light .control-sidebar-tabs > li > a:hover {
+ border-left-color: #d2d6de !important;
+ border-bottom-color: #d2d6de !important;
+}
+.control-sidebar-light .control-sidebar-tabs > li > a:hover,
+.control-sidebar-light .control-sidebar-tabs > li > a:focus,
+.control-sidebar-light .control-sidebar-tabs > li > a:active {
+ background: #eff1f7;
+}
+.control-sidebar-light .control-sidebar-tabs > li.active > a,
+.control-sidebar-light .control-sidebar-tabs > li.active > a:hover,
+.control-sidebar-light .control-sidebar-tabs > li.active > a:focus,
+.control-sidebar-light .control-sidebar-tabs > li.active > a:active {
+ background: #f9fafc;
+ color: #111;
+}
+.control-sidebar-light .control-sidebar-heading,
+.control-sidebar-light .control-sidebar-subheading {
+ color: #111;
+}
+.control-sidebar-light .control-sidebar-menu {
+ margin-left: -14px;
+}
+.control-sidebar-light .control-sidebar-menu > li > a:hover {
+ background: #f4f4f5;
+}
+.control-sidebar-light .control-sidebar-menu > li > a .menu-info > p {
+ color: #5e5e5e;
}
/*
- Dropdown menus
-----------------------------
-*/
+ * Component: Dropdown menus
+ * -------------------------
+ */
/*Dropdowns in general*/
.dropdown-menu {
- -webkit-box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.1);
- -moz-box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.1);
- box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.1);
- z-index: 2300;
+ box-shadow: none;
+ border-color: #eee;
+}
+.dropdown-menu > li > a {
+ color: #777;
}
.dropdown-menu > li > a > .glyphicon,
.dropdown-menu > li > a > .fa,
@@ -734,17 +901,21 @@ body > .header .logo .icon {
margin-right: 10px;
}
.dropdown-menu > li > a:hover {
- background-color: #3c8dbc;
- color: #f9f9f9;
+ background-color: #e1e3e9;
+ color: #333;
}
-/*Drodown in navbars*/
-.skin-blue .navbar .dropdown-menu > li > a {
- color: #444444;
+.dropdown-menu > .divider {
+ background-color: #eee;
+}
+.navbar-nav .dropdown-menu {
+ -webkit-box-shadow: none !important;
+ box-shadow: none !important;
+}
+.navbar-nav > .notifications-menu,
+.navbar-nav > .messages-menu,
+.navbar-nav > .tasks-menu {
+ position: relative;
}
-/*
- Navbar custom dropdown menu
-------------------------------------
-*/
.navbar-nav > .notifications-menu > .dropdown-menu,
.navbar-nav > .messages-menu > .dropdown-menu,
.navbar-nav > .tasks-menu > .dropdown-menu {
@@ -752,22 +923,15 @@ body > .header .logo .icon {
padding: 0 0 0 0!important;
margin: 0!important;
top: 100%;
- border: 1px solid #dfdfdf;
- -webkit-border-radius: 4px !important;
- -moz-border-radius: 4px !important;
- border-radius: 4px !important;
+}
+.navbar-nav > .notifications-menu > .dropdown-menu > li,
+.navbar-nav > .messages-menu > .dropdown-menu > li,
+.navbar-nav > .tasks-menu > .dropdown-menu > li {
+ position: relative;
}
.navbar-nav > .notifications-menu > .dropdown-menu > li.header,
.navbar-nav > .messages-menu > .dropdown-menu > li.header,
.navbar-nav > .tasks-menu > .dropdown-menu > li.header {
- -webkit-border-top-left-radius: 4px;
- -webkit-border-top-right-radius: 4px;
- -webkit-border-bottom-right-radius: 0;
- -webkit-border-bottom-left-radius: 0;
- -moz-border-radius-topleft: 4px;
- -moz-border-radius-topright: 4px;
- -moz-border-radius-bottomright: 0;
- -moz-border-radius-bottomleft: 0;
border-top-left-radius: 4px;
border-top-right-radius: 4px;
border-bottom-right-radius: 0;
@@ -778,54 +942,38 @@ body > .header .logo .icon {
color: #444444;
font-size: 14px;
}
-.navbar-nav > .notifications-menu > .dropdown-menu > li.header:after,
-.navbar-nav > .messages-menu > .dropdown-menu > li.header:after,
-.navbar-nav > .tasks-menu > .dropdown-menu > li.header:after {
- bottom: 100%;
- left: 92%;
- border: solid transparent;
- content: " ";
- height: 0;
- width: 0;
- position: absolute;
- pointer-events: none;
- border-color: rgba(255, 255, 255, 0);
- border-bottom-color: #ffffff;
- border-width: 7px;
- margin-left: -7px;
-}
.navbar-nav > .notifications-menu > .dropdown-menu > li.footer > a,
.navbar-nav > .messages-menu > .dropdown-menu > li.footer > a,
.navbar-nav > .tasks-menu > .dropdown-menu > li.footer > a {
- -webkit-border-top-left-radius: 0px;
- -webkit-border-top-right-radius: 0px;
- -webkit-border-bottom-right-radius: 4px;
- -webkit-border-bottom-left-radius: 4px;
- -moz-border-radius-topleft: 0px;
- -moz-border-radius-topright: 0px;
- -moz-border-radius-bottomright: 4px;
- -moz-border-radius-bottomleft: 4px;
border-top-left-radius: 0px;
border-top-right-radius: 0px;
border-bottom-right-radius: 4px;
border-bottom-left-radius: 4px;
font-size: 12px;
- background-color: #f4f4f4;
+ background-color: #fff;
padding: 7px 10px;
border-bottom: 1px solid #eeeeee;
- color: #444444;
+ color: #444!important;
text-align: center;
}
+@media (max-width: 991px) {
+ .navbar-nav > .notifications-menu > .dropdown-menu > li.footer > a,
+ .navbar-nav > .messages-menu > .dropdown-menu > li.footer > a,
+ .navbar-nav > .tasks-menu > .dropdown-menu > li.footer > a {
+ background: #fff!important;
+ color: #444!important;
+ }
+}
.navbar-nav > .notifications-menu > .dropdown-menu > li.footer > a:hover,
.navbar-nav > .messages-menu > .dropdown-menu > li.footer > a:hover,
.navbar-nav > .tasks-menu > .dropdown-menu > li.footer > a:hover {
- background: #f4f4f4;
text-decoration: none;
font-weight: normal;
}
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu,
.navbar-nav > .messages-menu > .dropdown-menu > li .menu,
.navbar-nav > .tasks-menu > .dropdown-menu > li .menu {
+ max-height: 200px;
margin: 0;
padding: 0;
list-style: none;
@@ -842,71 +990,43 @@ body > .header .logo .icon {
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a:hover,
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a:hover,
.navbar-nav > .tasks-menu > .dropdown-menu > li .menu > li > a:hover {
- background: #f6f6f6;
+ background: #f4f4f4;
text-decoration: none;
}
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a {
- font-size: 12px;
color: #444444;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ padding: 10px;
}
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .glyphicon,
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .fa,
.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .ion {
- font-size: 20px;
- width: 50px;
- text-align: center;
- padding: 15px 0px;
- margin-right: 5px;
- /* Default background and font colors */
- background: #00c0ef;
- color: #f9f9f9;
- /* Fallback for browsers that doesn't support rgba */
- color: rgba(255, 255, 255, 0.7);
-}
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .glyphicon.danger,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .fa.danger,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .ion.danger {
- background: #f56954;
-}
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .glyphicon.warning,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .fa.warning,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .ion.warning {
- background: #f39c12;
-}
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .glyphicon.success,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .fa.success,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .ion.success {
- background: #00a65a;
-}
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .glyphicon.info,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .fa.info,
-.navbar-nav > .notifications-menu > .dropdown-menu > li .menu > li > a > .ion.info {
- background: #00c0ef;
+ width: 20px;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a {
margin: 0px;
- line-height: 20px;
- padding: 10px 5px 10px 5px;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
+ padding: 10px 10px;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a > div > img {
margin: auto 10px auto auto;
width: 40px;
height: 40px;
- border: 1px solid #dddddd;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a > h4 {
padding: 0;
margin: 0 0 0 45px;
color: #444444;
font-size: 15px;
+ position: relative;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a > h4 > small {
color: #999999;
font-size: 10px;
- float: right;
+ position: absolute;
+ top: 0px;
+ right: 0px;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a > p {
margin: 0 0 0 45px;
@@ -915,8 +1035,8 @@ body > .header .logo .icon {
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a:before,
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a:after {
- display: table;
content: " ";
+ display: table;
}
.navbar-nav > .messages-menu > .dropdown-menu > li .menu > li > a:after {
clear: both;
@@ -935,180 +1055,196 @@ body > .header .logo .icon {
margin: 0;
}
.navbar-nav > .user-menu > .dropdown-menu {
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
+ border-top-right-radius: 0;
+ border-top-left-radius: 0;
padding: 1px 0 0 0;
border-top-width: 0;
width: 280px;
}
-.navbar-nav > .user-menu > .dropdown-menu:after {
- bottom: 100%;
- right: 10px;
- border: solid transparent;
- content: " ";
- height: 0;
- width: 0;
- position: absolute;
- pointer-events: none;
- border-color: rgba(255, 255, 255, 0);
- border-bottom-color: #ffffff;
- border-width: 10px;
- margin-left: -10px;
+.navbar-nav > .user-menu > .dropdown-menu,
+.navbar-nav > .user-menu > .dropdown-menu > .user-body {
+ border-bottom-right-radius: 4px;
+ border-bottom-left-radius: 4px;
}
.navbar-nav > .user-menu > .dropdown-menu > li.user-header {
height: 175px;
padding: 10px;
- background: #3c8dbc;
text-align: center;
}
.navbar-nav > .user-menu > .dropdown-menu > li.user-header > img {
z-index: 5;
height: 90px;
width: 90px;
- border: 8px solid;
+ border: 3px solid;
border-color: transparent;
border-color: rgba(255, 255, 255, 0.2);
}
.navbar-nav > .user-menu > .dropdown-menu > li.user-header > p {
z-index: 5;
- color: #f9f9f9;
+ color: #fff;
color: rgba(255, 255, 255, 0.8);
font-size: 17px;
- text-shadow: 2px 2px 3px #333333;
margin-top: 10px;
}
.navbar-nav > .user-menu > .dropdown-menu > li.user-header > p > small {
display: block;
font-size: 12px;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-body {
+.navbar-nav > .user-menu > .dropdown-menu > .user-body {
padding: 15px;
border-bottom: 1px solid #f4f4f4;
border-top: 1px solid #dddddd;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-body:before,
-.navbar-nav > .user-menu > .dropdown-menu > li.user-body:after {
- display: table;
+.navbar-nav > .user-menu > .dropdown-menu > .user-body:before,
+.navbar-nav > .user-menu > .dropdown-menu > .user-body:after {
content: " ";
+ display: table;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-body:after {
+.navbar-nav > .user-menu > .dropdown-menu > .user-body:after {
clear: both;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-body > div > a {
- color: #0073b7;
+.navbar-nav > .user-menu > .dropdown-menu > .user-body a {
+ color: #444 !important;
+}
+@media (max-width: 991px) {
+ .navbar-nav > .user-menu > .dropdown-menu > .user-body a {
+ background: #fff !important;
+ color: #444 !important;
+ }
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-footer {
+.navbar-nav > .user-menu > .dropdown-menu > .user-footer {
background-color: #f9f9f9;
padding: 10px;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-footer:before,
-.navbar-nav > .user-menu > .dropdown-menu > li.user-footer:after {
- display: table;
+.navbar-nav > .user-menu > .dropdown-menu > .user-footer:before,
+.navbar-nav > .user-menu > .dropdown-menu > .user-footer:after {
content: " ";
+ display: table;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-footer:after {
+.navbar-nav > .user-menu > .dropdown-menu > .user-footer:after {
clear: both;
}
-.navbar-nav > .user-menu > .dropdown-menu > li.user-footer .btn-default {
+.navbar-nav > .user-menu > .dropdown-menu > .user-footer .btn-default {
color: #666666;
}
-/* Add fade animation to dropdown menus */
-.open > .dropdown-menu {
- animation-name: fadeAnimation;
- animation-duration: .7s;
- animation-iteration-count: 1;
- animation-timing-function: ease;
- animation-fill-mode: forwards;
- -webkit-animation-name: fadeAnimation;
- -webkit-animation-duration: .7s;
- -webkit-animation-iteration-count: 1;
- -webkit-animation-timing-function: ease;
- -webkit-animation-fill-mode: forwards;
- -moz-animation-name: fadeAnimation;
- -moz-animation-duration: .7s;
- -moz-animation-iteration-count: 1;
- -moz-animation-timing-function: ease;
- -moz-animation-fill-mode: forwards;
-}
-@keyframes fadeAnimation {
- from {
+.navbar-nav > .user-menu .user-image {
+ float: left;
+ width: 25px;
+ height: 25px;
+ border-radius: 50%;
+ margin-right: 10px;
+ margin-top: -2px;
+}
+@media (max-width: 767px) {
+ .navbar-nav > .user-menu .user-image {
+ float: none;
+ margin-right: 0;
+ margin-top: -8px;
+ line-height: 10px;
+ }
+}
+/* Add fade animation to dropdown menus by appending
+ the class .animated-dropdown-menu to the .dropdown-menu ul (or ol)*/
+.open:not(.dropup) > .animated-dropdown-menu {
+ backface-visibility: visible !important;
+ -webkit-animation: flipInX 0.7s both;
+ -o-animation: flipInX 0.7s both;
+ animation: flipInX 0.7s both;
+}
+@keyframes flipInX {
+ 0% {
+ transform: perspective(400px) rotate3d(1, 0, 0, 90deg);
+ transition-timing-function: ease-in;
opacity: 0;
- top: 120%;
}
- to {
+ 40% {
+ transform: perspective(400px) rotate3d(1, 0, 0, -20deg);
+ transition-timing-function: ease-in;
+ }
+ 60% {
+ transform: perspective(400px) rotate3d(1, 0, 0, 10deg);
opacity: 1;
- top: 100%;
+ }
+ 80% {
+ transform: perspective(400px) rotate3d(1, 0, 0, -5deg);
+ }
+ 100% {
+ transform: perspective(400px);
}
}
-@-webkit-keyframes fadeAnimation {
- from {
+@-webkit-keyframes flipInX {
+ 0% {
+ -webkit-transform: perspective(400px) rotate3d(1, 0, 0, 90deg);
+ -webkit-transition-timing-function: ease-in;
opacity: 0;
- top: 120%;
}
- to {
- opacity: 1;
- top: 100%;
+ 40% {
+ -webkit-transform: perspective(400px) rotate3d(1, 0, 0, -20deg);
+ -webkit-transition-timing-function: ease-in;
}
-}
-/* Fix dropdown menu for small screens to display correctly on small screens */
-@media screen and (max-width: 767px) {
- .navbar-nav > .notifications-menu > .dropdown-menu,
- .navbar-nav > .user-menu > .dropdown-menu,
- .navbar-nav > .tasks-menu > .dropdown-menu,
- .navbar-nav > .messages-menu > .dropdown-menu {
- position: absolute;
- top: 100%;
- right: 0;
- left: auto;
- border-right: 1px solid #dddddd;
- border-bottom: 1px solid #dddddd;
- border-left: 1px solid #dddddd;
- background: #ffffff;
+ 60% {
+ -webkit-transform: perspective(400px) rotate3d(1, 0, 0, 10deg);
+ opacity: 1;
}
-}
-/* Fix menu positions on xs screens to appear correctly and fully */
-@media screen and (max-width: 480px) {
- .navbar-nav > .notifications-menu > .dropdown-menu > li.header,
- .navbar-nav > .tasks-menu > .dropdown-menu > li.header,
- .navbar-nav > .messages-menu > .dropdown-menu > li.header {
- /* Remove arrow from the top */
+ 80% {
+ -webkit-transform: perspective(400px) rotate3d(1, 0, 0, -5deg);
}
- .navbar-nav > .notifications-menu > .dropdown-menu > li.header:after,
- .navbar-nav > .tasks-menu > .dropdown-menu > li.header:after,
- .navbar-nav > .messages-menu > .dropdown-menu > li.header:after {
- border-width: 0px!important;
+ 100% {
+ -webkit-transform: perspective(400px);
}
- .navbar-nav > .tasks-menu > .dropdown-menu {
- position: absolute;
- right: -120px;
- left: auto;
+}
+/* Fix dropdown menu in navbars */
+.navbar-custom-menu > .navbar-nav > li {
+ position: relative;
+}
+.navbar-custom-menu > .navbar-nav > li > .dropdown-menu {
+ position: absolute;
+ right: 0;
+ left: auto;
+}
+@media (max-width: 991px) {
+ .navbar-custom-menu > .navbar-nav {
+ float: right;
}
- .navbar-nav > .notifications-menu > .dropdown-menu {
- position: absolute;
- right: -170px;
- left: auto;
+ .navbar-custom-menu > .navbar-nav > li {
+ position: static;
}
- .navbar-nav > .messages-menu > .dropdown-menu {
+ .navbar-custom-menu > .navbar-nav > li > .dropdown-menu {
position: absolute;
- right: -210px;
+ right: 5%;
left: auto;
+ border: 1px solid #ddd;
+ background: #fff;
}
}
/*
- All form elements including input, select, textarea etc.
------------------------------------------------------------------
-*/
+ * Component: Form
+ * ---------------
+ */
.form-control {
- -webkit-border-radius: 0px !important;
- -moz-border-radius: 0px !important;
border-radius: 0px !important;
box-shadow: none;
+ border-color: #d2d6de;
}
.form-control:focus {
border-color: #3c8dbc !important;
box-shadow: none;
}
+.form-control::-moz-placeholder {
+ color: #bbb;
+ opacity: 1;
+}
+.form-control:-ms-input-placeholder {
+ color: #bbb;
+}
+.form-control::-webkit-input-placeholder {
+ color: #bbb;
+}
+.form-control:not(select) {
+ -webkit-appearance: none;
+ -moz-appearance: none;
+ appearance: none;
+}
.form-group.has-success label {
color: #00a65a;
}
@@ -1124,38 +1260,72 @@ body > .header .logo .icon {
box-shadow: none;
}
.form-group.has-error label {
- color: #f56954;
+ color: #dd4b39;
}
.form-group.has-error .form-control {
- border-color: #f56954 !important;
+ border-color: #dd4b39 !important;
box-shadow: none;
}
/* Input group */
.input-group .input-group-addon {
- border-radius: 0;
- background-color: #f4f4f4;
+ border-radius: 0px;
+ border-color: #d2d6de;
+ background-color: #fff;
}
/* button groups */
.btn-group-vertical .btn.btn-flat:first-of-type,
.btn-group-vertical .btn.btn-flat:last-of-type {
border-radius: 0;
}
-/* Checkbox and radio inputs */
-.checkbox,
-.radio {
+.icheck > label {
padding-left: 0;
}
/*
- Compenent: Progress bars
---------------------------------
-*/
+ * Component: Progress Bar
+ * -----------------------
+ */
+.progress,
+.progress > .progress-bar {
+ -webkit-box-shadow: none;
+ box-shadow: none;
+}
+.progress,
+.progress > .progress-bar,
+.progress .progress-bar,
+.progress > .progress-bar .progress-bar {
+ border-radius: 1px;
+}
/* size variation */
-.progress.sm {
+.progress.sm,
+.progress-sm {
height: 10px;
}
-.progress.xs {
+.progress.sm,
+.progress-sm,
+.progress.sm .progress-bar,
+.progress-sm .progress-bar {
+ border-radius: 1px;
+}
+.progress.xs,
+.progress-xs {
height: 7px;
}
+.progress.xs,
+.progress-xs,
+.progress.xs .progress-bar,
+.progress-xs .progress-bar {
+ border-radius: 1px;
+}
+.progress.xxs,
+.progress-xxs {
+ height: 3px;
+}
+.progress.xxs,
+.progress-xxs,
+.progress.xxs .progress-bar,
+.progress-xxs .progress-bar {
+ border-radius: 1px;
+}
/* Vertical bars */
.progress.vertical {
position: relative;
@@ -1169,12 +1339,24 @@ body > .header .logo .icon {
position: absolute;
bottom: 0;
}
-.progress.vertical.sm {
+.progress.vertical.sm,
+.progress.vertical.progress-sm {
width: 20px;
}
-.progress.vertical.xs {
+.progress.vertical.xs,
+.progress.vertical.progress-xs {
width: 10px;
}
+.progress.vertical.xxs,
+.progress.vertical.progress-xxs {
+ width: 3px;
+}
+.progress-group .progress-text {
+ font-weight: 600;
+}
+.progress-group .progress-number {
+ float: right;
+}
/* Remove margins from progress bars when put in a table */
.table tr > td .progress {
margin: 0;
@@ -1185,9 +1367,8 @@ body > .header .logo .icon {
}
.progress-striped .progress-bar-light-blue,
.progress-striped .progress-bar-primary {
- background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
- background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+ background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-green,
@@ -1196,9 +1377,8 @@ body > .header .logo .icon {
}
.progress-striped .progress-bar-green,
.progress-striped .progress-bar-success {
- background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
- background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+ background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-aqua,
@@ -1207,9 +1387,8 @@ body > .header .logo .icon {
}
.progress-striped .progress-bar-aqua,
.progress-striped .progress-bar-info {
- background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
- background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+ background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-yellow,
@@ -1218,32 +1397,30 @@ body > .header .logo .icon {
}
.progress-striped .progress-bar-yellow,
.progress-striped .progress-bar-warning {
- background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
- background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+ background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-red,
.progress-bar-danger {
- background-color: #f56954;
+ background-color: #dd4b39;
}
.progress-striped .progress-bar-red,
.progress-striped .progress-bar-danger {
- background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
- background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+ background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
/*
- Component: Small boxes
-*/
+ * Component: Small Box
+ * --------------------
+ */
.small-box {
+ border-radius: 2px;
position: relative;
display: block;
- -webkit-border-radius: 2px;
- -moz-border-radius: 2px;
- border-radius: 2px;
- margin-bottom: 15px;
+ margin-bottom: 20px;
+ box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
}
.small-box > .inner {
padding: 10px;
@@ -1284,10 +1461,12 @@ body > .header .logo .icon {
z-index: 5px;
}
.small-box .icon {
+ -webkit-transition: all 0.3s linear;
+ -o-transition: all 0.3s linear;
+ transition: all 0.3s linear;
position: absolute;
- top: auto;
- bottom: 5px;
- right: 5px;
+ top: -10px;
+ right: 10px;
z-index: 0;
font-size: 90px;
color: rgba(0, 0, 0, 0.15);
@@ -1297,39 +1476,9 @@ body > .header .logo .icon {
color: #f9f9f9;
}
.small-box:hover .icon {
- animation-name: tansformAnimation;
- animation-duration: .5s;
- animation-iteration-count: 1;
- animation-timing-function: ease;
- animation-fill-mode: forwards;
- -webkit-animation-name: tansformAnimation;
- -webkit-animation-duration: .5s;
- -webkit-animation-iteration-count: 1;
- -webkit-animation-timing-function: ease;
- -webkit-animation-fill-mode: forwards;
- -moz-animation-name: tansformAnimation;
- -moz-animation-duration: .5s;
- -moz-animation-iteration-count: 1;
- -moz-animation-timing-function: ease;
- -moz-animation-fill-mode: forwards;
-}
-@keyframes tansformAnimation {
- from {
- font-size: 90px;
- }
- to {
- font-size: 100px;
- }
-}
-@-webkit-keyframes tansformAnimation {
- from {
- font-size: 90px;
- }
- to {
- font-size: 100px;
- }
-}
-@media screen and (max-width: 480px) {
+ font-size: 95px;
+}
+@media (max-width: 767px) {
.small-box {
text-align: center;
}
@@ -1341,19 +1490,17 @@ body > .header .logo .icon {
}
}
/*
- component: Boxes
--------------------------
-*/
+ * Component: Box
+ * --------------
+ */
.box {
position: relative;
+ border-radius: 3px;
background: #ffffff;
- border-top: 2px solid #c1c1c1;
+ border-top: 3px solid #d2d6de;
margin-bottom: 20px;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
width: 100%;
- box-shadow: 0px 1px 3px rgba(0, 0, 0, 0.1);
+ box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
}
.box.box-primary {
border-top-color: #3c8dbc;
@@ -1362,7 +1509,7 @@ body > .header .logo .icon {
border-top-color: #00c0ef;
}
.box.box-danger {
- border-top-color: #f56954;
+ border-top-color: #dd4b39;
}
.box.box-warning {
border-top-color: #f39c12;
@@ -1370,201 +1517,116 @@ body > .header .logo .icon {
.box.box-success {
border-top-color: #00a65a;
}
-.box.height-control .box-body {
- max-height: 300px;
- overflow: auto;
-}
-.box .box-header {
- position: relative;
- -webkit-border-top-left-radius: 3px;
- -webkit-border-top-right-radius: 3px;
- -webkit-border-bottom-right-radius: 0;
- -webkit-border-bottom-left-radius: 0;
- -moz-border-radius-topleft: 3px;
- -moz-border-radius-topright: 3px;
- -moz-border-radius-bottomright: 0;
- -moz-border-radius-bottomleft: 0;
- border-top-left-radius: 3px;
- border-top-right-radius: 3px;
- border-bottom-right-radius: 0;
- border-bottom-left-radius: 0;
- border-bottom: 0px solid #f4f4f4;
- color: #444;
-}
-.box .box-header:before,
-.box .box-header:after {
- display: table;
- content: " ";
+.box.box-default {
+ border-top-color: #d2d6de;
}
-.box .box-header:after {
- clear: both;
+.box.collapsed-box .box-body,
+.box.collapsed-box .box-footer {
+ display: none;
}
-.box .box-header > .fa,
-.box .box-header > .glyphicon,
-.box .box-header > .ion,
-.box .box-header .box-title {
- display: inline-block;
- padding: 10px 10px 10px 10px;
+.box .nav-stacked > li {
+ border-bottom: 1px solid #f4f4f4;
margin: 0;
- font-size: 20px;
- font-weight: 400;
- float: left;
- cursor: default;
-}
-.box .box-header a {
- color: #444;
-}
-.box .box-header > .box-tools {
- padding: 5px 10px 5px 5px;
-}
-.box .box-body {
- padding: 10px;
- -webkit-border-top-left-radius: 0;
- -webkit-border-top-right-radius: 0;
- -webkit-border-bottom-right-radius: 3px;
- -webkit-border-bottom-left-radius: 3px;
- -moz-border-radius-topleft: 0;
- -moz-border-radius-topright: 0;
- -moz-border-radius-bottomright: 3px;
- -moz-border-radius-bottomleft: 3px;
- border-top-left-radius: 0;
- border-top-right-radius: 0;
- border-bottom-right-radius: 3px;
- border-bottom-left-radius: 3px;
-}
-.box .box-body > table,
-.box .box-body > .table {
- margin-bottom: 0;
-}
-.box .box-body.chart-responsive {
- width: 100%;
- overflow: hidden;
-}
-.box .box-body > .chart {
- position: relative;
- overflow: hidden;
- width: 100%;
-}
-.box .box-body > .chart svg,
-.box .box-body > .chart canvas {
- width: 100%!important;
-}
-.box .box-body .fc {
- margin-top: 5px;
-}
-.box .box-body .fc-header-title h2 {
- font-size: 15px;
- line-height: 1.6em;
- color: #666;
- margin-left: 10px;
-}
-.box .box-body .fc-header-right {
- padding-right: 10px;
-}
-.box .box-body .fc-header-left {
- padding-left: 10px;
}
-.box .box-body .fc-widget-header {
- background: #fafafa;
- box-shadow: inset 0px -3px 1px rgba(0, 0, 0, 0.02);
+.box .nav-stacked > li:last-of-type {
+ border-bottom: none;
}
-.box .box-body .fc-grid {
- width: 100%;
- border: 0;
+.box.height-control .box-body {
+ max-height: 300px;
+ overflow: auto;
}
-.box .box-body .fc-widget-header:first-of-type,
-.box .box-body .fc-widget-content:first-of-type {
- border-left: 0;
- border-right: 0;
+.box .border-right {
+ border-right: 1px solid #f4f4f4;
}
-.box .box-body .fc-widget-header:last-of-type,
-.box .box-body .fc-widget-content:last-of-type {
- border-right: 0;
+.box .border-left {
+ border-left: 1px solid #f4f4f4;
}
-.box .box-body .table {
- margin-bottom: 0;
+.box.box-solid {
+ border-top: 0px;
}
-.box .box-body .full-width-chart {
- margin: -19px;
+.box.box-solid > .box-header .btn.btn-default {
+ background: transparent;
}
-.box .box-body.no-padding .full-width-chart {
- margin: -9px;
+.box.box-solid > .box-header .btn:hover,
+.box.box-solid > .box-header a:hover {
+ background: rgba(0, 0, 0, 0.1) !important;
}
-.box .box-footer {
- border-top: 1px solid #f4f4f4;
- -webkit-border-top-left-radius: 0;
- -webkit-border-top-right-radius: 0;
- -webkit-border-bottom-right-radius: 3px;
- -webkit-border-bottom-left-radius: 3px;
- -moz-border-radius-topleft: 0;
- -moz-border-radius-topright: 0;
- -moz-border-radius-bottomright: 3px;
- -moz-border-radius-bottomleft: 3px;
- border-top-left-radius: 0;
- border-top-right-radius: 0;
- border-bottom-right-radius: 3px;
- border-bottom-left-radius: 3px;
- padding: 10px;
- background-color: #ffffff;
+.box.box-solid.box-default {
+ border: 1px solid #d2d6de;
}
-.box.box-solid {
- border-top: 0px;
+.box.box-solid.box-default > .box-header {
+ color: #444444;
+ background: #d2d6de;
+ background-color: #d2d6de;
}
-.box.box-solid > .box-header {
- padding-bottom: 0px!important;
+.box.box-solid.box-default > .box-header a,
+.box.box-solid.box-default > .box-header .btn {
+ color: #444444;
}
-.box.box-solid > .box-header .btn.btn-default {
- background: transparent;
+.box.box-solid.box-primary {
+ border: 1px solid #3c8dbc;
}
.box.box-solid.box-primary > .box-header {
- color: #fff;
+ color: #ffffff;
background: #3c8dbc;
background-color: #3c8dbc;
}
-.box.box-solid.box-primary > .box-header a {
- color: #444;
+.box.box-solid.box-primary > .box-header a,
+.box.box-solid.box-primary > .box-header .btn {
+ color: #ffffff;
+}
+.box.box-solid.box-info {
+ border: 1px solid #00c0ef;
}
.box.box-solid.box-info > .box-header {
- color: #fff;
+ color: #ffffff;
background: #00c0ef;
background-color: #00c0ef;
}
-.box.box-solid.box-info > .box-header a {
- color: #444;
+.box.box-solid.box-info > .box-header a,
+.box.box-solid.box-info > .box-header .btn {
+ color: #ffffff;
+}
+.box.box-solid.box-danger {
+ border: 1px solid #dd4b39;
}
.box.box-solid.box-danger > .box-header {
- color: #fff;
- background: #f56954;
- background-color: #f56954;
+ color: #ffffff;
+ background: #dd4b39;
+ background-color: #dd4b39;
}
-.box.box-solid.box-danger > .box-header a {
- color: #444;
+.box.box-solid.box-danger > .box-header a,
+.box.box-solid.box-danger > .box-header .btn {
+ color: #ffffff;
+}
+.box.box-solid.box-warning {
+ border: 1px solid #f39c12;
}
.box.box-solid.box-warning > .box-header {
- color: #fff;
+ color: #ffffff;
background: #f39c12;
background-color: #f39c12;
}
-.box.box-solid.box-warning > .box-header a {
- color: #444;
+.box.box-solid.box-warning > .box-header a,
+.box.box-solid.box-warning > .box-header .btn {
+ color: #ffffff;
+}
+.box.box-solid.box-success {
+ border: 1px solid #00a65a;
}
.box.box-solid.box-success > .box-header {
- color: #fff;
+ color: #ffffff;
background: #00a65a;
background-color: #00a65a;
}
-.box.box-solid.box-success > .box-header a {
- color: #444;
+.box.box-solid.box-success > .box-header a,
+.box.box-solid.box-success > .box-header .btn {
+ color: #ffffff;
}
.box.box-solid > .box-header > .box-tools .btn {
border: 0;
box-shadow: none;
}
-.box.box-solid.collapsed-box .box-header {
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
-}
.box.box-solid[class*='bg'] > .box-header {
color: #fff;
}
@@ -1578,168 +1640,369 @@ body > .header .logo .icon {
font-size: 12px;
margin-bottom: 0.3em;
}
-.box .todo-list {
+.box > .overlay,
+.overlay-wrapper > .overlay,
+.box > .loading-img,
+.overlay-wrapper > .loading-img {
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+}
+.box .overlay,
+.overlay-wrapper .overlay {
+ z-index: 50;
+ background: rgba(255, 255, 255, 0.7);
+ border-radius: 3px;
+}
+.box .overlay > .fa,
+.overlay-wrapper .overlay > .fa {
+ position: absolute;
+ top: 50%;
+ left: 50%;
+ margin-left: -15px;
+ margin-top: -15px;
+ color: #000;
+ font-size: 30px;
+}
+.box .overlay.dark,
+.overlay-wrapper .overlay.dark {
+ background: rgba(0, 0, 0, 0.5);
+}
+.box-header:before,
+.box-body:before,
+.box-footer:before,
+.box-header:after,
+.box-body:after,
+.box-footer:after {
+ content: " ";
+ display: table;
+}
+.box-header:after,
+.box-body:after,
+.box-footer:after {
+ clear: both;
+}
+.box-header {
+ color: #444;
+ display: block;
+ padding: 10px;
+ position: relative;
+}
+.box-header.with-border {
+ border-bottom: 1px solid #f4f4f4;
+}
+.collapsed-box .box-header.with-border {
+ border-bottom: none;
+}
+.box-header > .fa,
+.box-header > .glyphicon,
+.box-header > .ion,
+.box-header .box-title {
+ display: inline-block;
+ font-size: 18px;
+ margin: 0;
+ line-height: 1;
+}
+.box-header > .fa,
+.box-header > .glyphicon,
+.box-header > .ion {
+ margin-right: 5px;
+}
+.box-header > .box-tools {
+ position: absolute;
+ right: 10px;
+ top: 5px;
+}
+.box-header > .box-tools [data-toggle="tooltip"] {
+ position: relative;
+}
+.box-header > .box-tools.pull-right .dropdown-menu {
+ right: 0;
+ left: auto;
+}
+.btn-box-tool {
+ padding: 5px;
+ font-size: 12px;
+ background: transparent;
+ box-shadow: none!important;
+ color: #97a0b3;
+}
+.open .btn-box-tool,
+.btn-box-tool:hover {
+ color: #606c84;
+}
+.btn-box-tool:active {
+ outline: none!important;
+}
+.box-body {
+ border-top-left-radius: 0;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 3px;
+ border-bottom-left-radius: 3px;
+ padding: 10px;
+}
+.no-header .box-body {
+ border-top-right-radius: 3px;
+ border-top-left-radius: 3px;
+}
+.box-body > .table {
+ margin-bottom: 0;
+}
+.box-body .fc {
+ margin-top: 5px;
+}
+.box-body .full-width-chart {
+ margin: -19px;
+}
+.box-body.no-padding .full-width-chart {
+ margin: -9px;
+}
+.box-body .box-pane {
+ border-top-left-radius: 0;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 0;
+ border-bottom-left-radius: 3px;
+}
+.box-body .box-pane-right {
+ border-top-left-radius: 0;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 3px;
+ border-bottom-left-radius: 0;
+}
+.box-footer {
+ border-top-left-radius: 0;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 3px;
+ border-bottom-left-radius: 3px;
+ border-top: 1px solid #f4f4f4;
+ padding: 10px;
+ background-color: #ffffff;
+}
+.chart-legend {
+ margin: 10px 0;
+}
+@media (max-width: 991px) {
+ .chart-legend > li {
+ float: left;
+ margin-right: 10px;
+ }
+}
+/* Widget: TODO LIST */
+.todo-list {
margin: 0;
padding: 0px 0px;
list-style: none;
+ overflow: auto;
}
-.box .todo-list > li {
- -webkit-border-radius: 2px;
- -moz-border-radius: 2px;
+.todo-list > li {
border-radius: 2px;
padding: 10px;
- background: #f3f4f5;
+ background: #f4f4f4;
margin-bottom: 2px;
border-left: 2px solid #e6e7e8;
color: #444;
}
-.box .todo-list > li:last-of-type {
+.todo-list > li:last-of-type {
margin-bottom: 0;
}
-.box .todo-list > li.danger {
- border-left-color: #f56954;
+.todo-list > li.danger {
+ border-left-color: #dd4b39;
}
-.box .todo-list > li.warning {
+.todo-list > li.warning {
border-left-color: #f39c12;
}
-.box .todo-list > li.info {
+.todo-list > li.info {
border-left-color: #00c0ef;
}
-.box .todo-list > li.success {
+.todo-list > li.success {
border-left-color: #00a65a;
}
-.box .todo-list > li.primary {
+.todo-list > li.primary {
border-left-color: #3c8dbc;
}
-.box .todo-list > li > input[type='checkbox'] {
+.todo-list > li > input[type='checkbox'] {
margin: 0 10px 0 5px;
}
-.box .todo-list > li .text {
+.todo-list > li .text {
display: inline-block;
margin-left: 5px;
font-weight: 600;
}
-.box .todo-list > li .label {
+.todo-list > li .label {
margin-left: 10px;
font-size: 9px;
}
-.box .todo-list > li .tools {
+.todo-list > li .tools {
display: none;
float: right;
- color: #f56954;
+ color: #dd4b39;
}
-.box .todo-list > li .tools > .fa,
-.box .todo-list > li .tools > .glyphicon,
-.box .todo-list > li .tools > .ion {
+.todo-list > li .tools > .fa,
+.todo-list > li .tools > .glyphicon,
+.todo-list > li .tools > .ion {
margin-right: 5px;
cursor: pointer;
}
-.box .todo-list > li:hover .tools {
+.todo-list > li:hover .tools {
display: inline-block;
}
-.box .todo-list > li.done {
+.todo-list > li.done {
color: #999;
}
-.box .todo-list > li.done .text {
+.todo-list > li.done .text {
text-decoration: line-through;
font-weight: 500;
}
-.box .todo-list > li.done .label {
- background: #eaeaec !important;
+.todo-list > li.done .label {
+ background: #d2d6de !important;
}
-.box .todo-list .handle {
+.todo-list .handle {
display: inline-block;
cursor: move;
margin: 0 5px;
}
-.box .chat {
+/* Chat widget (DEPRECATED - this will be removed in the next major release. Use Direct Chat instead)*/
+.chat {
padding: 5px 20px 5px 10px;
}
-.box .chat .item {
+.chat .item {
margin-bottom: 10px;
}
-.box .chat .item:before,
-.box .chat .item:after {
- display: table;
+.chat .item:before,
+.chat .item:after {
content: " ";
+ display: table;
}
-.box .chat .item:after {
+.chat .item:after {
clear: both;
}
-.box .chat .item > img {
+.chat .item > img {
width: 40px;
height: 40px;
border: 2px solid transparent;
- -webkit-border-radius: 50% !important;
- -moz-border-radius: 50% !important;
border-radius: 50% !important;
}
-.box .chat .item > img.online {
+.chat .item > img.online {
border: 2px solid #00a65a;
}
-.box .chat .item > img.offline {
- border: 2px solid #f56954;
+.chat .item > img.offline {
+ border: 2px solid #dd4b39;
}
-.box .chat .item > .message {
+.chat .item > .message {
margin-left: 55px;
margin-top: -40px;
}
-.box .chat .item > .message > .name {
+.chat .item > .message > .name {
display: block;
font-weight: 600;
}
-.box .chat .item > .attachment {
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
+.chat .item > .attachment {
border-radius: 3px;
- background: #f0f0f0;
+ background: #f4f4f4;
margin-left: 65px;
margin-right: 15px;
padding: 10px;
}
-.box .chat .item > .attachment > h4 {
+.chat .item > .attachment > h4 {
margin: 0 0 5px 0;
font-weight: 600;
font-size: 14px;
}
-.box .chat .item > .attachment > p,
-.box .chat .item > .attachment > .filename {
+.chat .item > .attachment > p,
+.chat .item > .attachment > .filename {
font-weight: 600;
font-size: 13px;
font-style: italic;
margin: 0;
}
-.box .chat .item > .attachment:before,
-.box .chat .item > .attachment:after {
- display: table;
+.chat .item > .attachment:before,
+.chat .item > .attachment:after {
content: " ";
+ display: table;
}
-.box .chat .item > .attachment:after {
+.chat .item > .attachment:after {
clear: both;
}
-.box > .overlay,
-.box > .loading-img {
- position: absolute;
- top: 0;
- left: 0;
+.box-input {
+ max-width: 200px;
+}
+.modal .panel-body {
+ color: #444;
+}
+/*
+ * Component: Info Box
+ * -------------------
+ */
+.info-box {
+ display: block;
+ min-height: 90px;
+ background: #fff;
width: 100%;
- height: 100%;
+ box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
+ border-radius: 2px;
+ margin-bottom: 15px;
}
-.box > .overlay {
- z-index: 1010;
- background: rgba(255, 255, 255, 0.7);
+.info-box small {
+ font-size: 14px;
}
-.box > .overlay.dark {
- background: rgba(0, 0, 0, 0.5);
+.info-box .progress {
+ background: rgba(0, 0, 0, 0.2);
+ margin: 5px -10px 5px -10px;
+ height: 2px;
+}
+.info-box .progress,
+.info-box .progress .progress-bar {
+ border-radius: 0;
+}
+.info-box .progress .progress-bar {
+ background: #fff;
+}
+.info-box-icon {
+ border-top-left-radius: 2px;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 0;
+ border-bottom-left-radius: 2px;
+ display: block;
+ float: left;
+ height: 90px;
+ width: 90px;
+ text-align: center;
+ font-size: 45px;
+ line-height: 90px;
+ background: rgba(0, 0, 0, 0.2);
+}
+.info-box-content {
+ padding: 5px 10px;
+ margin-left: 90px;
+}
+.info-box-number {
+ display: block;
+ font-weight: bold;
+ font-size: 18px;
}
-.box > .loading-img {
- z-index: 1020;
- background: transparent url('../img/ajax-loader1.gif') 50% 50% no-repeat;
+.progress-description,
+.info-box-text {
+ display: block;
+ font-size: 14px;
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+.info-box-text {
+ text-transform: uppercase;
+}
+.info-box-more {
+ display: block;
+}
+.progress-description {
+ margin: 0;
}
/*
-Component: timeline
---------------------
-*/
+ * Component: Timeline
+ * -------------------
+ */
.timeline {
position: relative;
margin: 0 0 30px 0;
@@ -1751,13 +2014,10 @@ Component: timeline
position: absolute;
top: 0px;
bottom: 0;
- width: 5px;
+ width: 4px;
background: #ddd;
- left: 30px;
- border: 1px solid #eee;
+ left: 31px;
margin: 0;
- -webkit-border-radius: 2px;
- -moz-border-radius: 2px;
border-radius: 2px;
}
.timeline > li {
@@ -1767,33 +2027,35 @@ Component: timeline
}
.timeline > li:before,
.timeline > li:after {
- display: table;
content: " ";
+ display: table;
}
.timeline > li:after {
clear: both;
}
.timeline > li > .timeline-item {
- margin-top: 10px;
- border: 0px solid #dfdfdf;
+ -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
+ box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
+ border-radius: 3px;
+ margin-top: 0px;
background: #fff;
- color: #555;
+ color: #444;
margin-left: 60px;
margin-right: 15px;
- padding: 5px;
+ padding: 0;
position: relative;
- box-shadow: 0px 1px 3px rgba(0, 0, 0, 0.1);
}
.timeline > li > .timeline-item > .time {
color: #999;
float: right;
- margin: 2px 0 0 0;
+ padding: 10px;
+ font-size: 12px;
}
.timeline > li > .timeline-item > .timeline-header {
margin: 0;
color: #555;
border-bottom: 1px solid #f4f4f4;
- padding: 5px;
+ padding: 10px;
font-size: 16px;
line-height: 1.1;
}
@@ -1809,214 +2071,201 @@ Component: timeline
padding: 5px;
display: inline-block;
background-color: #fff;
- box-shadow: 0 1px 1px rgba(0, 0, 0, 0.5);
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
border-radius: 4px;
}
.timeline > li > .fa,
.timeline > li > .glyphicon,
.timeline > li > .ion {
- box-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
width: 30px;
height: 30px;
font-size: 15px;
line-height: 30px;
position: absolute;
color: #666;
- background: #eee;
+ background: #d2d6de;
border-radius: 50%;
text-align: center;
left: 18px;
top: 0;
}
/*
- Component: Buttons
--------------------------
-*/
+ * Component: Button
+ * -----------------
+ */
.btn {
- font-weight: 500;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
border-radius: 3px;
+ -webkit-box-shadow: none;
+ box-shadow: none;
border: 1px solid transparent;
- -webkit-box-shadow: inset 0px -2px 0px 0px rgba(0, 0, 0, 0.09);
- -moz-box-shadow: inset 0px -2px 0px 0px rgba(0, 0, 0, 0.09);
- box-shadow: inset 0px -1px 0px 0px rgba(0, 0, 0, 0.09);
}
-.btn.btn-default {
- background-color: #fafafa;
- color: #666;
- border-color: #ddd;
- border-bottom-color: #ddd;
+.btn.uppercase {
+ text-transform: uppercase;
+}
+.btn.btn-flat {
+ border-radius: 0;
+ -webkit-box-shadow: none;
+ -moz-box-shadow: none;
+ box-shadow: none;
+ border-width: 1px;
+}
+.btn:active {
+ -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
+ -moz-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
+ box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
+}
+.btn:focus {
+ outline: none;
+}
+.btn.btn-file {
+ position: relative;
+ overflow: hidden;
+}
+.btn.btn-file > input[type='file'] {
+ position: absolute;
+ top: 0;
+ right: 0;
+ min-width: 100%;
+ min-height: 100%;
+ font-size: 100px;
+ text-align: right;
+ opacity: 0;
+ filter: alpha(opacity=0);
+ outline: none;
+ background: white;
+ cursor: inherit;
+ display: block;
}
-.btn.btn-default:hover,
-.btn.btn-default:active,
-.btn.btn-default.hover {
- background-color: #f4f4f4!important;
+.btn-default {
+ background-color: #f4f4f4;
+ color: #444;
+ border-color: #ddd;
}
-.btn.btn-default.btn-flat {
- border-bottom-color: #d9dadc;
+.btn-default:hover,
+.btn-default:active,
+.btn-default.hover {
+ background-color: #e7e7e7 !important;
}
-.btn.btn-primary {
+.btn-primary {
background-color: #3c8dbc;
border-color: #367fa9;
}
-.btn.btn-primary:hover,
-.btn.btn-primary:active,
-.btn.btn-primary.hover {
+.btn-primary:hover,
+.btn-primary:active,
+.btn-primary.hover {
background-color: #367fa9;
}
-.btn.btn-success {
+.btn-success {
background-color: #00a65a;
border-color: #008d4c;
}
-.btn.btn-success:hover,
-.btn.btn-success:active,
-.btn.btn-success.hover {
+.btn-success:hover,
+.btn-success:active,
+.btn-success.hover {
background-color: #008d4c;
}
-.btn.btn-info {
+.btn-info {
background-color: #00c0ef;
border-color: #00acd6;
}
-.btn.btn-info:hover,
-.btn.btn-info:active,
-.btn.btn-info.hover {
+.btn-info:hover,
+.btn-info:active,
+.btn-info.hover {
background-color: #00acd6;
}
-.btn.btn-danger {
- background-color: #f56954;
- border-color: #f4543c;
+.btn-danger {
+ background-color: #dd4b39;
+ border-color: #d73925;
}
-.btn.btn-danger:hover,
-.btn.btn-danger:active,
-.btn.btn-danger.hover {
- background-color: #f4543c;
+.btn-danger:hover,
+.btn-danger:active,
+.btn-danger.hover {
+ background-color: #d73925;
}
-.btn.btn-warning {
+.btn-warning {
background-color: #f39c12;
border-color: #e08e0b;
}
-.btn.btn-warning:hover,
-.btn.btn-warning:active,
-.btn.btn-warning.hover {
+.btn-warning:hover,
+.btn-warning:active,
+.btn-warning.hover {
background-color: #e08e0b;
}
-.btn.btn-flat {
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
- box-shadow: none;
- border-width: 1px;
-}
-.btn:active {
- -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
- -moz-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
- box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
+.btn-outline {
+ border: 1px solid #fff;
+ background: transparent;
+ color: #fff;
}
-.btn:focus {
- outline: none;
+.btn-outline:hover,
+.btn-outline:focus,
+.btn-outline:active {
+ color: rgba(255, 255, 255, 0.7);
+ border-color: rgba(255, 255, 255, 0.7);
}
-.btn.btn-file {
- position: relative;
- width: 120px;
- height: 35px;
- overflow: hidden;
+.btn-link {
+ -webkit-box-shadow: none;
+ box-shadow: none;
}
-.btn.btn-file > input[type='file'] {
- display: block !important;
- width: 100% !important;
- height: 35px !important;
- opacity: 0 !important;
- position: absolute;
- top: -10px;
- cursor: pointer;
+.btn[class*='bg-']:hover {
+ -webkit-box-shadow: inset 0 0 100px rgba(0, 0, 0, 0.2);
+ box-shadow: inset 0 0 100px rgba(0, 0, 0, 0.2);
}
-.btn.btn-app {
+.btn-app {
+ border-radius: 3px;
position: relative;
padding: 15px 5px;
margin: 0 0 10px 10px;
min-width: 80px;
height: 60px;
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
- box-shadow: none;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
text-align: center;
color: #666;
<TRUNCATED>
[11/50] [abbrv] incubator-kylin git commit: KYLIN-540 add the step
for coprocessor upgrade, and duplicate dict for FK
Posted by li...@apache.org.
KYLIN-540 add the step for coprocessor upgrade, and duplicate dict for FK
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b08df2bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b08df2bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b08df2bc
Branch: refs/heads/0.7
Commit: b08df2bce829efd2129c35985ccebbe9fcdb7a6b
Parents: faeb8a9
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jun 25 17:21:41 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jun 25 17:21:41 2015 +0800
----------------------------------------------------------------------
.../Migrate v0.6.x metadata to v0.7.md | 7 ++
.../apache/kylin/job/CubeMetadataUpgrade.java | 93 ++++++++++++++------
2 files changed, 71 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b08df2bc/docs/Operations/Migrate v0.6.x metadata to v0.7.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Migrate v0.6.x metadata to v0.7.md b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
index 26140aa..badd135 100644
--- a/docs/Operations/Migrate v0.6.x metadata to v0.7.md
+++ b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
@@ -38,4 +38,11 @@ Now the new format of metadata will be upload to the HBase to replace the old fo
hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.common.persistence.ResourceTool reset
hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.common.persistence.ResourceTool upload ./meta_dump_v2
+# Update HTables to use new coprocessor
+
+Kylin uses HBase coprocessor to do server side aggregation; When Kylin instance upgrades to V0.7, the HTables that created in V0.6 should also be updated to use the new coprocessor:
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.job.tools.DeployCoprocessorCLI ${KYLIN_HOME}/lib/kylin-coprocessor-x.x.x-SNAPSHOT.jar
+
+
Done; Update your v0.7 Kylin configure to point to the same metadata HBase table, then start Kylin server; Check whether all cubes and other information are kept;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b08df2bc/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index b970775..13103ac 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeDescUpgrader;
import org.apache.kylin.cube.CubeManager;
@@ -54,8 +55,7 @@ import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.*;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -101,6 +101,7 @@ public class CubeMetadataUpgrade {
upgradeProjectInstance();
upgradeCubeInstance();
upgradeJobInstance();
+ copyDictionaryForFK();
verify();
}
@@ -395,6 +396,7 @@ public class CubeMetadataUpgrade {
newInstance.setStatus(RealizationStatusEnum.READY);
}
+
List<org.apache.kylin.cube.CubeSegment> newSegments = Lists.newArrayList();
// segment
for (CubeSegment segment : cubeInstance.getSegments()) {
@@ -449,10 +451,69 @@ public class CubeMetadataUpgrade {
}
}
+ private void copyDictionaryForFK() {
+ CubeManager cubeManager = CubeManager.getInstance(config);
+ List<org.apache.kylin.cube.CubeInstance> cubeInstances = cubeManager.listAllCubes();
+
+ Set<String> changedCubes = Sets.newHashSet();
+ for (org.apache.kylin.cube.CubeInstance newInstance : cubeInstances) {
+
+ boolean updated = false;
+ DataModelDesc dataModelDesc = null;
+ try {
+ String modelName = this.getCubeDescManager().getCubeDesc(newInstance.getDescName()).getModelName();
+ dataModelDesc = this.getMetadataManager().getDataModelDesc(modelName);
+ Map<String, String> pkToFK = Maps.newHashMap();
+ for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
+ if (lookupDesc.getJoin() != null) {
+ JoinDesc join = lookupDesc.getJoin();
+ for (int i=0; i< join.getForeignKey().length; i++) {
+ pkToFK.put(lookupDesc.getTable() + "/" +join.getPrimaryKey()[i], dataModelDesc.getFactTable() + "/" +join.getForeignKey()[i]);
+ }
+ }
+ }
+
+ List<Pair<String, String>> newDictionaries = Lists.newArrayList();
+
+ // segment
+ for (org.apache.kylin.cube.CubeSegment newSeg : newInstance.getSegments()) {
+
+ for (Map.Entry<String, String> e : newSeg.getDictionaries().entrySet()) {
+ String key = e.getKey();
+ if (pkToFK.containsKey(key) && !newSeg.getDictionaries().containsKey(pkToFK.get(key))) {
+ logger.debug("Duplicate dictionary for FK " + pkToFK.get(key) + " in cube " + newInstance.getName());
+ changedCubes.add(newInstance.getName());
+ newDictionaries.add(new Pair(pkToFK.get(key), e.getValue()));
+
+ }
+ }
+ for(Pair<String, String> dict: newDictionaries) {
+ newSeg.getDictionaries().put(dict.getFirst(), dict.getSecond());
+ updated = true;
+ }
+ }
+
+ if (updated)
+ store.putResource(newInstance.getResourcePath(), newInstance, CubeManager.CUBE_SERIALIZER);
+ } catch (Exception e) {
+ logger.error("Error during upgrade cube instance " + newInstance.getName(), e);
+ }
+ }
+
+ if (changedCubes.size() > 0)
+ logger.info("Updated these cubeInstances: " + changedCubes);
+ }
+
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(config);
}
+
+ private CubeDescManager getCubeDescManager() {
+ return CubeDescManager.getInstance(config);
+ }
+
+
private ResourceStore getStore() {
return ResourceStore.getStore(config);
}
@@ -650,33 +711,6 @@ public class CubeMetadataUpgrade {
}
- public static void main1(String[] args) {
-
- if (!(args != null && args.length == 1)) {
- System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder>; e.g, /export/kylin/meta");
- return;
- }
-
- String exportFolder = args[0];
-
- File oldMetaFolder = new File(exportFolder);
- if (!oldMetaFolder.exists()) {
- System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
- return;
- }
-
- if (!oldMetaFolder.isDirectory()) {
- System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
- return;
- }
-
-
- String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
- CubeMetadataUpgrade instance = new CubeMetadataUpgrade(newMetadataUrl);
- instance.verify();
- }
-
-
public static void main(String[] args) {
if (!(args != null && (args.length == 1 || args.length == 2))) {
@@ -695,6 +729,7 @@ public class CubeMetadataUpgrade {
if (verify) {
instance = new CubeMetadataUpgrade(exportFolder);
instance.verify();
+ instance.addDictionaryForFK();
} else {
File oldMetaFolder = new File(exportFolder);
if (!oldMetaFolder.exists()) {
[18/50] [abbrv] incubator-kylin git commit: KYLIN-865 flush cache on
cube update
Posted by li...@apache.org.
KYLIN-865 flush cache on cube update
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/231fcc49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/231fcc49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/231fcc49
Branch: refs/heads/0.7
Commit: 231fcc4976914fe0b717612ab1e34763f9574813
Parents: 3ce0233
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 30 18:06:19 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 30 18:22:46 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/AbstractKylinTestCase.java | 2 +-
.../java/org/apache/kylin/rest/controller/CacheController.java | 2 ++
.../main/java/org/apache/kylin/rest/service/BasicService.java | 5 ++---
.../main/java/org/apache/kylin/rest/service/CacheService.java | 6 ++++--
.../java/org/apache/kylin/rest/service/ServiceTestBase.java | 2 ++
5 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/231fcc49/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 36a9287..a4a6dc4 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -36,7 +36,7 @@ public abstract class AbstractKylinTestCase {
public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.metadata.project.ProjectManager", "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.metadata.realization.RealizationRegistry" };
+ public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.metadata.project.ProjectManager" };
public abstract void createTestMetadata() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/231fcc49/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index 912d5bc..3ed30c5 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -67,9 +67,11 @@ public class CacheController extends BasicController {
case CREATE:
case UPDATE:
cacheService.rebuildCache(wipeType, name);
+ cacheService.cleanDataCache();
break;
case DROP:
cacheService.removeCache(wipeType, name);
+ cacheService.cleanDataCache();
break;
default:
throw new RuntimeException("invalid type:" + wipeEvent);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/231fcc49/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 6043edb..f248e95 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -37,6 +37,7 @@ import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -131,9 +132,7 @@ public abstract class BasicService {
*/
@Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public void cleanDataCache() {
- CubeManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
+ logger.debug("clean query cache...");
}
public final KylinConfig getKylinConfig() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/231fcc49/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 164ca12..9ec1621 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -23,8 +23,10 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.springframework.stereotype.Component;
@@ -69,14 +71,14 @@ public class CacheService extends BasicService {
CubeDescManager.clearCache();
break;
case ALL:
- getMetadataManager().reload();
+ MetadataManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ RealizationRegistry.clearCache();
ProjectManager.clearCache();
BasicService.resetOLAPDataSources();
- cleanDataCache();
break;
default:
throw new RuntimeException("invalid cacheType:" + cacheType);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/231fcc49/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index cc0dfc6..7865ac0 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -26,6 +26,7 @@ import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.junit.*;
import org.junit.runner.RunWith;
import org.springframework.security.authentication.TestingAuthenticationToken;
@@ -64,6 +65,7 @@ public class ServiceTestBase extends HBaseMetadataTestCase {
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ RealizationRegistry.clearCache();
ProjectManager.clearCache();
}
[25/50] [abbrv] incubator-kylin git commit: KYLIN-792 ,
add performance module
Posted by li...@apache.org.
KYLIN-792 ,add performance module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a935c7ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a935c7ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a935c7ba
Branch: refs/heads/0.7
Commit: a935c7bae805b2e93736e8f7cb93dc0f0e6f467d
Parents: cbcfd59
Author: jiazhong <ji...@ebay.com>
Authored: Wed May 27 23:01:10 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Thu Jul 2 19:51:40 2015 +0800
----------------------------------------------------------------------
conf/kylin.properties | 12 +
.../test_case_data/sandbox/kylin.properties | 14 +
monitor/pom.xml | 169 +
.../apache/kylin/monitor/ApiRequestParser.java | 241 +
.../java/org/apache/kylin/monitor/Client.java | 63 +
.../org/apache/kylin/monitor/ConfigUtils.java | 220 +
.../org/apache/kylin/monitor/DebugClient.java | 62 +
.../org/apache/kylin/monitor/FileUtils.java | 116 +
.../apache/kylin/monitor/HiveJdbcClient.java | 223 +
.../kylin/monitor/MonitorMetaManager.java | 217 +
.../org/apache/kylin/monitor/QueryParser.java | 249 +
monitor/src/main/resources/log4j.properties | 31 +
.../org/apache/kylin/monitor/ParseLogTest.java | 77 +
pom.xml | 1 +
script/prepare.sh | 2 +
server/pom.xml | 5 +
.../rest/controller/PerformanceController.java | 125 +
.../kylin/rest/filter/KylinApiFilter.java | 121 +
.../apache/kylin/rest/service/CubeService.java | 4 +
.../kylin/rest/service/PerformService.java | 123 +
server/src/main/resources/kylinSecurity.xml | 1 +
server/src/main/webapp/WEB-INF/web.xml | 122 +-
webapp/app/css/AdminLTE.css | 4782 ++++++++++--------
webapp/app/index.html | 16 +-
webapp/app/js/app.js | 2 +-
webapp/app/js/controllers/dashboard.js | 237 +
webapp/app/js/services/dashboard.js | 32 +
webapp/app/js/services/users.js | 13 +-
webapp/app/less/app.less | 17 +
webapp/app/partials/admin/admin.html | 3 +
webapp/app/partials/dashboard.html | 228 +
webapp/app/partials/header.html | 2 +-
webapp/app/routes.json | 7 +
webapp/bower.json | 10 +-
webapp/grunt.json | 2 +
35 files changed, 5391 insertions(+), 2158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index cf9756e..b1ebccc 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -101,3 +101,15 @@ deploy.env=DEV
###########################config info for sandbox#######################
kylin.sandbox=true
+
+
+###########################config info for kylin monitor#######################
+# hive jdbc url
+kylin.hive.jdbc.connection.url=
+
+#config where to parse query log,split with comma ,will also read $KYLIN_HOME/tomcat/logs/ by default
+ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2
+
+#will create external hive table to query result csv file
+#will set to kylin_query_log by default if not config here
+query.log.parse.result.table = kylin_query_log
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index efaef5c..8f73f64 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -102,3 +102,17 @@ deploy.env=DEV
###########################config info for sandbox#######################
kylin.sandbox=true
+
+###########################config info for kylin monitor#######################
+# hive jdbc url
+kylin.hive.jdbc.connection.url= jdbc:hive2://sandbox:10000
+
+#config where to parse query log,split with comma ,will also read $KYLIN_HOME/tomcat/logs/ by default
+ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2
+
+#will create external hive table to query result csv file
+#will set to kylin_query_log by default if not config here
+query.log.parse.result.table = kylin_query_log
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/pom.xml
----------------------------------------------------------------------
diff --git a/monitor/pom.xml b/monitor/pom.xml
new file mode 100644
index 0000000..a6fc87c
--- /dev/null
+++ b/monitor/pom.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin</artifactId>
+ <version>0.7.2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-monitor</artifactId>
+ <packaging>jar</packaging>
+ <name>Kylin:Monitor</name>
+ <url>http://maven.apache.org</url>
+
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <hadoop.version>2.6.0</hadoop.version>
+ <hive.version>0.14.0</hive.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ <version>2.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!--hbase dependency-->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!--hbase dependency-->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase-hadoop2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Hive dependencies -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.kylin.monitor.Client</mainClass>
+ </manifest>
+ </archive>
+
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/ApiRequestParser.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/ApiRequestParser.java b/monitor/src/main/java/org/apache/kylin/monitor/ApiRequestParser.java
new file mode 100644
index 0000000..0321549
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/ApiRequestParser.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.monitor;
+
+import au.com.bytecode.opencsv.CSVWriter;
+import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.log4j.Logger;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author jiazhong
+ */
+public class ApiRequestParser {
+
+ final static Logger logger = Logger.getLogger(ApiRequestParser.class);
+ final static Charset ENCODING = StandardCharsets.UTF_8;
+ static String REQUEST_PARSE_RESULT_PATH = null;
+ final static String REQUEST_LOG_FILE_PATTERN = "kylin_request.log.(\\d{4}-\\d{2}-\\d{2})$";
+ final static String REQUEST_LOG_PARSE_RESULT_FILENAME = "kylin_request_log.csv";
+ static String DEPLOY_ENV;
+
+
+ final static String[] KYLIN_REQUEST_CSV_HEADER = {"REQUESTER", "REQ_TIME","REQ_DATE", "URI", "METHOD", "QUERY_STRING", "PAYLOAD", "RESP_STATUS", "TARGET", "ACTION","DEPLOY_ENV"};
+
+ private ConfigUtils monitorConfig;
+
+ public ApiRequestParser() {
+ monitorConfig = ConfigUtils.getInstance();
+ try {
+ monitorConfig.loadMonitorParam();
+ DEPLOY_ENV = monitorConfig.getDeployEnv();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void start() throws IOException, ParseException {
+ ApiRequestParser.REQUEST_PARSE_RESULT_PATH = ConfigUtils.getInstance().getRequestLogParseResultDir() + REQUEST_LOG_PARSE_RESULT_FILENAME;
+ this.parseRequestInit();
+
+ //get api req log files have been read
+ String[] hasReadFiles = MonitorMetaManager.getReadApiReqLogFileList();
+
+ List<File> files = this.getRequestLogFiles();
+ for (File file : files) {
+ if (!Arrays.asList(hasReadFiles).contains(file.getName())) {
+ this.parseRequestLog(file.getPath(), ApiRequestParser.REQUEST_PARSE_RESULT_PATH);
+ MonitorMetaManager.markApiReqLogFileAsRead(file.getName());
+ }
+ }
+ }
+
+ public void parseRequestInit() throws IOException {
+ logger.info("parse api request initializing...");
+ FileSystem fs = null;
+ try {
+ Configuration conf = new Configuration();
+ fs = FileSystem.get(conf);
+ org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(ApiRequestParser.REQUEST_PARSE_RESULT_PATH);
+ if (!fs.exists(path)) {
+ fs.create(path);
+
+ //need to close before get FileSystem again
+ fs.close();
+ this.writeResultToHdfs(ApiRequestParser.REQUEST_PARSE_RESULT_PATH, ApiRequestParser.KYLIN_REQUEST_CSV_HEADER);
+ }
+ } catch (Exception e) {
+ fs.close();
+ logger.info("Failed to init:", e);
+ }
+ }
+
+ //parse query log and convert to csv file to hdfs
+ public void parseRequestLog(String filePath, String dPath) throws ParseException, IOException {
+
+ logger.info("Start parsing kylin api request file " + filePath + " !");
+
+// writer config init
+ FileSystem fs = this.getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ OutputStreamWriter writer = new OutputStreamWriter(fs.append(resultStorePath));
+ CSVWriter cwriter = new CSVWriter(writer,'|',CSVWriter.NO_QUOTE_CHARACTER);
+
+ Pattern p_available = Pattern.compile("/kylin/api/(cubes|user)+.*");
+ Pattern p_request = Pattern.compile("^.*\\[.*KylinApiFilter.logRequest.*\\].*REQUEST:.*REQUESTER=(.*);REQ_TIME=(\\w+ (\\d{4}-\\d{2}-\\d{2}).*);URI=(.*);METHOD=(.*);QUERY_STRING=(.*);PAYLOAD=(.*);RESP_STATUS=(.*);$");
+ Pattern p_uri = Pattern.compile("/kylin/api/(\\w+)(/.*/)*(.*)$");
+ Matcher m_available = p_available.matcher("");
+ Matcher m_request = p_request.matcher("");
+ Matcher m_uri = p_uri.matcher("");
+
+ Path path = Paths.get(filePath);
+ try {
+ BufferedReader reader = Files.newBufferedReader(path, ENCODING);
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ //reset the input
+ m_available.reset(line);
+ m_request.reset(line);
+
+ //filter unnecessary info
+ if (m_available.find()) {
+ //filter GET info
+ if (m_request.find() && !m_request.group(5).equals("GET")) {
+
+ List<String> groups = new ArrayList<String>();
+ for (int i = 1; i <= m_request.groupCount(); i++) {
+ groups.add(m_request.group(i));
+ }
+
+ String uri = m_request.group(4);
+ m_uri.reset(uri);
+ if (m_uri.find()) {
+
+ //add target
+ groups.add(m_uri.group(1));
+
+ //add action
+ if (m_uri.group(1).equals("cubes")) {
+ switch (m_request.group(5)) {
+ case "DELETE":
+ groups.add("drop");
+ break;
+ case "POST":
+ groups.add("save");
+ break;
+ default:
+ //add parse action
+ groups.add(m_uri.group(3));
+ break;
+ }
+ }
+
+ }
+ groups.add(DEPLOY_ENV);
+ String[] recordArray = groups.toArray(new String[groups.size()]);
+ //write to hdfs
+ cwriter.writeNext(recordArray);
+ }
+ }
+
+
+ }
+ } catch (IOException ex) {
+ logger.info("Failed to write to hdfs:", ex);
+ } finally {
+ writer.close();
+ cwriter.close();
+ fs.close();
+ }
+
+ logger.info("Finish parsing file " + filePath + " !");
+ }
+
+ public void writeResultToHdfs(String dPath, String[] record) throws IOException {
+ OutputStreamWriter writer = null;
+ CSVWriter cwriter = null;
+ FileSystem fs = null;
+ try {
+
+ fs = this.getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ writer = new OutputStreamWriter(fs.append(resultStorePath));
+ cwriter = new CSVWriter(writer,'|',CSVWriter.NO_QUOTE_CHARACTER);
+ cwriter.writeNext(record);
+
+ } catch (IOException e) {
+ logger.info("Exception", e);
+ } finally {
+ writer.close();
+ cwriter.close();
+ fs.close();
+ }
+ }
+
+ public List<File> getRequestLogFiles() {
+ List<File> logFiles = new ArrayList<File>();
+
+// String request_log_file_pattern = monitorConfig.getRequestLogFilePattern();
+
+ List<String> request_log_dir_list = monitorConfig.getLogBaseDir();
+ FileFilter filter = new RegexFileFilter(REQUEST_LOG_FILE_PATTERN);
+
+ for (String path : request_log_dir_list) {
+ logger.info("fetch api request log file from path:" + path);
+ File request_log_dir = new File(path);
+ File[] request_log_files = request_log_dir.listFiles(filter);
+ if (request_log_files == null) {
+ logger.warn("no api request log file found under path" + path);
+ break;
+ }
+ Collections.addAll(logFiles, request_log_files);
+ }
+
+ return logFiles;
+ }
+
+ public FileSystem getHdfsFileSystem() throws IOException {
+ Configuration conf = new Configuration();
+// conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(conf);
+ } catch (IOException e) {
+ fs.close();
+ logger.info("Failed to get hdfs FileSystem", e);
+ }
+ return fs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/Client.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/Client.java b/monitor/src/main/java/org/apache/kylin/monitor/Client.java
new file mode 100644
index 0000000..eadce54
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/Client.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+
+
+/**
+ * Created by jiazhong on 2015/5/7.
+ */
+public class Client {
+
+
+ static {
+ //set monitor log path
+ String KYLIN_HOME = ConfigUtils.getKylinHome();
+ String CATALINA_HOME = null;
+ if(!StringUtils.isEmpty(KYLIN_HOME)){
+ CATALINA_HOME = ConfigUtils.getKylinHome() + File.separator + "tomcat"+File.separator;
+ System.out.println("will use "+CATALINA_HOME+"/logs to put monitor log");
+ }else{
+ CATALINA_HOME = "";
+ System.out.println("will use default path to put monitor log");
+ }
+ //log4j config will use this
+ System.setProperty("CATALINA_HOME",CATALINA_HOME);
+ }
+ final static Logger logger = Logger.getLogger(Client.class);
+
+
+
+ public static void main(String[] args) {
+ logger.info("monitor client start parsing...");
+ QueryParser queryParser = new QueryParser();
+ HiveJdbcClient jdbcClient = new HiveJdbcClient();
+ try {
+ queryParser.start();
+ jdbcClient.start();
+ } catch (Exception e) {
+ logger.info("Exception",e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java b/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
new file mode 100644
index 0000000..f38b701
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/ConfigUtils.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import java.io.*;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Created by jiazhong on 2015/4/28.
+ */
+public class ConfigUtils {
+
+ final static Logger logger = Logger.getLogger(ConfigUtils.class);
+
+ private static ConfigUtils ourInstance = new ConfigUtils();
+
+ private Properties monitorConfig = new Properties();
+
+ public static ConfigUtils getInstance() {
+ return ourInstance;
+ }
+
+ private ConfigUtils() {
+ }
+
+ public static final String KYLIN_EXT_LOG_BASE_DIR = "ext.log.base.dir";
+ public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
+
+ public static final String KYLIN_HOME = "KYLIN_HOME";
+ public static final String KYLIN_CONF = "KYLIN_CONF";
+ public static final String KYLIN_LOG_CONF_HOME = "KYLIN_LOG_CONF_HOME";
+ public static final String CATALINA_HOME = "CATALINA_HOME";
+
+ public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
+
+ public static final String KYLIN_MONITOR_CONF_PROP_FILE = "kylin.properties";
+ public static final String QUERY_LOG_PARSE_RESULT_TABLE = "query.log.parse.result.table";
+ public static final String DEFAULT_QUERY_LOG_PARSE_RESULT_TABLE = "kylin_query_log";
+
+ public static final String DEPLOY_ENV = "deploy.env";
+
+
+ public static final String HIVE_JDBC_CON_URL = "kylin.hive.jdbc.connection.url";
+
+
+ public void loadMonitorParam() throws IOException {
+ Properties props = new Properties();
+ InputStream resourceStream = this.getKylinPropertiesAsInputSteam();
+ props.load(resourceStream);
+ this.monitorConfig = props;
+ }
+
+
+ public static InputStream getKylinPropertiesAsInputSteam() {
+ File propFile = getKylinMonitorProperties();
+ if (propFile == null || !propFile.exists()) {
+ logger.error("fail to locate kylin.properties");
+ throw new RuntimeException("fail to locate kylin.properties");
+ }
+ try {
+ return new FileInputStream(propFile);
+ } catch (FileNotFoundException e) {
+ logger.error("this should not happen");
+ throw new RuntimeException(e);
+ }
+
+ }
+
+
+ private static File getKylinMonitorProperties() {
+ String kylinConfHome = System.getProperty(KYLIN_CONF);
+ if (!StringUtils.isEmpty(kylinConfHome)) {
+ logger.info("load kylin.properties file from " + kylinConfHome + ". (from KYLIN_CONF System.property)");
+ return getKylinPropertiesFile(kylinConfHome);
+ }
+
+ logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable");
+
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome))
+ throw new RuntimeException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
+
+ String path = kylinHome + File.separator + "conf";
+ logger.info("load kylin.properties file from " + kylinHome+". (from KYLIN_HOME System.env)");
+ return getKylinPropertiesFile(path);
+
+ }
+
+ public static String getKylinHome() {
+ String kylinHome = System.getenv(KYLIN_HOME);
+ if (StringUtils.isEmpty(kylinHome)) {
+ logger.warn("KYLIN_HOME was not set");
+ return kylinHome;
+ }
+ logger.info("KYLIN_HOME is :"+kylinHome);
+ return kylinHome;
+ }
+
+ private static File getKylinPropertiesFile(String path) {
+ if (path == null) {
+ return null;
+ }
+ return new File(path, KYLIN_MONITOR_CONF_PROP_FILE);
+ }
+
+
+ /*
+ * get where to path log
+ */
+ public List<String> getLogBaseDir() {
+ List<String> logDirList = new ArrayList<String>();
+
+ String kylinLogConfHome = System.getProperty(KYLIN_LOG_CONF_HOME);
+ if (!StringUtils.isEmpty(kylinLogConfHome)) {
+ logger.info("Use KYLIN_LOG_CONF_HOME=" + KYLIN_LOG_CONF_HOME);
+ logDirList.add(kylinLogConfHome);
+ }
+
+ String kylinExtLogBaseDir = getExtLogBaseDir();
+ if (!StringUtils.isEmpty(kylinExtLogBaseDir)) {
+ String[] extPaths = kylinExtLogBaseDir.split(",");
+ for(String path:extPaths){
+ if(!StringUtils.isEmpty(path)){
+ logger.info("Use ext log dir=" + path);
+ logDirList.add(path.trim());
+ }
+ }
+ }
+
+ String kylinHome = getKylinHome();
+ if (!StringUtils.isEmpty(kylinHome))
+ if(logDirList.isEmpty()){
+ throw new RuntimeException("Didn't find KYLIN_CONF or KYLIN_HOME or KYLIN_EXT_LOG_BASE_DIR, please set one of them");
+ }
+ else{
+ String path = kylinHome + File.separator + "tomcat" + File.separator + "logs";
+ logDirList.add(path);
+ }
+
+ return logDirList;
+ }
+
+
+ public String getMetadataUrl(){
+ return this.monitorConfig.getProperty(KYLIN_METADATA_URL);
+ }
+
+ public String getExtLogBaseDir() {
+ return this.monitorConfig.getProperty(KYLIN_EXT_LOG_BASE_DIR);
+ }
+
+ public String getKylinHdfsWorkingDir() {
+ return this.monitorConfig.getProperty(KYLIN_HDFS_WORKING_DIR);
+ }
+ public String getQueryLogParseResultDir() {
+ return this.getKylinHdfsWorkingDir()+"/performance/query/";
+ }
+
+ public String getQueryLogResultTable() {
+ String query_log_parse_result_table = this.monitorConfig.getProperty(QUERY_LOG_PARSE_RESULT_TABLE);
+ if(!StringUtils.isEmpty(query_log_parse_result_table)){
+ return query_log_parse_result_table;
+ }else{
+ return DEFAULT_QUERY_LOG_PARSE_RESULT_TABLE;
+ }
+ }
+
+ public String getRequestLogParseResultDir() {
+ return this.getKylinHdfsWorkingDir()+"/performance/request/";
+ }
+
+ public String getHiveJdbcConUrl() {
+ return this.monitorConfig.getProperty(HIVE_JDBC_CON_URL);
+ }
+
+ public String getLogParseResultMetaDir() {
+ return this.getKylinHdfsWorkingDir()+"/performance/metadata/";
+ }
+
+ public String getDeployEnv(){
+ return this.monitorConfig.getProperty(DEPLOY_ENV);
+ }
+
+
+ public static void addClasspath(String path) throws Exception {
+ File file = new File(path);
+
+ if (file.exists()) {
+ URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+ Class<URLClassLoader> urlClass = URLClassLoader.class;
+ Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
+ method.setAccessible(true);
+ method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java b/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
new file mode 100644
index 0000000..a7230e6
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/DebugClient.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.apache.log4j.Logger;
+import java.io.File;
+
+
+/**
+ * Created by jiazhong on 2015/5/7
+ */
+public class DebugClient {
+
+ static{
+ //set catalina.home temp
+ System.setProperty(ConfigUtils.CATALINA_HOME, "../server/");
+ }
+
+ final static Logger logger = Logger.getLogger(DebugClient.class);
+
+
+ public static void main(String[] args) throws Exception {
+
+ // test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
+ ConfigUtils.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
+
+ //set log base dir ,will also get from $KYLIN_HOME/tomcat/logs and config [ext.log.base.dir] in kylin.properties
+ System.setProperty(ConfigUtils.KYLIN_LOG_CONF_HOME, "../server/logs");
+
+ //get kylin.properties ,if not exist will get from $KYLIN_HOME/conf/
+ System.setProperty(ConfigUtils.KYLIN_CONF, "../examples/test_case_data/sandbox");
+
+
+ QueryParser queryParser = new QueryParser();
+ HiveJdbcClient jdbcClient = new HiveJdbcClient();
+
+ try {
+ queryParser.start();
+ jdbcClient.start();
+ } catch (Exception e) {
+ logger.info("Exception ",e);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java b/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
new file mode 100644
index 0000000..555409b
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/FileUtils.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import au.com.bytecode.opencsv.CSVWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+/**
+ * Created by jiazhong on 2015/6/18.
+ */
+public class FileUtils {
+
+ final static Logger logger = Logger.getLogger(FileUtils.class);
+
+ public static boolean pathCheck(String filePath) throws IOException {
+ logger.info("checking file:"+filePath);
+ FileSystem fs = null;
+ try {
+ Configuration conf = new Configuration();
+ fs = FileSystem.get(conf);
+ org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(filePath);
+ if (!fs.exists(path)) {
+ fs.create(path);
+ fs.close();
+ return false;
+ }
+ } catch (Exception e) {
+ fs.close();
+ logger.info("Failed to init:", e);
+ }
+ return true;
+ }
+
+
+ /*
+ * write parse result to hdfs
+ */
+ public static void clearHdfsFile(String dPath) throws IOException {
+ OutputStreamWriter writer = null;
+ FileSystem fs = null;
+ try {
+ fs = getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ writer = new OutputStreamWriter(fs.create(resultStorePath, true));
+
+ } catch (Exception e) {
+ logger.info("Exception", e);
+ } finally {
+ writer.close();
+ fs.close();
+ }
+ }
+
+ /*
+ * write parse result to hdfs
+ */
+ public static void appendResultToHdfs(String dPath, String[] record) throws IOException {
+ OutputStreamWriter writer = null;
+ CSVWriter cwriter = null;
+ FileSystem fs = null;
+ try {
+ fs = getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ writer = new OutputStreamWriter(fs.append(resultStorePath));
+ cwriter = new CSVWriter(writer, '|', CSVWriter.NO_QUOTE_CHARACTER);
+
+ cwriter.writeNext(record);
+
+ } catch (Exception e) {
+ logger.info("Exception", e);
+ } finally {
+ writer.close();
+ cwriter.close();
+ fs.close();
+ }
+ }
+
+ /*
+ * get hdfs fileSystem
+ */
+ public static FileSystem getHdfsFileSystem() throws IOException {
+ Configuration conf = new Configuration();
+// conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.newInstance(conf);
+ } catch (IOException e) {
+ fs.close();
+ logger.info("Failed to get hdfs FileSystem", e);
+ }
+ return fs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java b/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
new file mode 100644
index 0000000..71a3a68
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/HiveJdbcClient.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.apache.log4j.Logger;
+import org.datanucleus.util.StringUtils;
+
+import java.io.IOException;
+import java.sql.*;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+/**
+ * Created by jiazhong on 2015/6/17.
+ */
+public class HiveJdbcClient {
+
+ static String SQL_GENERATE_QUERY_LOG_TABLE = "CREATE EXTERNAL TABLE IF NOT EXISTS [QUERY_LOG_TABLE_NAME] (REQUEST_TIME STRING,REQUEST_DATE DATE, QUERY_SQL STRING,QUERY_USER STRING,IS_SUCCESS STRING,QUERY_LATENCY DECIMAL(19,4),QUERY_PROJECT STRING,REALIZATION_NAMES STRING,CUBOID_IDS STRING,TOTAL_SCAN_COUNT INT,RESULT_ROW_COUNT INT,ACCEPT_PARTIAL STRING,IS_PARTIAL_RESULT STRING,HIT_CACHE STRING,MESSAGE STRING,DEPLOY_ENV STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\\n' LOCATION '[QUERY_LOG_PARSE_RESULT_DIR]' TBLPROPERTIES (\"SKIP.HEADER.LINE.COUNT\"=\"1\")";
+
+ static String SQL_TOTAL_QUERY_USER = "SELECT COUNT(DISTINCT QUERY_USER) FROM [QUERY_LOG_TABLE_NAME]";
+
+ static String SQL_AVG_DAY_QUERY = "SELECT AVG(A.COUNT_QUERY) FROM (SELECT COUNT(*) COUNT_QUERY,REQUEST_DATE FROM [QUERY_LOG_TABLE_NAME] GROUP BY REQUEST_DATE) A";
+
+ static String SQL_LAST_30_DAYILY_QUERY_COUNT = "SELECT REQUEST_DATE, COUNT(*) FROM [QUERY_LOG_TABLE_NAME] WHERE REQUEST_DATE>=[START_DATE] AND REQUEST_DATE<[END_DATE] GROUP BY REQUEST_DATE";
+
+
+ //last 30 days
+ static String SQL_90_PERCENTTILE_LAST_30_DAY = "SELECT PERCENTILE_APPROX(LOG.QUERY_LATENCY,0.9) FROM (SELECT QUERY_LATENCY FROM [QUERY_LOG_TABLE_NAME] WHERE IS_SUCCESS='true' AND REQUEST_DATE>=[START_DATE] AND REQUEST_DATE<[END_DATE]) LOG";
+
+ //0.9,0.95 [each day] percentile in last 30 days
+ static String SQL_EACH_DAY_PERCENTILE = "SELECT REQUEST_DATE, PERCENTILE_APPROX(QUERY_LATENCY,ARRAY(0.9,0.95)),COUNT(*) FROM [QUERY_LOG_TABLE_NAME] WHERE IS_SUCCESS='true' AND REQUEST_DATE>=[START_DATE] AND REQUEST_DATE<[END_DATE] GROUP BY REQUEST_DATE";
+
+ //0.9,0.95 [project] percentile in last 30 days
+ static String SQL_DAY_PERCENTILE_BY_PROJECT = "SELECT QUERY_PROJECT, PERCENTILE_APPROX(QUERY_LATENCY,ARRAY(0.9,0.95)) FROM [QUERY_LOG_TABLE_NAME] WHERE IS_SUCCESS='true' AND REQUEST_DATE>=[START_DATE] AND REQUEST_DATE<[END_DATE] GROUP BY QUERY_PROJECT";
+
+
+
+ static String QUERY_LOG_TABLE_NAME = "KYLIN_QUERY_LOG";
+
+ final static Logger logger = Logger.getLogger(HiveJdbcClient.class);
+
+
+ private static String driverName = "org.apache.hive.jdbc.HiveDriver";
+ private static ConfigUtils monitorConfig = ConfigUtils.getInstance();
+
+ static {
+ try {
+ Class.forName(driverName);
+ monitorConfig.loadMonitorParam();
+ QUERY_LOG_TABLE_NAME = monitorConfig.getQueryLogResultTable();
+ if (StringUtils.isEmpty(QUERY_LOG_TABLE_NAME)) {
+ logger.error("table name not defined ,please set param [query.log.parse.result.table] in kylin.properties");
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ /*
+ * will create external hive table for [query] log parse result csv file on hdfs
+ * and will generate metric data in csv file on hdfs for web dashboard
+ */
+ public void start() throws SQLException, IOException {
+
+ String CON_URL = monitorConfig.getHiveJdbcConUrl();
+
+ Connection con = DriverManager.getConnection(CON_URL, "", "");
+ Statement stmt = con.createStatement();
+ ResultSet res = null;
+
+ SQL_GENERATE_QUERY_LOG_TABLE = generateQueryLogSql();
+ logger.info("Running Sql (Create Table):" + SQL_GENERATE_QUERY_LOG_TABLE);
+ stmt.execute(SQL_GENERATE_QUERY_LOG_TABLE);
+
+ SQL_TOTAL_QUERY_USER = generateUserCountSql();
+ logger.info("Running Sql (Total User):" + SQL_TOTAL_QUERY_USER);
+ res = stmt.executeQuery(SQL_TOTAL_QUERY_USER);
+
+ String total_query_user_path = monitorConfig.getLogParseResultMetaDir() + "total_query_user.csv";
+ FileUtils.pathCheck(total_query_user_path);
+ FileUtils.clearHdfsFile(total_query_user_path);
+ while (res.next()) {
+ FileUtils.appendResultToHdfs(total_query_user_path, new String[]{res.getString(1)});
+ logger.info("Total User:" + res.getString(1));
+ }
+
+ SQL_AVG_DAY_QUERY = generateAvgDayQuery();
+ logger.info("Running Sql (Avg Day Query):" + SQL_AVG_DAY_QUERY);
+ res = stmt.executeQuery(SQL_AVG_DAY_QUERY);
+
+ String avg_day_query_path = monitorConfig.getLogParseResultMetaDir() + "avg_day_query.csv";
+ FileUtils.pathCheck(avg_day_query_path);
+ FileUtils.clearHdfsFile(avg_day_query_path);
+ while (res.next()) {
+ FileUtils.appendResultToHdfs(avg_day_query_path, new String[]{res.getString(1)});
+ logger.info("avg day query:" + res.getString(1));
+ }
+
+
+ SQL_LAST_30_DAYILY_QUERY_COUNT = generateLast30DayilyQueryCount();
+ logger.info("Running Sql (Daily Query Count):" + SQL_LAST_30_DAYILY_QUERY_COUNT);
+ res = stmt.executeQuery(SQL_LAST_30_DAYILY_QUERY_COUNT);
+
+ String last_30_daily_query_count_path = monitorConfig.getLogParseResultMetaDir() + "last_30_daily_query_count.csv";
+ FileUtils.pathCheck(last_30_daily_query_count_path);
+ FileUtils.clearHdfsFile(last_30_daily_query_count_path);
+ while (res.next()) {
+ FileUtils.appendResultToHdfs(last_30_daily_query_count_path, new String[]{res.getString(1),res.getString(2)});
+ logger.info("last 30 daily query count:" + res.getString(1)+","+res.getString(2));
+ }
+
+
+ //90 percentile latency for all query in last 30 days
+ SQL_90_PERCENTTILE_LAST_30_DAY = generateNintyPercentileSql();
+ logger.info("Running Sql (last 30 days ,90 percentile query latency):" + SQL_90_PERCENTTILE_LAST_30_DAY);
+ res = stmt.executeQuery(SQL_90_PERCENTTILE_LAST_30_DAY);
+
+ String last_30_day_90_percentile_latency = monitorConfig.getLogParseResultMetaDir() + "last_30_day_90_percentile_latency.csv";
+ FileUtils.pathCheck(last_30_day_90_percentile_latency);
+ FileUtils.clearHdfsFile(last_30_day_90_percentile_latency);
+ while (res.next()) {
+ FileUtils.appendResultToHdfs(last_30_day_90_percentile_latency, new String[]{res.getString(1)});
+ logger.info("last 30 day 90 percentile latency:" + res.getString(1));
+ }
+
+ //90,95 project percentile latency for all query in last 30 days
+ SQL_DAY_PERCENTILE_BY_PROJECT = generateProjectPercentileSql();
+ logger.info("Running Sql (last 30 days ,90,95 percentile query latency by project):" + SQL_DAY_PERCENTILE_BY_PROJECT);
+ res = stmt.executeQuery(SQL_DAY_PERCENTILE_BY_PROJECT);
+
+ String last_30_day_project_percentile_latency_path = monitorConfig.getLogParseResultMetaDir() + "project_90_95_percentile_latency.csv";
+ FileUtils.pathCheck(last_30_day_project_percentile_latency_path);
+ FileUtils.clearHdfsFile(last_30_day_project_percentile_latency_path);
+ while (res.next() && res.getMetaData().getColumnCount() == 2) {
+ FileUtils.appendResultToHdfs(last_30_day_project_percentile_latency_path, new String[]{res.getString(1), res.getString(2)});
+ logger.info(res.getString(1) + "," + res.getString(2));
+ }
+
+ //0.9,0.95 percentile latency of every day in last 30 day
+ SQL_EACH_DAY_PERCENTILE = generateEachDayPercentileSql();
+ logger.info("Running sql (0.9,0.95 latency):" + SQL_EACH_DAY_PERCENTILE);
+ String each_day_percentile_file = monitorConfig.getLogParseResultMetaDir() + "each_day_90_95_percentile_latency.csv";
+ FileUtils.pathCheck(each_day_percentile_file);
+ FileUtils.clearHdfsFile(each_day_percentile_file);
+
+ res = stmt.executeQuery(SQL_EACH_DAY_PERCENTILE);
+ while (res.next() && res.getMetaData().getColumnCount() == 3) {
+ FileUtils.appendResultToHdfs(each_day_percentile_file, new String[]{res.getString(1), res.getString(2),res.getString(3)});
+ logger.info(res.getString(1) + "," + res.getString(2)+ "," + res.getString(3));
+ }
+
+ }
+
+ public String generateQueryLogSql() {
+ String query_log_parse_result_dir = monitorConfig.getQueryLogParseResultDir();
+ String query_log_table_name = monitorConfig.getQueryLogResultTable();
+ return SQL_GENERATE_QUERY_LOG_TABLE.replace("[QUERY_LOG_PARSE_RESULT_DIR]", query_log_parse_result_dir).replace("[QUERY_LOG_TABLE_NAME]", query_log_table_name);
+ }
+
+ public String generateUserCountSql() {
+ return SQL_TOTAL_QUERY_USER.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ }
+
+ public String generateAvgDayQuery() {
+ return SQL_AVG_DAY_QUERY.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ }
+
+ public String generateLast30DayilyQueryCount() {
+ SQL_LAST_30_DAYILY_QUERY_COUNT = SQL_LAST_30_DAYILY_QUERY_COUNT.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ return monthStasticSqlConvert(SQL_LAST_30_DAYILY_QUERY_COUNT);
+ }
+
+
+
+ //last 30 days
+ public String generateNintyPercentileSql() {
+ SQL_90_PERCENTTILE_LAST_30_DAY = SQL_90_PERCENTTILE_LAST_30_DAY.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ return monthStasticSqlConvert(SQL_90_PERCENTTILE_LAST_30_DAY);
+ }
+
+ //last 30 days,each day 90,95 percentile
+ public String generateProjectPercentileSql() {
+ SQL_DAY_PERCENTILE_BY_PROJECT = SQL_DAY_PERCENTILE_BY_PROJECT.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ return monthStasticSqlConvert(SQL_DAY_PERCENTILE_BY_PROJECT);
+ }
+
+ public String generateEachDayPercentileSql() {
+ SQL_EACH_DAY_PERCENTILE = SQL_EACH_DAY_PERCENTILE.replace("[QUERY_LOG_TABLE_NAME]", QUERY_LOG_TABLE_NAME);
+ return monthStasticSqlConvert(SQL_EACH_DAY_PERCENTILE);
+ }
+
+ public String monthStasticSqlConvert(String sql) {
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+ Calendar cal = Calendar.getInstance();
+ cal.add(Calendar.DATE, -1);
+ String endDate = format.format(cal.getTime());
+ cal.add(Calendar.DATE, -30);
+ String startDate = format.format(cal.getTime());
+ return sql.replace("[START_DATE]", "'" + startDate + "'").replace("[END_DATE]", "'" + endDate + "'");
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java b/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
new file mode 100644
index 0000000..c01cf66
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/MonitorMetaManager.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Created by jiazhong on 2015/5/25.
+ */
+public class MonitorMetaManager {
+
+ private static ConfigUtils monitorConfig = ConfigUtils.getInstance();
+
+ static String TABLE_NAME = "kylin_metadata";
+ final static String COLUMN_FAMILY = "f";
+ final static String COLUMN = "c";
+ final static String ROW_KEY_QUERY_READ_FILES = "/performance/query_log_files_already_read";
+ final static String ROW_KEY_QUERY_READING_FILE = "/performance/query_log_file_reading";
+ final static String ROW_KEY_QUERY_READING_FILE_LINE = "/performance/query_log_file_reading";
+
+
+ final static String ROW_KEY_API_REQ_LOG_READ_FILES = "/performance/api_req_log_files_already_read";
+
+
+ final static Logger logger = Logger.getLogger(MonitorMetaManager.class);
+
+ static Configuration conf = null;
+
+ static {
+ try {
+ monitorConfig.loadMonitorParam();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ conf = HBaseConfiguration.create();
+ }
+
+
+ /*
+ * meta data initialize
+ * @unused
+ */
+ public static void init() throws Exception {
+ MonitorMetaManager.TABLE_NAME = MonitorMetaManager.getMetadataUrlPrefix();
+ logger.info("Monitor Metadata Table :"+MonitorMetaManager.TABLE_NAME);
+ logger.info("init monitor metadata,create table if not exist");
+ MonitorMetaManager.creatTable(TABLE_NAME, new String[]{COLUMN_FAMILY});
+ }
+
+ public static String getMetadataUrlPrefix() {
+ String hbaseMetadataUrl = monitorConfig.getMetadataUrl();
+ String defaultPrefix = "kylin_metadata";
+ int cut = hbaseMetadataUrl.indexOf('@');
+ String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
+ return tmp;
+ }
+
+
+ /*
+ * mark query file as read after parsing
+ */
+ public static void markQueryFileAsRead(String filename) throws IOException {
+ String read_query_log_file = MonitorMetaManager.getReadQueryLogFiles();
+ if(StringUtils.isEmpty(read_query_log_file)){
+ read_query_log_file = filename;
+ }else{
+ read_query_log_file = read_query_log_file.concat(",").concat(filename);
+ }
+ MonitorMetaManager.updateData(TABLE_NAME, ROW_KEY_QUERY_READ_FILES, COLUMN_FAMILY,COLUMN, read_query_log_file);
+ }
+
+ /*
+ * mark reading file for tracking
+ */
+ public static void markQueryReadingFile(String query_reading_file) throws IOException {
+ MonitorMetaManager.updateData(TABLE_NAME, ROW_KEY_QUERY_READING_FILE, COLUMN_FAMILY,COLUMN, query_reading_file);
+ }
+
+ /*
+ * mark reading line for tracking
+ */
+ public static void markQueryReadingLine(String line_num) throws IOException {
+ MonitorMetaManager.updateData(TABLE_NAME, ROW_KEY_QUERY_READING_FILE_LINE, COLUMN_FAMILY,COLUMN, line_num);
+ }
+
+
+ /*
+ * get has been read file name list
+ */
+ public static String[] getReadQueryLogFileList() throws IOException {
+ String fileList = MonitorMetaManager.getReadQueryLogFiles();
+ return fileList.split(",");
+ }
+
+ /*
+ * get has been read query log file
+ */
+ public static String getReadQueryLogFiles() throws IOException {
+ return getListWithRowkey(TABLE_NAME, ROW_KEY_QUERY_READ_FILES);
+ }
+
+
+ /*
+ * get has been read file
+ */
+ public static String getListWithRowkey(String table, String rowkey) throws IOException {
+ Result result = getResultByRowKey(table, rowkey);
+ String fileList = null;
+ if (result.list() != null) {
+ for (KeyValue kv : result.list()) {
+ fileList = Bytes.toString(kv.getValue());
+ }
+
+ }
+ fileList = fileList==null?"":fileList;
+ return fileList;
+ }
+
+
+ /*
+ * mark api req log file as read after parsing
+ */
+ public static void markApiReqLogFileAsRead(String filename) throws IOException {
+ String read_api_req_log_files = MonitorMetaManager.getReadApiReqLogFiles();
+ if (StringUtils.isEmpty(read_api_req_log_files)) {
+ read_api_req_log_files = filename;
+ } else {
+ read_api_req_log_files = read_api_req_log_files.concat(",").concat(filename);
+ }
+ MonitorMetaManager.updateData(TABLE_NAME, ROW_KEY_API_REQ_LOG_READ_FILES, COLUMN_FAMILY,COLUMN, read_api_req_log_files);
+ }
+
+
+ /*
+ * get has been read log file name list
+ */
+ public static String[] getReadApiReqLogFileList() throws IOException {
+ String fileList = MonitorMetaManager.getReadApiReqLogFiles();
+ return fileList.split(",");
+ }
+
+ /*
+ * get has been read api request log file
+ */
+ public static String getReadApiReqLogFiles() throws IOException {
+ return getListWithRowkey(TABLE_NAME, ROW_KEY_API_REQ_LOG_READ_FILES);
+ }
+
+
+ /*
+ * create table in hbase
+ */
+ public static void creatTable(String tableName, String[] family) throws Exception {
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ for (int i = 0; i < family.length; i++) {
+ desc.addFamily(new HColumnDescriptor(family[i]));
+ }
+ if (admin.tableExists(tableName)) {
+ logger.info("table Exists!");
+ } else {
+ admin.createTable(desc);
+ logger.info("create table Success!");
+ }
+ }
+
+ /*
+ * update cell in hbase
+ */
+ public static void updateData(String tableName, String rowKey, String family,String column, String value) throws IOException {
+ HTable table = new HTable(conf, Bytes.toBytes(tableName));
+ Put put = new Put(rowKey.getBytes());
+ put.add(family.getBytes(), column.getBytes(), value.getBytes());
+ try {
+ table.put(put);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ logger.info("end insert data ......");
+ }
+
+ /*
+ * get result by rowkey
+ */
+ public static Result getResultByRowKey(String tableName, String rowKey) throws IOException {
+ HTable table = new HTable(conf, Bytes.toBytes(tableName));
+ Get get = new Get(Bytes.toBytes(rowKey));
+ Result result = table.get(get);
+ return result;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java b/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
new file mode 100644
index 0000000..fc8a403
--- /dev/null
+++ b/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.monitor;
+
+import au.com.bytecode.opencsv.CSVWriter;
+import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.log4j.Logger;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author jiazhong
+ */
+public class QueryParser {
+
+ final static Logger logger = Logger.getLogger(QueryParser.class);
+ final static Charset ENCODING = StandardCharsets.UTF_8;
+ final static String QUERY_LOG_FILE_PATTERN = "kylin_query.log.(\\d{4}-\\d{2}-\\d{2})$";
+ final static String QUERY_LOG_PARSE_RESULT_FILENAME = "kylin_query_log.csv";
+ static String QUERY_PARSE_RESULT_PATH = null;
+ static String DEPLOY_ENV;
+
+ final static String[] KYLIN_QUERY_CSV_HEADER = {"REQ_TIME","REQ_DATE", "SQL", "USER", "IS_SUCCESS", "LATENCY", "PROJECT", "REALIZATION NAMES", "CUBOID IDS", "TOTAL SCAN COUNT", "RESULT ROW COUNT", "ACCEPT PARTIAL", "IS PARTIAL RESULT", "HIT CACHE", "MESSAGE","DEPLOY_ENV"};
+
+ private ConfigUtils monitorConfig;
+
+ public QueryParser() {
+ monitorConfig = ConfigUtils.getInstance();
+ try {
+ monitorConfig.loadMonitorParam();
+ DEPLOY_ENV = monitorConfig.getDeployEnv();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * will parse kylin query log files ,and append result to csv on hdfs
+ * files read will be marked,will not read again
+ */
+
+ public void start() throws IOException, ParseException {
+ QueryParser.QUERY_PARSE_RESULT_PATH = ConfigUtils.getInstance().getQueryLogParseResultDir() + QUERY_LOG_PARSE_RESULT_FILENAME;
+ this.parseQueryInit();
+
+ //get query file has been read
+ String[] hasReadFiles = MonitorMetaManager.getReadQueryLogFileList();
+
+ //get all log files
+ List<File> files = this.getQueryLogFiles();
+
+ for (File file : files) {
+ if (!Arrays.asList(hasReadFiles).contains(file.getName())) {
+ this.parseQueryLog(file.getPath(), QueryParser.QUERY_PARSE_RESULT_PATH);
+ MonitorMetaManager.markQueryFileAsRead(file.getName());
+ }
+ }
+ }
+
+ public void parseQueryInit() throws IOException {
+ logger.info("parse query initializing...");
+ FileSystem fs = null;
+ try {
+ Configuration conf = new Configuration();
+ fs = FileSystem.get(conf);
+ org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(QueryParser.QUERY_PARSE_RESULT_PATH);
+ if (!fs.exists(path)) {
+ fs.create(path);
+ fs.close(); //need to close before get FileSystem again
+ this.writeResultToHdfs(QueryParser.QUERY_PARSE_RESULT_PATH, QueryParser.KYLIN_QUERY_CSV_HEADER);
+ }
+ } catch (IOException e) {
+ fs.close();
+ logger.info("Failed to init:", e);
+ }
+ }
+
+ //parse query log and convert to csv file to hdfs
+ public void parseQueryLog(String filePath, String dPath) throws ParseException, IOException {
+
+ logger.info("Start parsing file " + filePath + " !");
+
+// writer config init
+ FileSystem fs = this.getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ OutputStreamWriter writer = new OutputStreamWriter(fs.append(resultStorePath));
+ CSVWriter cwriter = new CSVWriter(writer,'|',CSVWriter.NO_QUOTE_CHARACTER);
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ Pattern p_query_start = Pattern.compile("^\\[.*\\]:\\[(.*),.*\\]\\[.*\\]\\[.*QueryService.logQuery.*\\].*");
+ Pattern p_query_end = Pattern.compile("^Message:(.*)$");
+ Pattern p_query_body = Pattern.compile("^\\[.*\\]:\\[((\\d{4}-\\d{2}-\\d{2}).*)\\]\\[.*\\]\\[.*\\].*\n^=+\\[QUERY\\]=+\n^SQL:(.*)\n^User:(.*)\n^Success:(.*)\n^Duration:(.*)\n^Project:(.*)\n^(Realization Names|Cube Names): \\[(.*)\\]\n^Cuboid Ids: \\[(.*)\\]\n^Total scan count:(.*)\n^Result row count:(.*)\n^Accept Partial:(.*)\n(^Is Partial Result:(.*)\n)?^Hit Cache:(.*)\n^Message:(.*)", Pattern.MULTILINE);
+ Matcher m_query_start = p_query_start.matcher("");
+ Matcher m_query_end = p_query_end.matcher("");
+ Matcher m_query_body = p_query_body.matcher("");
+
+ boolean query_start = false;
+ StringBuffer query_body = new StringBuffer("");
+ Path path = Paths.get(filePath);
+ try {
+ BufferedReader reader = Files.newBufferedReader(path, ENCODING);
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ m_query_start.reset(line); //reset the input
+ m_query_end.reset(line);
+
+ // set start flag ,clear StringBuffer
+ if (m_query_start.find()) {
+ query_start = true;
+ query_body = new StringBuffer("");
+ }
+ if (query_start) {
+ query_body.append(line + "\n");
+ }
+ if (m_query_end.find()) {
+ query_start = false;
+ m_query_body.reset(query_body);
+ logger.info("parsing query...");
+ logger.info(query_body);
+// skip group(8) and group(14)
+ if (m_query_body.find()) {
+ ArrayList<String> groups = new ArrayList<String>();
+ int grp_count = m_query_body.groupCount();
+ for (int i = 1; i <= grp_count; i++) {
+ if (i != 8 && i != 14) {
+ String grp_item = m_query_body.group(i);
+ grp_item = grp_item ==null?"":grp_item.trim();
+ groups.add(grp_item);
+ }
+ }
+
+ long start_time = format.parse(groups.get(0)).getTime() - (int) (Double.parseDouble(groups.get(5)) * 1000);
+ groups.set(0, format.format(new Date(start_time)));
+ groups.add(DEPLOY_ENV);
+ String[] recordArray = groups.toArray(new String[groups.size()]);
+// write to hdfs
+ cwriter.writeNext(recordArray);
+
+ }
+
+ }
+
+ }
+ } catch (IOException ex) {
+ logger.info("Failed to write to hdfs:", ex);
+ } finally {
+ writer.close();
+ cwriter.close();
+ fs.close();
+ }
+
+ logger.info("Finish parsing file " + filePath + " !");
+
+ }
+
+ /*
+ * write parse result to hdfs
+ */
+ public void writeResultToHdfs(String dPath, String[] record) throws IOException {
+ OutputStreamWriter writer = null;
+ CSVWriter cwriter = null;
+ FileSystem fs = null;
+ try {
+ fs = this.getHdfsFileSystem();
+ org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath);
+ writer = new OutputStreamWriter(fs.append(resultStorePath));
+ cwriter = new CSVWriter(writer,'|',CSVWriter.NO_QUOTE_CHARACTER);
+
+ cwriter.writeNext(record);
+
+ } catch (IOException e) {
+ logger.info("Exception", e);
+ } finally {
+ writer.close();
+ cwriter.close();
+ fs.close();
+ }
+ }
+
+
+ /*
+ * get all query log files
+ */
+ public List<File> getQueryLogFiles() {
+ List<File> logFiles = new ArrayList<File>();
+
+ List<String> query_log_dir_list = monitorConfig.getLogBaseDir();
+ FileFilter filter = new RegexFileFilter(QUERY_LOG_FILE_PATTERN);
+
+ for (String path : query_log_dir_list) {
+ logger.info("fetching query log file from path:" + path);
+ File query_log_dir = new File(path);
+ File[] query_log_files = query_log_dir.listFiles(filter);
+ if (query_log_files == null) {
+ logger.warn("no query log file found under path" + path);
+ continue;
+ }
+
+ Collections.addAll(logFiles, query_log_files);
+ }
+ return logFiles;
+ }
+
+
+ /*
+ * get hdfs fileSystem
+ */
+ public FileSystem getHdfsFileSystem() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(conf);
+ } catch (IOException e) {
+ fs.close();
+ logger.info("Failed to get hdfs FileSystem", e);
+ }
+ return fs;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/monitor/src/main/resources/log4j.properties b/monitor/src/main/resources/log4j.properties
new file mode 100644
index 0000000..6dec581
--- /dev/null
+++ b/monitor/src/main/resources/log4j.properties
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
+
+log4j.appender.monitor=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.monitor.layout=org.apache.log4j.PatternLayout
+log4j.appender.monitor.File=${CATALINA_HOME}logs/kylin_monitor.log
+log4j.appender.monitor.layout.ConversionPattern=[%t]:[%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%l] - %m%n
+log4j.appender.monitor.Append=true
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.org.apache.kylin=DEBUG,monitor
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/monitor/src/test/java/org/apache/kylin/monitor/ParseLogTest.java
----------------------------------------------------------------------
diff --git a/monitor/src/test/java/org/apache/kylin/monitor/ParseLogTest.java b/monitor/src/test/java/org/apache/kylin/monitor/ParseLogTest.java
new file mode 100644
index 0000000..7b4e7b7
--- /dev/null
+++ b/monitor/src/test/java/org/apache/kylin/monitor/ParseLogTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.monitor;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+
+/**
+ * Created by jiazhong on 2015/5/27.
+ */
+public class ParseLogTest {
+
+ private QueryParser queryParser;
+ private ApiRequestParser apiReqParser;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ //set catalina.home temp
+ System.setProperty(ConfigUtils.CATALINA_HOME, "../server/");
+
+ //test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
+ ConfigUtils.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
+
+ //get log base dir
+ System.setProperty(ConfigUtils.KYLIN_LOG_CONF_HOME, "../examples/test_case_data/performance_data");
+
+ //get kylin.properties
+ System.setProperty(ConfigUtils.KYLIN_CONF, "../examples/test_case_data/sandbox");
+
+ }
+
+
+ @Before
+ public void before(){
+ queryParser = new QueryParser();
+ apiReqParser = new ApiRequestParser();
+ }
+
+ @Test
+ public void test() throws Exception {
+ testQueryParseing();
+ testApiReqParsing();
+ }
+
+ private void testQueryParseing() throws IOException, ParseException {
+ queryParser.getQueryLogFiles();
+ }
+
+ private void testApiReqParsing(){
+ apiReqParser.getRequestLogFiles();
+ }
+
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5748328..cdd8191 100644
--- a/pom.xml
+++ b/pom.xml
@@ -557,6 +557,7 @@
<module>query</module>
<module>server</module>
<module>jdbc</module>
+ <module>monitor</module>
<module>invertedindex</module>
</modules>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/script/prepare.sh
----------------------------------------------------------------------
diff --git a/script/prepare.sh b/script/prepare.sh
index a8b1c7d..f0702f8 100755
--- a/script/prepare.sh
+++ b/script/prepare.sh
@@ -33,11 +33,13 @@ cp server/target/kylin-server-${version}.war tomcat/webapps/kylin.war
cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
cp jdbc/target/kylin-jdbc-${version}.jar lib/kylin-jdbc-${version}.jar
+cp monitor/target/kylin-monitor-${version}.jar lib/kylin-monitor-${version}.jar
# Copied file becomes 000 for some env (e.g. my Cygwin)
chmod 644 tomcat/webapps/kylin.war
chmod 644 lib/kylin-job-${version}.jar
chmod 644 lib/kylin-coprocessor-${version}.jar
chmod 644 lib/kylin-jdbc-${version}.jar
+chmod 644 lib/kylin-monitor-${version}.jar
echo "add js css to war"
if [ ! -d "webapp/dist" ]
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 1578c62..8500426 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -444,6 +444,11 @@
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.isrsal</groupId>
+ <artifactId>spring-mvc-logger</artifactId>
+ <version>0.2</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/java/org/apache/kylin/rest/controller/PerformanceController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/PerformanceController.java b/server/src/main/java/org/apache/kylin/rest/controller/PerformanceController.java
new file mode 100644
index 0000000..735db51
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/controller/PerformanceController.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import com.codahale.metrics.annotation.Metered;
+import com.codahale.metrics.annotation.Timed;
+import net.sf.ehcache.CacheManager;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.PerformService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.*;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Handle query requests.
+ *
+ * @author xduo
+ */
+@Controller
+@RequestMapping(value = "/performance")
+public class PerformanceController extends BasicController {
+
+ private static final Logger logger = LoggerFactory.getLogger(PerformanceController.class);
+
+
+ @Autowired
+ private PerformService performService;
+
+ @Autowired
+ private CubeService cubeService;
+
+ @Autowired
+ private CacheManager cacheManager;
+
+ @RequestMapping(value = "/eachDayPercentile", method = RequestMethod.GET)
+ @ResponseBody
+ @Timed(name = "eachDayPercentile")
+ public List<String[]> eachDayPercentile() throws IOException {
+ return performService.eachDayPercentile();
+ }
+
+ @RequestMapping(value = "/projectPercentile", method = RequestMethod.GET)
+ @ResponseBody
+ @Timed(name = "projectPercentile")
+ public List<String[]> projectPercentile() throws IOException {
+ return performService.projectPercentile();
+ }
+
+
+ @RequestMapping(value = "/last30DayPercentile", method = RequestMethod.GET)
+ @ResponseBody
+ @Timed(name = "last30DayPercentile")
+ public List<String[]> last30DayPercentile() throws IOException {
+ return performService.last30DayPercentile();
+ }
+
+ @RequestMapping(value = "/cubesStorage", method = {RequestMethod.GET})
+ @ResponseBody
+ @Metered(name = "cubesStorage")
+ public List<CubeInstance> getCubeStorage() {
+ return cubeService.listAllCubes(null, null);
+ }
+
+
+ @RequestMapping(value = "/totalQueryUser", method = {RequestMethod.GET})
+ @ResponseBody
+ @Metered(name = "totalQueryUser")
+ public List<String[]> totalQueryUser() throws IOException {
+ return performService.getTotalQueryUser();
+ }
+
+ @RequestMapping(value = "/dailyQueryCount", method = {RequestMethod.GET})
+ @ResponseBody
+ @Metered(name = "dailyQueryCount")
+ public List<String[]> dailyQueryCount() throws IOException {
+ return performService.dailyQueryCount();
+ }
+
+ @RequestMapping(value = "/avgDayQuery", method = {RequestMethod.GET})
+ @ResponseBody
+ @Metered(name = "avgDayQuery")
+ public List<String[]> avgDayQuery() throws IOException {
+ return performService.avgDayQuery();
+ }
+
+ @RequestMapping(value = "/listCubes", method = {RequestMethod.GET})
+ @ResponseBody
+ @Metered(name = "listCubes")
+ public List<CubeInstance> getCubes() {
+ return cubeService.listAllCubes(null,null);
+ }
+
+
+
+ public void setCubeService(CubeService cubeService) {
+ this.cubeService = cubeService;
+ }
+
+ public void setPerformService(PerformService performService) {
+ this.performService = performService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/java/org/apache/kylin/rest/filter/KylinApiFilter.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/filter/KylinApiFilter.java b/server/src/main/java/org/apache/kylin/rest/filter/KylinApiFilter.java
new file mode 100644
index 0000000..035b57d
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/filter/KylinApiFilter.java
@@ -0,0 +1,121 @@
+package org.apache.kylin.rest.filter;
+
+import com.github.isrsal.logging.RequestWrapper;
+import com.github.isrsal.logging.ResponseWrapper;
+import com.sun.jersey.core.util.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContext;
+import org.springframework.web.filter.OncePerRequestFilter;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Created by jiazhong on 2015/4/20.
+ */
+
+public class KylinApiFilter extends OncePerRequestFilter {
+ protected static final Logger logger = LoggerFactory.getLogger(KylinApiFilter.class);
+ private static final String REQUEST_PREFIX = "Request: ";
+ private static final String RESPONSE_PREFIX = "Response: ";
+ private AtomicLong id = new AtomicLong(1L);
+
+ public KylinApiFilter() {
+ }
+
+ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException {
+ if(logger.isDebugEnabled()) {
+ long requestId = this.id.incrementAndGet();
+ request = new RequestWrapper(Long.valueOf(requestId), (HttpServletRequest)request);
+ response = new ResponseWrapper(Long.valueOf(requestId), (HttpServletResponse)response);
+ }
+
+ try {
+ filterChain.doFilter((ServletRequest)request, (ServletResponse)response);
+ } finally {
+ if(logger.isDebugEnabled()) {
+ this.logRequest((HttpServletRequest)request,(ResponseWrapper)response);
+// this.logResponse((ResponseWrapper)response);
+ }
+
+ }
+
+ }
+
+ private void logRequest(HttpServletRequest request,ResponseWrapper response) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("REQUEST: ");
+ HttpSession session = request.getSession(true);
+ SecurityContext context = (SecurityContext)session.getAttribute("SPRING_SECURITY_CONTEXT");
+
+ String requester="";
+ if(context!=null){
+ Authentication authentication= context.getAuthentication();
+ if(authentication!=null){
+ requester = authentication.getName();
+ }
+
+ }else {
+ final String authorization = request.getHeader("Authorization");
+ if (authorization != null && authorization.startsWith("Basic")) {
+ // Authorization: Basic base64credentials
+ String base64Credentials = authorization.substring("Basic".length()).trim();
+ String credentials = new String(Base64.decode(base64Credentials), Charset.forName("UTF-8"));
+ // credentials = username:password
+ String[] values = credentials.split(":", 2);
+ requester = values[0];
+ }
+ }
+ msg.append("REQUESTER="+requester);
+
+ SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
+ msg.append(";REQ_TIME=" + format.format(new Date()));
+ msg.append(";URI=").append(request.getRequestURI());
+ msg.append(";METHOD=").append(request.getMethod());
+ msg.append(";QUERY_STRING=").append(request.getQueryString());
+ if(request instanceof RequestWrapper && !this.isMultipart(request)) {
+ RequestWrapper requestWrapper = (RequestWrapper)request;
+
+ try {
+ String e = requestWrapper.getCharacterEncoding() != null?requestWrapper.getCharacterEncoding():"UTF-8";
+ msg.append(";PAYLOAD=").append(new String(requestWrapper.toByteArray(), e));
+ } catch (UnsupportedEncodingException var6) {
+ logger.warn("Failed to parse request payload", var6);
+ }
+ }
+ msg.append(";RESP_STATUS="+response.getStatus()).append(";");
+
+ logger.debug(msg.toString());
+ }
+
+ private boolean isMultipart(HttpServletRequest request) {
+ return request.getContentType() != null && request.getContentType().startsWith("multipart/form-data");
+ }
+
+ private void logResponse(ResponseWrapper response) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("RESPONSE: ");
+ msg.append("REQUEST_ID=").append(response.getId());
+
+ try {
+ msg.append("; payload=").append(new String(response.toByteArray(), response.getCharacterEncoding()));
+ } catch (UnsupportedEncodingException var4) {
+ logger.warn("Failed to parse response payload", var4);
+ }
+
+ logger.debug(msg.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a935c7ba/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index de97a7b..71e9f19 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -19,6 +19,9 @@
package org.apache.kylin.rest.service;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseRegionSizeCalculator;
@@ -556,4 +559,5 @@ public class CubeService extends BasicService {
}
+
}
[31/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java
deleted file mode 100644
index dd7ee01..0000000
--- a/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java
+++ /dev/null
@@ -1,4795 +0,0 @@
-/*
- * OVERRIDE POINT:
- * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
- * - isTrimUnusedFields(), override to false
- * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.eigenbase.sql2rel;
-
-import java.lang.reflect.Type;
-import java.math.*;
-import java.util.*;
-import java.util.logging.*;
-
-import org.eigenbase.rel.*;
-import org.eigenbase.rel.metadata.*;
-import org.eigenbase.relopt.*;
-import org.eigenbase.reltype.*;
-import org.eigenbase.rex.*;
-import org.eigenbase.sql.*;
-import org.eigenbase.sql.fun.*;
-import org.eigenbase.sql.parser.*;
-import org.eigenbase.sql.type.*;
-import org.eigenbase.sql.util.*;
-import org.eigenbase.sql.validate.*;
-import org.eigenbase.trace.*;
-import org.eigenbase.util.*;
-import org.eigenbase.util.mapping.Mappings;
-import org.eigenbase.util14.*;
-
-import net.hydromatic.linq4j.Ord;
-
-import net.hydromatic.optiq.ModifiableTable;
-import net.hydromatic.optiq.TranslatableTable;
-import net.hydromatic.optiq.prepare.Prepare;
-import net.hydromatic.optiq.prepare.RelOptTableImpl;
-import net.hydromatic.optiq.util.BitSets;
-
-import com.google.common.base.Function;
-import com.google.common.collect.*;
-
-import static org.eigenbase.sql.SqlUtil.stripAs;
-import static org.eigenbase.util.Static.RESOURCE;
-
-/**
- * Converts a SQL parse tree (consisting of {@link org.eigenbase.sql.SqlNode}
- * objects) into a relational algebra expression (consisting of
- * {@link org.eigenbase.rel.RelNode} objects).
- *
- * <p>The public entry points are: {@link #convertQuery},
- * {@link #convertExpression(SqlNode)}.
- */
-public class SqlToRelConverter {
- //~ Static fields/initializers ---------------------------------------------
-
- protected static final Logger SQL2REL_LOGGER =
- EigenbaseTrace.getSqlToRelTracer();
-
- private static final Function<SubQuery, SqlNode> FN =
- new Function<SubQuery, SqlNode>() {
- public SqlNode apply(SubQuery input) {
- return input.node;
- }
- };
-
- //~ Instance fields --------------------------------------------------------
-
- protected final SqlValidator validator;
- protected final RexBuilder rexBuilder;
- protected final Prepare.CatalogReader catalogReader;
- protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
- protected final List<RelNode> leaves = new ArrayList<RelNode>();
- private final List<SqlDynamicParam> dynamicParamSqlNodes =
- new ArrayList<SqlDynamicParam>();
- private final SqlOperatorTable opTab;
- private boolean shouldConvertTableAccess;
- protected final RelDataTypeFactory typeFactory;
- private final SqlNodeToRexConverter exprConverter;
- private boolean decorrelationEnabled;
- private boolean trimUnusedFields;
- private boolean shouldCreateValuesRel;
- private boolean isExplain;
- private int nDynamicParamsInExplain;
-
- /**
- * Fields used in name resolution for correlated subqueries.
- */
- private final Map<String, DeferredLookup> mapCorrelToDeferred =
- new HashMap<String, DeferredLookup>();
- private int nextCorrel = 0;
-
- private static final String CORREL_PREFIX = "$cor";
-
- /**
- * Stack of names of datasets requested by the <code>
- * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
- */
- private final Stack<String> datasetStack = new Stack<String>();
-
- /**
- * Mapping of non-correlated subqueries that have been converted to their
- * equivalent constants. Used to avoid re-evaluating the subquery if it's
- * already been evaluated.
- */
- private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
- new HashMap<SqlNode, RexNode>();
-
- public final RelOptTable.ViewExpander viewExpander;
-
- //~ Constructors -----------------------------------------------------------
- /**
- * Creates a converter.
- *
- * @param viewExpander Preparing statement
- * @param validator Validator
- * @param catalogReader Schema
- * @param planner Planner
- * @param rexBuilder Rex builder
- * @param convertletTable Expression converter
- */
- public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptPlanner planner,
- RexBuilder rexBuilder,
- SqlRexConvertletTable convertletTable) {
- this.viewExpander = viewExpander;
- this.opTab =
- (validator
- == null) ? SqlStdOperatorTable.instance()
- : validator.getOperatorTable();
- this.validator = validator;
- this.catalogReader = catalogReader;
- this.defaultValueFactory = new NullDefaultValueFactory();
- this.subqueryConverter = new NoOpSubqueryConverter();
- this.rexBuilder = rexBuilder;
- this.typeFactory = rexBuilder.getTypeFactory();
- RelOptQuery query = new RelOptQuery(planner);
- this.cluster = query.createCluster(typeFactory, rexBuilder);
- this.shouldConvertTableAccess = true;
- this.exprConverter =
- new SqlNodeToRexConverterImpl(convertletTable);
- decorrelationEnabled = true;
- trimUnusedFields = false;
- shouldCreateValuesRel = true;
- isExplain = false;
- nDynamicParamsInExplain = 0;
- }
-
- //~ Methods ----------------------------------------------------------------
-
- /**
- * @return the RelOptCluster in use.
- */
- public RelOptCluster getCluster() {
- return cluster;
- }
-
- /**
- * Returns the row-expression builder.
- */
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- /**
- * Returns the number of dynamic parameters encountered during translation;
- * this must only be called after {@link #convertQuery}.
- *
- * @return number of dynamic parameters
- */
- public int getDynamicParamCount() {
- return dynamicParamSqlNodes.size();
- }
-
- /**
- * Returns the type inferred for a dynamic parameter.
- *
- * @param index 0-based index of dynamic parameter
- * @return inferred type, never null
- */
- public RelDataType getDynamicParamType(int index) {
- SqlNode sqlNode = dynamicParamSqlNodes.get(index);
- if (sqlNode == null) {
- throw Util.needToImplement("dynamic param type inference");
- }
- return validator.getValidatedNodeType(sqlNode);
- }
-
- /**
- * Returns the current count of the number of dynamic parameters in an
- * EXPLAIN PLAN statement.
- *
- * @param increment if true, increment the count
- * @return the current count before the optional increment
- */
- public int getDynamicParamCountInExplain(boolean increment) {
- int retVal = nDynamicParamsInExplain;
- if (increment) {
- ++nDynamicParamsInExplain;
- }
- return retVal;
- }
-
- /**
- * @return mapping of non-correlated subqueries that have been converted to
- * the constants that they evaluate to
- */
- public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
- return mapConvertedNonCorrSubqs;
- }
-
- /**
- * Adds to the current map of non-correlated converted subqueries the
- * elements from another map that contains non-correlated subqueries that
- * have been converted by another SqlToRelConverter.
- *
- * @param alreadyConvertedNonCorrSubqs the other map
- */
- public void addConvertedNonCorrSubqs(
- Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
- mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
- }
-
- /**
- * Set a new DefaultValueFactory. To have any effect, this must be called
- * before any convert method.
- *
- * @param factory new DefaultValueFactory
- */
- public void setDefaultValueFactory(DefaultValueFactory factory) {
- defaultValueFactory = factory;
- }
-
- /**
- * Sets a new SubqueryConverter. To have any effect, this must be called
- * before any convert method.
- *
- * @param converter new SubqueryConverter
- */
- public void setSubqueryConverter(SubqueryConverter converter) {
- subqueryConverter = converter;
- }
-
- /**
- * Indicates that the current statement is part of an EXPLAIN PLAN statement
- *
- * @param nDynamicParams number of dynamic parameters in the statement
- */
- public void setIsExplain(int nDynamicParams) {
- isExplain = true;
- nDynamicParamsInExplain = nDynamicParams;
- }
-
- /**
- * Controls whether table access references are converted to physical rels
- * immediately. The optimizer doesn't like leaf rels to have
- * {@link Convention#NONE}. However, if we are doing further conversion
- * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
- * defer conversion. To have any effect, this must be called before any
- * convert method.
- *
- * @param enabled true for immediate conversion (the default); false to
- * generate logical TableAccessRel instances
- */
- public void enableTableAccessConversion(boolean enabled) {
- shouldConvertTableAccess = enabled;
- }
-
- /**
- * Controls whether instances of {@link ValuesRel} are generated. These may
- * not be supported by all physical implementations. To have any effect,
- * this must be called before any convert method.
- *
- * @param enabled true to allow ValuesRel to be generated (the default);
- * false to force substitution of ProjectRel+OneRowRel instead
- */
- public void enableValuesRelCreation(boolean enabled) {
- shouldCreateValuesRel = enabled;
- }
-
- private void checkConvertedType(SqlNode query, RelNode result) {
- if (!query.isA(SqlKind.DML)) {
- // Verify that conversion from SQL to relational algebra did
- // not perturb any type information. (We can't do this if the
- // SQL statement is something like an INSERT which has no
- // validator type information associated with its result,
- // hence the namespace check above.)
- RelDataType convertedRowType = result.getRowType();
- if (!checkConvertedRowType(query, convertedRowType)) {
- RelDataType validatedRowType =
- validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
- throw Util.newInternal(
- "Conversion to relational algebra failed to preserve "
- + "datatypes:\n"
- + "validated type:\n"
- + validatedRowType.getFullTypeString()
- + "\nconverted type:\n"
- + convertedRowType.getFullTypeString()
- + "\nrel:\n"
- + RelOptUtil.toString(result));
- }
- }
- }
-
- public RelNode flattenTypes(
- RelNode rootRel,
- boolean restructure) {
- RelStructuredTypeFlattener typeFlattener =
- new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
- return typeFlattener.rewrite(rootRel, restructure);
- }
-
- /**
- * If subquery is correlated and decorrelation is enabled, performs
- * decorrelation.
- *
- * @param query Query
- * @param rootRel Root relational expression
- * @return New root relational expression after decorrelation
- */
- public RelNode decorrelate(SqlNode query, RelNode rootRel) {
- if (!enableDecorrelation()) {
- return rootRel;
- }
- final RelNode result = decorrelateQuery(rootRel);
- if (result != rootRel) {
- checkConvertedType(query, result);
- }
- return result;
- }
-
- /**
- * Walks over a tree of relational expressions, replacing each
- * {@link RelNode} with a 'slimmed down' relational expression that projects
- * only the fields required by its consumer.
- *
- * <p>This may make things easier for the optimizer, by removing crud that
- * would expand the search space, but is difficult for the optimizer itself
- * to do it, because optimizer rules must preserve the number and type of
- * fields. Hence, this transform that operates on the entire tree, similar
- * to the {@link RelStructuredTypeFlattener type-flattening transform}.
- *
- * <p>Currently this functionality is disabled in farrago/luciddb; the
- * default implementation of this method does nothing.
- *
- * @param rootRel Relational expression that is at the root of the tree
- * @return Trimmed relational expression
- */
- public RelNode trimUnusedFields(RelNode rootRel) {
- // Trim fields that are not used by their consumer.
- if (isTrimUnusedFields()) {
- final RelFieldTrimmer trimmer = newFieldTrimmer();
- rootRel = trimmer.trim(rootRel);
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after trimming unused fields",
- rootRel,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
- }
- return rootRel;
- }
-
- /**
- * Creates a RelFieldTrimmer.
- *
- * @return Field trimmer
- */
- protected RelFieldTrimmer newFieldTrimmer() {
- return new RelFieldTrimmer(validator);
- }
-
- /**
- * Converts an unvalidated query's parse tree into a relational expression.
- *
- * @param query Query to convert
- * @param needsValidation Whether to validate the query before converting;
- * <code>false</code> if the query has already been
- * validated.
- * @param top Whether the query is top-level, say if its result
- * will become a JDBC result set; <code>false</code> if
- * the query will be part of a view.
- */
- public RelNode convertQuery(
- SqlNode query,
- final boolean needsValidation,
- final boolean top) {
- if (needsValidation) {
- query = validator.validate(query);
- }
-
- RelNode result = convertQueryRecursive(query, top, null);
- checkConvertedType(query, result);
-
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after converting SqlNode to RelNode",
- result,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
-
- return result;
- }
-
- protected boolean checkConvertedRowType(
- SqlNode query,
- RelDataType convertedRowType) {
- RelDataType validatedRowType = validator.getValidatedNodeType(query);
- validatedRowType = uniquifyFields(validatedRowType);
-
- return RelOptUtil.equal(
- "validated row type",
- validatedRowType,
- "converted row type",
- convertedRowType,
- false);
- }
-
- protected RelDataType uniquifyFields(RelDataType rowType) {
- return validator.getTypeFactory().createStructType(
- RelOptUtil.getFieldTypeList(rowType),
- SqlValidatorUtil.uniquify(rowType.getFieldNames()));
- }
-
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertSelect(SqlSelect select) {
- final SqlValidatorScope selectScope = validator.getWhereScope(select);
- final Blackboard bb = createBlackboard(selectScope, null);
- convertSelectImpl(bb, select);
- return bb.root;
- }
-
- /**
- * Factory method for creating translation workspace.
- */
- protected Blackboard createBlackboard(
- SqlValidatorScope scope,
- Map<String, RexNode> nameToNodeMap) {
- return new Blackboard(scope, nameToNodeMap);
- }
-
- /**
- * Implementation of {@link #convertSelect(SqlSelect)}; derived class may
- * override.
- */
- protected void convertSelectImpl(
- final Blackboard bb,
- SqlSelect select) {
- convertFrom(
- bb,
- select.getFrom());
- convertWhere(
- bb,
- select.getWhere());
-
- List<SqlNode> orderExprList = new ArrayList<SqlNode>();
- List<RelFieldCollation> collationList =
- new ArrayList<RelFieldCollation>();
- gatherOrderExprs(
- bb,
- select,
- select.getOrderList(),
- orderExprList,
- collationList);
- final RelCollation collation =
- cluster.traitSetOf().canonize(RelCollationImpl.of(collationList));
-
- if (validator.isAggregate(select)) {
- convertAgg(
- bb,
- select,
- orderExprList);
- } else {
- convertSelectList(
- bb,
- select,
- orderExprList);
- }
-
- if (select.isDistinct()) {
- distinctify(bb, true);
- }
- convertOrder(
- select, bb, collation, orderExprList, select.getOffset(),
- select.getFetch());
- bb.setRoot(bb.root, true);
- }
-
- /**
- * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
- * a relational expression to make the results unique.
- *
- * <p>If the SELECT clause contains duplicate expressions, adds {@link
- * ProjectRel}s so that we are grouping on the minimal set of keys. The
- * performance gain isn't huge, but it is difficult to detect these
- * duplicate expressions later.
- *
- * @param bb Blackboard
- * @param checkForDupExprs Check for duplicate expressions
- */
- private void distinctify(
- Blackboard bb,
- boolean checkForDupExprs) {
- // Look for duplicate expressions in the project.
- // Say we have 'select x, y, x, z'.
- // Then dups will be {[2, 0]}
- // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
- RelNode rel = bb.root;
- if (checkForDupExprs && (rel instanceof ProjectRel)) {
- ProjectRel project = (ProjectRel) rel;
- final List<RexNode> projectExprs = project.getProjects();
- List<Integer> origins = new ArrayList<Integer>();
- int dupCount = 0;
- for (int i = 0; i < projectExprs.size(); i++) {
- int x = findExpr(projectExprs.get(i), projectExprs, i);
- if (x >= 0) {
- origins.add(x);
- ++dupCount;
- } else {
- origins.add(i);
- }
- }
- if (dupCount == 0) {
- distinctify(bb, false);
- return;
- }
-
- final Map<Integer, Integer> squished = Maps.newHashMap();
- final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
- final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- if (origins.get(i) == i) {
- squished.put(i, newProjects.size());
- newProjects.add(RexInputRef.of2(i, fields));
- }
- }
- rel =
- new ProjectRel(
- cluster,
- rel,
- Pair.left(newProjects),
- Pair.right(newProjects),
- ProjectRel.Flags.BOXED);
-
- bb.root = rel;
- distinctify(bb, false);
- rel = bb.root;
-
- // Create the expressions to reverse the mapping.
- // Project($0, $1, $0, $2).
- final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- final int origin = origins.get(i);
- RelDataTypeField field = fields.get(i);
- undoProjects.add(
- Pair.of(
- (RexNode) new RexInputRef(
- squished.get(origin), field.getType()),
- field.getName()));
- }
-
- rel =
- new ProjectRel(
- cluster,
- rel,
- Pair.left(undoProjects),
- Pair.right(undoProjects),
- ProjectRel.Flags.BOXED);
-
- bb.setRoot(
- rel,
- false);
-
- return;
- }
-
- // Usual case: all of the expressions in the SELECT clause are
- // different.
- rel =
- createAggregate(
- bb,
- BitSets.range(rel.getRowType().getFieldCount()),
- ImmutableList.<AggregateCall>of());
-
- bb.setRoot(
- rel,
- false);
- }
-
- private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
- for (int i = 0; i < count; i++) {
- RexNode expr = exprs.get(i);
- if (expr.toString().equals(seek.toString())) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Converts a query's ORDER BY clause, if any.
- *
- * @param select Query
- * @param bb Blackboard
- * @param collation Collation list
- * @param orderExprList Method populates this list with orderBy expressions
- * not present in selectList
- * @param offset Expression for number of rows to discard before
- * returning first row
- * @param fetch Expression for number of rows to fetch
- */
- protected void convertOrder(
- SqlSelect select,
- Blackboard bb,
- RelCollation collation,
- List<SqlNode> orderExprList,
- SqlNode offset,
- SqlNode fetch) {
- if (select.getOrderList() == null) {
- assert collation.getFieldCollations().isEmpty();
- if (offset == null && fetch == null) {
- return;
- }
- }
-
- // Create a sorter using the previously constructed collations.
- bb.setRoot(
- new SortRel(
- cluster,
- cluster.traitSetOf(Convention.NONE, collation),
- bb.root,
- collation,
- offset == null ? null : convertExpression(offset),
- fetch == null ? null : convertExpression(fetch)),
- false);
-
- // If extra expressions were added to the project list for sorting,
- // add another project to remove them.
- if (orderExprList.size() > 0) {
- List<RexNode> exprs = new ArrayList<RexNode>();
- final RelDataType rowType = bb.root.getRowType();
- final int fieldCount =
- rowType.getFieldCount() - orderExprList.size();
- for (int i = 0; i < fieldCount; i++) {
- exprs.add(rexBuilder.makeInputRef(bb.root, i));
- }
- bb.setRoot(
- new ProjectRel(
- cluster,
- cluster.traitSetOf(RelCollationImpl.PRESERVE),
- bb.root,
- exprs,
- cluster.getTypeFactory().createStructType(
- rowType.getFieldList().subList(0, fieldCount)),
- ProjectRelBase.Flags.BOXED),
- false);
- }
- }
-
- /**
- * Returns whether a given node contains a {@link SqlInOperator}.
- *
- * @param node a RexNode tree
- */
- private static boolean containsInOperator(
- SqlNode node) {
- try {
- SqlVisitor<Void> visitor =
- new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
- }
- };
- node.accept(visitor);
- return false;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return true;
- }
- }
-
- /**
- * Push down all the NOT logical operators into any IN/NOT IN operators.
- *
- * @param sqlNode the root node from which to look for NOT operators
- * @return the transformed SqlNode representation with NOT pushed down.
- */
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
- if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
- SqlCall sqlCall = (SqlCall) sqlNode;
- if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
- || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
- SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
- for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
- }
- return sqlNode;
- } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode childNode = sqlCall.operand(0);
- assert childNode instanceof SqlCall;
- SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
- if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
- SqlNode[] andOperands = childSqlCall.getOperands();
- SqlNode[] orOperands = new SqlNode[andOperands.length];
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- andOperands[i]);
- }
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
- }
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
- orOperands[0], orOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
- SqlNode[] orOperands = childSqlCall.getOperands();
- SqlNode[] andOperands = new SqlNode[orOperands.length];
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- orOperands[i]);
- }
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
- }
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- andOperands[0], andOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode[] notOperands = childSqlCall.getOperands();
- assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
- } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
- SqlNode[] inOperands = childSqlCall.getOperands();
- SqlInOperator inOp =
- (SqlInOperator) childSqlCall.getOperator();
- if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- } else {
- return SqlStdOperatorTable.NOT_IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- }
- } else {
- // childSqlCall is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // sqlNode is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // tree rooted at sqlNode does not contain inOperator
- return sqlNode;
- }
- }
-
- /**
- * Converts a WHERE clause.
- *
- * @param bb Blackboard
- * @param where WHERE clause, may be null
- */
- private void convertWhere(
- final Blackboard bb,
- final SqlNode where) {
- if (where == null) {
- return;
- }
- SqlNode newWhere = pushDownNotForIn(where);
- replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- final RexNode convertedWhere = bb.convertExpression(newWhere);
-
- // only allocate filter if the condition is not TRUE
- if (!convertedWhere.isAlwaysTrue()) {
- bb.setRoot(
- RelOptUtil.createFilter(bb.root, convertedWhere),
- false);
- }
- }
-
- private void replaceSubqueries(
- final Blackboard bb,
- final SqlNode expr,
- RelOptUtil.Logic logic) {
- findSubqueries(bb, expr, logic, false);
- for (SubQuery node : bb.subqueryList) {
- substituteSubquery(bb, node);
- }
- }
-
- private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
- final RexNode expr = subQuery.expr;
- if (expr != null) {
- // Already done.
- return;
- }
-
- final SqlBasicCall call;
- final RelNode rel;
- final SqlNode query;
- final Pair<RelNode, Boolean> converted;
- switch (subQuery.node.getKind()) {
- case CURSOR:
- convertCursor(bb, subQuery);
- return;
-
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
- subQuery.expr = bb.register(rel, JoinRelType.INNER);
- return;
-
- case IN:
- call = (SqlBasicCall) subQuery.node;
- final SqlNode[] operands = call.getOperands();
-
- SqlNode leftKeyNode = operands[0];
- query = operands[1];
-
- final List<RexNode> leftKeys;
- switch (leftKeyNode.getKind()) {
- case ROW:
- leftKeys = Lists.newArrayList();
- for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
- leftKeys.add(bb.convertExpression(sqlExpr));
- }
- break;
- default:
- leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
- }
-
- final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
- if (query instanceof SqlNodeList) {
- SqlNodeList valueList = (SqlNodeList) query;
- if (!containsNullLiteral(valueList)
- && valueList.size() < getInSubqueryThreshold()) {
- // We're under the threshold, so convert to OR.
- subQuery.expr =
- convertInToOr(
- bb,
- leftKeys,
- valueList,
- isNotIn);
- return;
- }
-
- // Otherwise, let convertExists translate
- // values list into an inline table for the
- // reference to Q below.
- }
-
- // Project out the search columns from the left side
-
- // Q1:
- // "select from emp where emp.deptno in (select col1 from T)"
- //
- // is converted to
- //
- // "select from
- // emp inner join (select distinct col1 from T)) q
- // on emp.deptno = q.col1
- //
- // Q2:
- // "select from emp where emp.deptno not in (Q)"
- //
- // is converted to
- //
- // "select from
- // emp left outer join (select distinct col1, TRUE from T) q
- // on emp.deptno = q.col1
- // where emp.deptno <> null
- // and q.indicator <> TRUE"
- //
- final boolean outerJoin = bb.subqueryNeedsOuterJoin
- || isNotIn
- || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
- converted =
- convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic,
- outerJoin);
- if (converted.right) {
- // Generate
- // emp CROSS JOIN (SELECT COUNT(*) AS c,
- // COUNT(deptno) AS ck FROM dept)
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RelNode seek = converted.left.getInput(0); // fragile
- final int keyCount = leftKeys.size();
- final List<Integer> args = ImmutableIntList.range(0, keyCount);
- AggregateRel aggregate =
- new AggregateRel(cluster, seek, BitSets.of(),
- ImmutableList.of(
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- ImmutableList.<Integer>of(), longType, null),
- new AggregateCall(SqlStdOperatorTable.COUNT, false,
- args, longType, null)));
- JoinRel join =
- new JoinRel(cluster, bb.root, aggregate,
- rexBuilder.makeLiteral(true), JoinRelType.INNER,
- ImmutableSet.<String>of());
- bb.setRoot(join, false);
- }
- RexNode rex =
- bb.register(converted.left,
- outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
-
- subQuery.expr = translateIn(subQuery, bb.root, rex);
- if (isNotIn) {
- subQuery.expr =
- rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
- }
- return;
-
- case EXISTS:
- // "select from emp where exists (select a from T)"
- //
- // is converted to the following if the subquery is correlated:
- //
- // "select from emp left outer join (select AGG_TRUE() as indicator
- // from T group by corr_var) q where q.indicator is true"
- //
- // If there is no correlation, the expression is replaced with a
- // boolean indicating whether the subquery returned 0 or >= 1 row.
- call = (SqlBasicCall) subQuery.node;
- query = call.getOperands()[0];
- converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS,
- subQuery.logic, true);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
- return;
- }
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- case SCALAR_QUERY:
- // Convert the subquery. If it's non-correlated, convert it
- // to a constant expression.
- call = (SqlBasicCall) subQuery.node;
- query = call.getOperands()[0];
- converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
- return;
- }
- rel = convertToSingleValueSubq(query, converted.left);
- subQuery.expr = bb.register(rel, JoinRelType.LEFT);
- return;
-
- case SELECT:
- // This is used when converting multiset queries:
- //
- // select * from unnest(select multiset[deptno] from emps);
- //
- converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true);
- assert !converted.right;
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- default:
- throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
- }
- }
-
- private RexNode translateIn(SubQuery subQuery, RelNode root,
- final RexNode rex) {
- switch (subQuery.logic) {
- case TRUE:
- return rexBuilder.makeLiteral(true);
-
- case UNKNOWN_AS_FALSE:
- assert rex instanceof RexRangeRef;
- final int fieldCount = rex.getType().getFieldCount();
- RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
- rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
-
- // Then append the IS NOT NULL(leftKeysForIn).
- //
- // RexRangeRef contains the following fields:
- // leftKeysForIn,
- // rightKeysForIn (the original subquery select list),
- // nullIndicator
- //
- // The first two lists contain the same number of fields.
- final int k = (fieldCount - 1) / 2;
- for (int i = 0; i < k; i++) {
- rexNode =
- rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- rexNode,
- rexBuilder.makeCall(
- SqlStdOperatorTable.IS_NOT_NULL,
- rexBuilder.makeFieldAccess(rex, i)));
- }
- return rexNode;
-
- case TRUE_FALSE_UNKNOWN:
- case UNKNOWN_AS_TRUE:
- // select e.deptno,
- // case
- // when ct.c = 0 then false
- // when dt.i is not null then true
- // when e.deptno is null then null
- // when ct.ck < ct.c then null
- // else false
- // end
- // from e
- // cross join (select count(*) as c, count(deptno) as ck from v) as ct
- // left join (select distinct deptno, true as i from v) as dt
- // on e.deptno = dt.deptno
- final JoinRelBase join = (JoinRelBase) root;
- final ProjectRelBase left = (ProjectRelBase) join.getLeft();
- final RelNode leftLeft = ((JoinRelBase) left.getInput(0)).getLeft();
- final int leftLeftCount = leftLeft.getRowType().getFieldCount();
- final RelDataType nullableBooleanType =
- typeFactory.createTypeWithNullability(
- typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
- final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
- final RexNode iRef =
- rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
-
- final RexLiteral zero =
- rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
- final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
- final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
- final RexNode unknownLiteral =
- rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
-
- final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
- falseLiteral,
- rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
- trueLiteral);
- final JoinInfo joinInfo = join.analyzeCondition();
- for (int leftKey : joinInfo.leftKeys) {
- final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
- unknownLiteral);
- }
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
- unknownLiteral,
- falseLiteral);
-
- return rexBuilder.makeCall(
- nullableBooleanType,
- SqlStdOperatorTable.CASE,
- args.build());
-
- default:
- throw new AssertionError(subQuery.logic);
- }
- }
-
- private static boolean containsNullLiteral(SqlNodeList valueList) {
- for (SqlNode node : valueList.getList()) {
- if (node instanceof SqlLiteral) {
- SqlLiteral lit = (SqlLiteral) node;
- if (lit.getValue() == null) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Determines if a subquery is non-correlated and if so, converts it to a
- * constant.
- *
- * @param subQuery the call that references the subquery
- * @param bb blackboard used to convert the subquery
- * @param converted RelNode tree corresponding to the subquery
- * @param isExists true if the subquery is part of an EXISTS expression
- * @return if the subquery can be converted to a constant
- */
- private boolean convertNonCorrelatedSubQuery(
- SubQuery subQuery,
- Blackboard bb,
- RelNode converted,
- boolean isExists) {
- SqlCall call = (SqlBasicCall) subQuery.node;
- if (subqueryConverter.canConvertSubquery()
- && isSubqNonCorrelated(converted, bb)) {
- // First check if the subquery has already been converted
- // because it's a nested subquery. If so, don't re-evaluate
- // it again.
- RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
- if (constExpr == null) {
- constExpr =
- subqueryConverter.convertSubquery(
- call,
- this,
- isExists,
- isExplain);
- }
- if (constExpr != null) {
- subQuery.expr = constExpr;
- mapConvertedNonCorrSubqs.put(call, constExpr);
- return true;
- }
- }
- return false;
- }
-
- /**
- * Converts the RelNode tree for a select statement to a select that
- * produces a single value.
- *
- * @param query the query
- * @param plan the original RelNode tree corresponding to the statement
- * @return the converted RelNode tree
- */
- public RelNode convertToSingleValueSubq(
- SqlNode query,
- RelNode plan) {
- // Check whether query is guaranteed to produce a single value.
- if (query instanceof SqlSelect) {
- SqlSelect select = (SqlSelect) query;
- SqlNodeList selectList = select.getSelectList();
- SqlNodeList groupList = select.getGroup();
-
- if ((selectList.size() == 1)
- && ((groupList == null) || (groupList.size() == 0))) {
- SqlNode selectExpr = selectList.get(0);
- if (selectExpr instanceof SqlCall) {
- SqlCall selectExprCall = (SqlCall) selectExpr;
- if (selectExprCall.getOperator()
- instanceof SqlAggFunction) {
- return plan;
- }
- }
- }
- }
-
- // If not, project SingleValueAgg
- return RelOptUtil.createSingleValueAggRel(
- cluster,
- plan);
- }
-
- /**
- * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
- *
- * @param leftKeys LHS
- * @param valuesList RHS
- * @param isNotIn is this a NOT IN operator
- * @return converted expression
- */
- private RexNode convertInToOr(
- final Blackboard bb,
- final List<RexNode> leftKeys,
- SqlNodeList valuesList,
- boolean isNotIn) {
- List<RexNode> comparisons = new ArrayList<RexNode>();
- for (SqlNode rightVals : valuesList) {
- RexNode rexComparison;
- if (leftKeys.size() == 1) {
- rexComparison =
- rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- leftKeys.get(0),
- bb.convertExpression(rightVals));
- } else {
- assert rightVals instanceof SqlCall;
- final SqlBasicCall call = (SqlBasicCall) rightVals;
- assert (call.getOperator() instanceof SqlRowOperator)
- && call.getOperands().length == leftKeys.size();
- rexComparison =
- RexUtil.composeConjunction(
- rexBuilder,
- Iterables.transform(
- Pair.zip(leftKeys, call.getOperandList()),
- new Function<Pair<RexNode, SqlNode>, RexNode>() {
- public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- pair.left, bb.convertExpression(pair.right));
- }
- }),
- false);
- }
- comparisons.add(rexComparison);
- }
-
- RexNode result =
- RexUtil.composeDisjunction(rexBuilder, comparisons, true);
- assert result != null;
-
- if (isNotIn) {
- result =
- rexBuilder.makeCall(
- SqlStdOperatorTable.NOT,
- result);
- }
-
- return result;
- }
-
- /**
- * Gets the list size threshold under which {@link #convertInToOr} is used.
- * Lists of this size or greater will instead be converted to use a join
- * against an inline table ({@link ValuesRel}) rather than a predicate. A
- * threshold of 0 forces usage of an inline table in all cases; a threshold
- * of Integer.MAX_VALUE forces usage of OR in all cases
- *
- * @return threshold, default 20
- */
- protected int getInSubqueryThreshold() {
- // OVERRIDE POINT
- return Integer.MAX_VALUE; // was 20
- }
-
- /**
- * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
- * produces an indicator variable, and the result is a relational expression
- * which outer joins that indicator to the original query. After performing
- * the outer join, the condition will be TRUE if the EXISTS condition holds,
- * NULL otherwise.
- *
- * @param seek A query, for example 'select * from emp' or
- * 'values (1,2,3)' or '('Foo', 34)'.
- * @param subqueryType Whether sub-query is IN, EXISTS or scalar
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept an
- * approximation (say representing UNKNOWN as FALSE)
- * @param needsOuterJoin Whether an outer join is needed
- * @return join expression
- * @pre extraExpr == null || extraName != null
- */
- private Pair<RelNode, Boolean> convertExists(
- SqlNode seek,
- RelOptUtil.SubqueryType subqueryType,
- RelOptUtil.Logic logic,
- boolean needsOuterJoin) {
- final SqlValidatorScope seekScope =
- (seek instanceof SqlSelect)
- ? validator.getSelectScope((SqlSelect) seek)
- : null;
- final Blackboard seekBb = createBlackboard(seekScope, null);
- RelNode seekRel = convertQueryOrInList(seekBb, seek);
-
- return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic,
- needsOuterJoin);
- }
-
- private RelNode convertQueryOrInList(
- Blackboard bb,
- SqlNode seek) {
- // NOTE: Once we start accepting single-row queries as row constructors,
- // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
- // Z)). The SQL standard resolves the ambiguity by saying that a lone
- // select should be interpreted as a table expression, not a row
- // expression. The semantic difference is that a table expression can
- // return multiple rows.
- if (seek instanceof SqlNodeList) {
- return convertRowValues(
- bb,
- seek,
- ((SqlNodeList) seek).getList(),
- false,
- null);
- } else {
- return convertQueryRecursive(seek, false, null);
- }
- }
-
- private RelNode convertRowValues(
- Blackboard bb,
- SqlNode rowList,
- Collection<SqlNode> rows,
- boolean allowLiteralsOnly,
- RelDataType targetRowType) {
- // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
- // literals into a single ValuesRel; this gives the optimizer a smaller
- // input tree. For everything else (computed expressions, row
- // subqueries), we union each row in as a projection on top of a
- // OneRowRel.
-
- final List<List<RexLiteral>> tupleList =
- new ArrayList<List<RexLiteral>>();
- final RelDataType rowType;
- if (targetRowType != null) {
- rowType = targetRowType;
- } else {
- rowType =
- SqlTypeUtil.promoteToRowType(
- typeFactory,
- validator.getValidatedNodeType(rowList),
- null);
- }
-
- List<RelNode> unionInputs = new ArrayList<RelNode>();
- for (SqlNode node : rows) {
- SqlBasicCall call;
- if (isRowConstructor(node)) {
- call = (SqlBasicCall) node;
- List<RexLiteral> tuple = new ArrayList<RexLiteral>();
- for (SqlNode operand : call.operands) {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- operand,
- bb,
- rowType,
- tuple.size());
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- if ((rexLiteral == null) || !shouldCreateValuesRel) {
- // fallback to convertRowConstructor
- tuple = null;
- break;
- }
- tuple.add(rexLiteral);
- }
- if (tuple != null) {
- tupleList.add(tuple);
- continue;
- }
- } else {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- node,
- bb,
- rowType,
- 0);
- if ((rexLiteral != null) && shouldCreateValuesRel) {
- tupleList.add(
- Collections.singletonList(rexLiteral));
- continue;
- } else {
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- }
-
- // convert "1" to "row(1)"
- call =
- (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
- SqlParserPos.ZERO,
- node);
- }
- unionInputs.add(convertRowConstructor(bb, call));
- }
- ValuesRel valuesRel =
- new ValuesRel(
- cluster,
- rowType,
- tupleList);
- RelNode resultRel;
- if (unionInputs.isEmpty()) {
- resultRel = valuesRel;
- } else {
- if (!tupleList.isEmpty()) {
- unionInputs.add(valuesRel);
- }
- UnionRel unionRel =
- new UnionRel(
- cluster,
- unionInputs,
- true);
- resultRel = unionRel;
- }
- leaves.add(resultRel);
- return resultRel;
- }
-
- private RexLiteral convertLiteralInValuesList(
- SqlNode sqlNode,
- Blackboard bb,
- RelDataType rowType,
- int iField) {
- if (!(sqlNode instanceof SqlLiteral)) {
- return null;
- }
- RelDataTypeField field = rowType.getFieldList().get(iField);
- RelDataType type = field.getType();
- if (type.isStruct()) {
- // null literals for weird stuff like UDT's need
- // special handling during type flattening, so
- // don't use ValuesRel for those
- return null;
- }
-
- RexNode literalExpr =
- exprConverter.convertLiteral(
- bb,
- (SqlLiteral) sqlNode);
-
- if (!(literalExpr instanceof RexLiteral)) {
- assert literalExpr.isA(SqlKind.CAST);
- RexNode child = ((RexCall) literalExpr).getOperands().get(0);
- assert RexLiteral.isNullLiteral(child);
-
- // NOTE jvs 22-Nov-2006: we preserve type info
- // in ValuesRel digest, so it's OK to lose it here
- return (RexLiteral) child;
- }
-
- RexLiteral literal = (RexLiteral) literalExpr;
-
- Comparable value = literal.getValue();
-
- if (SqlTypeUtil.isExactNumeric(type)) {
- BigDecimal roundedValue =
- NumberUtil.rescaleBigDecimal(
- (BigDecimal) value,
- type.getScale());
- return rexBuilder.makeExactLiteral(
- roundedValue,
- type);
- }
-
- if ((value instanceof NlsString)
- && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
- // pad fixed character type
- NlsString unpadded = (NlsString) value;
- return rexBuilder.makeCharLiteral(
- new NlsString(
- Util.rpad(unpadded.getValue(), type.getPrecision()),
- unpadded.getCharsetName(),
- unpadded.getCollation()));
- }
- return literal;
- }
-
- private boolean isRowConstructor(SqlNode node) {
- if (!(node.getKind() == SqlKind.ROW)) {
- return false;
- }
- SqlCall call = (SqlCall) node;
- return call.getOperator().getName().equalsIgnoreCase("row");
- }
-
- /**
- * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
- * inside SQL parse tree. Does not traverse inside queries.
- *
- * @param bb blackboard
- * @param node the SQL parse tree
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept
- * an approximation (say representing UNKNOWN as FALSE)
- * @param registerOnlyScalarSubqueries if set to true and the parse tree
- * corresponds to a variation of a select
- * node, only register it if it's a scalar
- * subquery
- */
- private void findSubqueries(
- Blackboard bb,
- SqlNode node,
- RelOptUtil.Logic logic,
- boolean registerOnlyScalarSubqueries) {
- final SqlKind kind = node.getKind();
- switch (kind) {
- case EXISTS:
- case SELECT:
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case CURSOR:
- case SCALAR_QUERY:
- if (!registerOnlyScalarSubqueries
- || (kind == SqlKind.SCALAR_QUERY)) {
- bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
- }
- return;
- case IN:
- if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
- logic = logic.negate();
- }
- break;
- case NOT:
- logic = logic.negate();
- break;
- }
- if (node instanceof SqlCall) {
- if (kind == SqlKind.OR
- || kind == SqlKind.NOT) {
- // It's always correct to outer join subquery with
- // containing query; however, when predicates involve Or
- // or NOT, outer join might be necessary.
- bb.subqueryNeedsOuterJoin = true;
- }
- for (SqlNode operand : ((SqlCall) node).getOperandList()) {
- if (operand != null) {
- // In the case of an IN expression, locate scalar
- // subqueries so we can convert them to constants
- findSubqueries(
- bb,
- operand,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
- } else if (node instanceof SqlNodeList) {
- for (SqlNode child : (SqlNodeList) node) {
- findSubqueries(
- bb,
- child,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
-
- // Now that we've located any scalar subqueries inside the IN
- // expression, register the IN expression itself. We need to
- // register the scalar subqueries first so they can be converted
- // before the IN expression is converted.
- if (kind == SqlKind.IN) {
- if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
- && !validator.getValidatedNodeType(node).isNullable()) {
- logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
- }
- // TODO: This conversion is only valid in the WHERE clause
- if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE
- && !bb.subqueryNeedsOuterJoin) {
- logic = RelOptUtil.Logic.TRUE;
- }
- bb.registerSubquery(node, logic);
- }
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format.
- *
- * @param node Expression to translate
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node) {
- Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
- Blackboard bb =
- createBlackboard(
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
- null);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format,
- * mapping identifier references to predefined expressions.
- *
- * @param node Expression to translate
- * @param nameToNodeMap map from String to {@link RexNode}; when an
- * {@link SqlIdentifier} is encountered, it is used as a
- * key and translated to the corresponding value from
- * this map
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node,
- Map<String, RexNode> nameToNodeMap) {
- final Map<String, RelDataType> nameToTypeMap =
- new HashMap<String, RelDataType>();
- for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
- nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
- }
- Blackboard bb =
- createBlackboard(
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap),
- nameToNodeMap);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts a non-standard expression.
- *
- * <p>This method is an extension-point that derived classes can override. If
- * this method returns a null result, the normal expression translation
- * process will proceed. The default implementation always returns null.
- *
- * @param node Expression
- * @param bb Blackboard
- * @return null to proceed with the usual expression translation process
- */
- protected RexNode convertExtendedExpression(
- SqlNode node,
- Blackboard bb) {
- return null;
- }
-
- private RexNode convertOver(Blackboard bb, SqlNode node) {
- SqlCall call = (SqlCall) node;
- SqlCall aggCall = call.operand(0);
- SqlNode windowOrRef = call.operand(1);
- final SqlWindow window =
- validator.resolveWindow(windowOrRef, bb.scope, true);
- final SqlNodeList partitionList = window.getPartitionList();
- final ImmutableList.Builder<RexNode> partitionKeys =
- ImmutableList.builder();
- for (SqlNode partition : partitionList) {
- partitionKeys.add(bb.convertExpression(partition));
- }
- RexNode lowerBound = bb.convertExpression(window.getLowerBound());
- RexNode upperBound = bb.convertExpression(window.getUpperBound());
- SqlNodeList orderList = window.getOrderList();
- if ((orderList.size() == 0) && !window.isRows()) {
- // A logical range requires an ORDER BY clause. Use the implicit
- // ordering of this relation. There must be one, otherwise it would
- // have failed validation.
- orderList = bb.scope.getOrderList();
- if (orderList == null) {
- throw new AssertionError(
- "Relation should have sort key for implicit ORDER BY");
- }
- }
- final ImmutableList.Builder<RexFieldCollation> orderKeys =
- ImmutableList.builder();
- final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
- for (SqlNode order : orderList) {
- flags.clear();
- RexNode e = bb.convertSortExpression(order, flags);
- orderKeys.add(new RexFieldCollation(e, flags));
- }
- try {
- Util.permAssert(bb.window == null, "already in window agg mode");
- bb.window = window;
- RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
- rexAgg =
- rexBuilder.ensureType(
- validator.getValidatedNodeType(call), rexAgg, false);
-
- // Walk over the tree and apply 'over' to all agg functions. This is
- // necessary because the returned expression is not necessarily a call
- // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
- final RexShuttle visitor =
- new HistogramShuttle(
- partitionKeys.build(), orderKeys.build(),
- RexWindowBound.create(window.getLowerBound(), lowerBound),
- RexWindowBound.create(window.getUpperBound(), upperBound),
- window);
- return rexAgg.accept(visitor);
- } finally {
- bb.window = null;
- }
- }
-
- /**
- * Converts a FROM clause into a relational expression.
- *
- * @param bb Scope within which to resolve identifiers
- * @param from FROM clause of a query. Examples include:
- *
- * <ul>
- * <li>a single table ("SALES.EMP"),
- * <li>an aliased table ("EMP AS E"),
- * <li>a list of tables ("EMP, DEPT"),
- * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
- * DEPT.DEPTNO"),
- * <li>a VALUES clause ("VALUES ('Fred', 20)"),
- * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
- * <li>or any combination of the above.
- * </ul>
- */
- protected void convertFrom(
- Blackboard bb,
- SqlNode from) {
- SqlCall call;
- final SqlNode[] operands;
- switch (from.getKind()) {
- case AS:
- operands = ((SqlBasicCall) from).getOperands();
- convertFrom(bb, operands[0]);
- return;
-
- case WITH_ITEM:
- convertFrom(bb, ((SqlWithItem) from).query);
- return;
-
- case WITH:
- convertFrom(bb, ((SqlWith) from).body);
- return;
-
- case TABLESAMPLE:
- operands = ((SqlBasicCall) from).getOperands();
- SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
- if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
- String sampleName =
- ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
- .getName();
- datasetStack.push(sampleName);
- convertFrom(bb, operands[0]);
- datasetStack.pop();
- } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
- SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
- (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
- convertFrom(bb, operands[0]);
- RelOptSamplingParameters params =
- new RelOptSamplingParameters(
- tableSampleSpec.isBernoulli(),
- tableSampleSpec.getSamplePercentage(),
- tableSampleSpec.isRepeatable(),
- tableSampleSpec.getRepeatableSeed());
- bb.setRoot(new SamplingRel(cluster, bb.root, params), false);
- } else {
- throw Util.newInternal(
- "unknown TABLESAMPLE type: " + sampleSpec);
- }
- return;
-
- case IDENTIFIER:
- final SqlValidatorNamespace fromNamespace =
- validator.getNamespace(from).resolve();
- if (fromNamespace.getNode() != null) {
- convertFrom(bb, fromNamespace.getNode());
- return;
- }
- final String datasetName =
- datasetStack.isEmpty() ? null : datasetStack.peek();
- boolean[] usedDataset = {false};
- RelOptTable table =
- SqlValidatorUtil.getRelOptTable(
- fromNamespace,
- catalogReader,
- datasetName,
- usedDataset);
- final RelNode tableRel;
- if (shouldConvertTableAccess) {
- tableRel = toRel(table);
- } else {
- tableRel = new TableAccessRel(cluster, table);
- }
- bb.setRoot(tableRel, true);
- if (usedDataset[0]) {
- bb.setDataset(datasetName);
- }
- return;
-
- case JOIN:
- final SqlJoin join = (SqlJoin) from;
- final Blackboard fromBlackboard =
- createBlackboard(validator.getJoinScope(from), null);
- SqlNode left = join.getLeft();
- SqlNode right = join.getRight();
- final boolean isNatural = join.isNatural();
- final JoinType joinType = join.getJoinType();
- final Blackboard leftBlackboard =
- createBlackboard(
- Util.first(validator.getJoinScope(left),
- ((DelegatingScope) bb.scope).getParent()), null);
- final Blackboard rightBlackboard =
- createBlackboard(
- Util.first(validator.getJoinScope(right),
- ((DelegatingScope) bb.scope).getParent()), null);
- convertFrom(leftBlackboard, left);
- RelNode leftRel = leftBlackboard.root;
- convertFrom(rightBlackboard, right);
- RelNode rightRel = rightBlackboard.root;
- JoinRelType convertedJoinType = convertJoinType(joinType);
- RexNode conditionExp;
- if (isNatural) {
- final List<String> columnList =
- SqlValidatorUtil.deriveNaturalJoinColumnList(
- validator.getNamespace(left).getRowType(),
- validator.getNamespace(right).getRowType());
- conditionExp = convertUsing(leftRel, rightRel, columnList);
- } else {
- conditionExp =
- convertJoinCondition(
- fromBlackboard,
- join.getCondition(),
- join.getConditionType(),
- leftRel,
- rightRel);
- }
-
- final RelNode joinRel =
- createJoin(
- fromBlackboard,
- leftRel,
- rightRel,
- conditionExp,
- convertedJoinType);
- bb.setRoot(joinRel, false);
- return;
-
- case SELECT:
- case INTERSECT:
- case EXCEPT:
- case UNION:
- final RelNode rel = convertQueryRecursive(from, false, null);
- bb.setRoot(rel, true);
- return;
-
- case VALUES:
- convertValuesImpl(bb, (SqlCall) from, null);
- return;
-
- case UNNEST:
- final SqlNode node = ((SqlCall) from).operand(0);
- replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- final RelNode childRel =
- RelOptUtil.createProject(
- (null != bb.root) ? bb.root : new OneRowRel(cluster),
- Collections.singletonList(bb.convertExpression(node)),
- Collections.singletonList(validator.deriveAlias(node, 0)),
- true);
-
- UncollectRel uncollectRel =
- new UncollectRel(cluster, cluster.traitSetOf(Convention.NONE),
- childRel);
- bb.setRoot(uncollectRel, true);
- return;
-
- case COLLECTION_TABLE:
- call = (SqlCall) from;
-
- // Dig out real call; TABLE() wrapper is just syntactic.
- assert call.getOperandList().size() == 1;
- call = call.operand(0);
- convertCollectionTable(bb, call);
- return;
-
- default:
- throw Util.newInternal("not a join operator " + from);
- }
- }
-
- protected void convertCollectionTable(
- Blackboard bb,
- SqlCall call) {
- final SqlOperator operator = call.getOperator();
- if (operator == SqlStdOperatorTable.TABLESAMPLE) {
- final String sampleName =
- SqlLiteral.stringValue(call.operand(0));
- datasetStack.push(sampleName);
- SqlCall cursorCall = call.operand(1);
- SqlNode query = cursorCall.operand(0);
- RelNode converted = convertQuery(query, false, false);
- bb.setRoot(converted, false);
- datasetStack.pop();
- return;
- }
- replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // Expand table macro if possible. It's more efficient than
- // TableFunctionRel.
- if (operator instanceof SqlUserDefinedTableMacro) {
- final SqlUserDefinedTableMacro udf =
- (SqlUserDefinedTableMacro) operator;
- final TranslatableTable table = udf.getTable(typeFactory,
- call.getOperandList());
- final RelDataType rowType = table.getRowType(typeFactory);
- RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
- RelNode converted = toRel(relOptTable);
- bb.setRoot(converted, true);
- return;
- }
-
- Type elementType;
- if (operator instanceof SqlUserDefinedTableFunction) {
- SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
- elementType = udtf.getElementType(typeFactory, call.getOperandList());
- } else {
- elementType = null;
- }
-
- RexNode rexCall = bb.convertExpression(call);
- final List<RelNode> inputs = bb.retrieveCursors();
- Set<RelColumnMapping> columnMappings =
- getColumnMappings(operator);
- TableFunctionRel callRel =
- new TableFunctionRel(
- cluster,
- inputs,
- rexCall,
- elementType,
- validator.getValidatedNodeType(call),
- columnMappings);
- bb.setRoot(callRel, true);
- afterTableFunction(bb, call, callRel);
- }
-
- protected void afterTableFunction(
- SqlToRelConverter.Blackboard bb,
- SqlCall call,
- TableFunctionRel callRel) {
- }
-
- private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
- SqlReturnTypeInference rti = op.getReturnTypeInference();
- if (rti == null) {
- return null;
- }
- if (rti instanceof TableFunctionReturnTypeInference) {
- TableFunctionReturnTypeInference tfrti =
- (TableFunctionReturnTypeInference) rti;
- return tfrti.getColumnMappings();
- } else {
- return null;
- }
- }
-
- protected RelNode createJoin(
- Blackboard bb,
- RelNode leftRel,
- RelNode rightRel,
- RexNode joinCond,
- JoinRelType joinType) {
- assert joinCond != null;
-
- Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(rightRel);
- if (correlatedVariables.size() > 0) {
- final List<Correlation> correlations = Lists.newArrayList();
-
- for (String correlName : correlatedVariables) {
- DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
- RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
- String originalRelName = lookup.getOriginalRelName();
- String originalFieldName = fieldAccess.getField().getName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- int childNamespaceIndex = nsIndexes[0];
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
- boolean correlInCurrentScope = ancestorScope == bb.scope;
-
- if (correlInCurrentScope) {
- int namespaceOffset = 0;
- if (childNamespaceIndex > 0) {
- // If not the first child, need to figure out the width
- // of output types from all the preceding namespaces
- assert ancestorScope instanceof ListScope;
- List<SqlValidatorNamespace> children =
- ((ListScope) ancestorScope).getChildren();
-
- for (int i = 0; i < childNamespaceIndex; i++) {
- SqlValidatorNamespace child = children.get(i);
- namespaceOffset +=
- child.getRowType().getFieldCount();
- }
- }
-
- RelDataTypeField field =
- catalogReader.field(foundNs.getRowType(), originalFieldName);
- int pos = namespaceOffset + field.getIndex();
-
- assert field.getType()
- == lookup.getFieldAccess(correlName).getField().getType();
-
- assert pos != -1;
-
- if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
- // bb.root is an aggregate and only projects group by
- // keys.
- Map<Integer, Integer> exprProjection =
- bb.mapRootRelToFieldProjection.get(bb.root);
-
- // subquery can reference group by keys projected from
- // the root of the outer relation.
- if (exprProjection.containsKey(pos)) {
- pos = exprProjection.get(pos);
- } else {
- // correl not grouped
- throw Util.newInternal(
- "Identifier '" + originalRelName + "."
- + originalFieldName + "' is not a group expr");
- }
- }
-
- Correlation newCorVar =
- new Correlation(
- getCorrelOrdinal(correlName),
- pos);
-
- correlations.add(newCorVar);
- }
- }
-
- if (!correlations.isEmpty()) {
- return new CorrelatorRel(
- rightRel.getCluster(),
- leftRel,
- rightRel,
- joinCond,
- correlations,
- joinType);
- }
- }
-
- final List<RexNode> extraLeftExprs = new ArrayList<RexNode>();
- final List<RexNode> extraRightExprs = new ArrayList<RexNode>();
- final int leftCount = leftRel.getRowType().getFieldCount();
- final int rightCount = rightRel.getRowType().getFieldCount();
- if (!containsGet(joinCond)) {
- joinCond = pushDownJoinConditions(
- joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs);
- }
- if (!extraLeftExprs.isEmpty()) {
- final List<RelDataTypeField> fields =
- leftRel.getRowType().getFieldList();
- leftRel = RelOptUtil.createProject(
- leftRel,
- new AbstractList<Pair<RexNode, String>>() {
- @Override
- public int size() {
- return leftCount + extraLeftExprs.size();
- }
-
- @Override
- public Pair<RexNode, String> get(int index) {
- if (index < leftCount) {
- RelDataTypeField field = fields.get(index);
- return Pair.<RexNode, String>of(
- new RexInputRef(index, field.getType()),
- field.getName());
- } else {
- return Pair.<RexNode, String>of(
- extraLeftExprs.get(index - leftCount), null);
- }
- }
- },
- true);
- }
- if (!extraRightExprs.isEmpty()) {
- final List<RelDataTypeField> fields =
- rightRel.getRowType().getFieldList();
- final int newLeftCount = leftCount + extraLeftExprs.size();
- rightRel = RelOptUtil.createProject(
- rightRel,
- new AbstractList<Pair<RexNode, String>>() {
- @Override
- public int size() {
- return rightCount + extraRightExprs.size();
- }
-
- @Override
- public Pair<RexNode, String> get(int index) {
- if (index < rightCount) {
- RelDataTypeField field = fields.get(index);
- return Pair.<RexNode, String>of(
- new RexInputRef(index, field.getType()),
- field.getName());
- } else {
- return Pair.of(
- RexUtil.shift(
- extraRightExprs.get(index - rightCount),
- -newLeftCount),
- null);
- }
- }
- },
- true);
- }
- RelNode join = createJoin(
- leftRel,
- rightRel,
- joinCond,
- joinType,
- ImmutableSet.<String>of());
- if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) {
- Mappings.TargetMapping mapping =
- Mappings.createShiftMapping(
- leftCount + extraLeftExprs.size()
- + rightCount + extraRightExprs.size(),
- 0, 0, leftCount,
- leftCount, leftCount + extraLeftExprs.size(), rightCount);
- return RelOptUtil.project(join, mapping);
- }
- return join;
- }
-
- private static boolean containsGet(RexNode node) {
- try {
- node.accept(
- new RexVisitorImpl<Void>(true) {
- @Override public Void visitCall(RexCall call) {
- if (call.getOperator() == RexBuilder.GET_OPERATOR) {
- throw Util.FoundOne.NULL;
- }
- return super.visitCall(call);
- }
- });
- return false;
- } catch (Util.FoundOne e) {
- return true;
- }
- }
-
- /**
- * Pushes down parts of a join condition. For example, given
- * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above
- * "emp" that computes the expression
- * "emp.deptno + 1". The resulting join condition is a simple combination
- * of AND, equals, and input fields.
- */
- private RexNode pushDownJoinConditions(
- RexNode node,
- int leftCount,
- int rightCount,
- List<RexNode> extraLeftExprs,
- List<RexNode> extraRightExprs) {
- switch (node.getKind()) {
- case AND:
- case OR:
- case EQUALS:
- RexCall call = (RexCall) node;
- List<RexNode> list = new ArrayList<RexNode>();
- List<RexNode> operands = Lists.newArrayList(call.getOperands());
- for (int i = 0; i < operands.size(); i++) {
- RexNode operand = operands.get(i);
- final int left2 = leftCount + extraLeftExprs.size();
- final int right2 = rightCount + extraRightExprs.size();
- final RexNode e =
- pushDownJoinConditions(
- operand,
- leftCount,
- rightCount,
- extraLeftExprs,
- extraRightExprs);
- final List<RexNode> remainingOperands = Util.skip(operands, i + 1);
- final int left3 = leftCount + extraLeftExprs.size();
- final int right3 = rightCount + extraRightExprs.size();
- fix(remainingOperands, left2, left3);
- fix(list, left2, left3);
- list.add(e);
- }
- if (!list.equals(call.getOperands())) {
- return call.clone(call.getType(), list);
- }
- return call;
- case INPUT_REF:
- case LITERAL:
- return node;
- default:
- BitSet bits = RelOptUtil.InputFinder.bits(node);
- final int mid = leftCount + extraLeftExprs.size();
- switch (Side.of(bits, mid)) {
- case LEFT:
- fix(extraRightExprs, mid, mid + 1);
- extraLeftExprs.add(node);
- return new RexInputRef(mid, node.getType());
- case RIGHT:
- final int index2 = mid + rightCount + extraRightExprs.size();
- extraRightExprs.add(node);
- return new RexInputRef(index2, node.getType());
- case BOTH:
- case EMPTY:
- default:
- return node;
- }
- }
- }
-
- private void fix(List<RexNode> operands, int before, int after) {
- if (before == after) {
- return;
- }
- for (int i = 0; i < operands.size(); i++) {
- RexNode node = operands.get(i);
- operands.set(i, RexUtil.shift(node, before, after - before));
- }
- }
-
- /**
- * Categorizes whether a bit set contains bits left and right of a
- * line.
- */
- enum Side {
- LEFT, RIGHT, BOTH, EMPTY;
-
- static Side of(BitSet bitSet, int middle) {
- final int firstBit = bitSet.nextSetBit(0);
- if (firstBit < 0) {
- return EMPTY;
- }
- if (firstBit >= middle) {
- return RIGHT;
- }
- if (bitSet.nextSetBit(middle) < 0) {
- return LEFT;
- }
- return BOTH;
- }
- }
-
- /**
- * Determines whether a subquery is non-correlated. Note that a
- * non-correlated subquery can contain correlated references, provided those
- * references do not reference select statements that are parents of the
- * subquery.
- *
- * @param subq the subquery
- * @param bb blackboard used while converting the subquery, i.e., the
- * blackboard of the parent query of this subquery
- * @return true if the subquery is non-correlated.
- */
- private boolean isSubqNonCorrelated(RelNode subq, Blackboard bb) {
- Set<String> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
- for (String correlName : correlatedVariables) {
- DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
- String originalRelName = lookup.getOriginalRelName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
-
- // If the correlated reference is in a scope that's "above" the
- // subquery, then this is a correlated subquery.
- SqlValidatorScope parentScope = bb.scope;
- do {
- if (ancestorScope == parentScope) {
- return false;
- }
- if (parentScope instanceof DelegatingScope) {
- parentScope = ((DelegatingScope) parentScope).getParent();
- } else {
- break;
- }
- } while (parentScope != null);
- }
- return true;
- }
-
- /**
- * Returns a list of fields to be prefixed to each relational expression.
- *
- * @return List of system fields
- */
- protected List<RelDataTypeField> getSystemFields() {
- return Collections.emptyList();
- }
-
- private RexNode convertJoinCondition(
- Blackboard bb,
- SqlNode condition,
- JoinConditionType conditionType,
- RelNode leftRel,
- RelNode rightRel) {
- if (condition == null) {
- return rexBuilder.makeLiteral(true);
- }
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- switch (conditionType) {
- case ON:
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- return bb.convertExpression(condition);
- case USING:
- SqlNodeList list = (SqlNodeList) condition;
- List<String> nameList = new ArrayList<String>();
- for (SqlNode columnName : list) {
- final SqlIdentifier id = (SqlIdentifier) columnName;
- String name = id.getSimple();
- nameList.add(name);
- }
- return convertUsing(leftRel, rightRel, nameList);
- default:
- throw Util.unexpected(conditionType);
- }
- }
-
- /**
- * Returns an expression for matching columns of a USING clause or inferred
- * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
- * b.y". Returns null if the column list is empty.
- *
- * @param leftRel Left input to the join
- * @param rightRel Right input to the join
- * @param nameList List of column names to join on
- * @return Expression to match columns from name list, or true if name list
- * is empty
- */
- private RexNode convertUsing(
- RelNode leftRel,
- RelNode rightRel,
- List<String> nameList) {
- final List<RexNode> list = Lists.newArrayList();
- for (String name : nameList) {
- final RelDataType leftRowType = leftRel.getRowType();
- RelDataTypeField leftField = catalogReader.field(leftRowType, name);
- RexNode left =
- rexBuilder.makeInputRef(
- leftField.getType(),
- leftField.getIndex());
- final RelDataType rightRowType = rightRel.getRowType();
- RelDataTypeField rightField =
- catalogReader.field(rightRowType, name);
- RexNode right =
- rexBuilder.makeInputRef(
- rightField.getType(),
- leftRowType.getFieldList().size() + rightField.getIndex());
- RexNode equalsCall =
- rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- left,
- right);
- list.add(equalsCall);
- }
- return RexUtil.composeConjunction(rexBuilder, list, false);
- }
-
- private static JoinRelType convertJoinType(JoinType joinType) {
- switch (joinType) {
- case COMMA:
- case INNER:
- case CROSS:
- return JoinRelType.INNER;
- case FULL:
- return JoinRelType.FULL;
- case LEFT:
- return JoinRelType.LEFT;
- case RIGHT:
- return JoinRelType.RIGHT;
- default:
- throw Util.unexpected(joinType);
- }
- }
-
- /**
- * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query.
- *
- * <p>This method extracts SELECT, GROUP BY and HAVING clauses, and creates
- * an {@link AggConverter}, then delegates to {@link #createAggImpl}.
- * Derived class may override this method to change any of those clauses or
- * specify a different {@link AggConverter}.
- *
- * @param bb Scope within which to resolve identifiers
- * @param select Query
- * @param orderExprList Additional expressions needed to implement ORDER BY
- */
- protected void convertAgg(
- Blackboard bb,
- SqlSelect select,
- List<SqlNode> orderExprList) {
- assert bb.root != null : "precondition: child != null";
- SqlNodeList groupList = select.getGroup();
- SqlNodeList selectList = select.getSelectList();
- SqlNode having = select.getHaving();
-
- final AggConverter aggConverter = new AggConverter(bb, select);
- createAggImpl(
- bb,
- aggConverter,
- selectList,
- groupList,
- having,
- orderExprList);
- }
-
- protected final void createAggImpl(
- Blackboard bb,
- AggConverter aggConverter,
- SqlNodeList selectList,
- SqlNodeList groupList,
- SqlNode having,
- List<SqlNode> orderExprList) {
- SqlNodeList aggList = new SqlNodeList(SqlParserPos.ZERO);
-
- for (SqlNode selectNode : selectList) {
- if (validator.isAggregate(selectNode)) {
- aggList.add(selectNode);
- }
- }
-
- // first replace the subqueries inside the aggregates
- // because they will provide input rows to the aggregates.
- replaceSubqueries(bb, aggList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // If group-by clause is missing, pretend that it has zero elements.
- if (groupList == null) {
- groupList = SqlNodeList.EMPTY;
- }
-
- // register the group exprs
-
- // build a map to remember the projections from the top scope to the
- // output of the current root.
- //
- // Currently farrago allows expressions, not just column references in
- // group by list. This is not SQL 2003 compliant.
-
- Map<Integer, Integer> groupExprProjection =
- new HashMap<Integer, Integer>();
-
- int i = -1;
- for (SqlNode groupExpr : groupList) {
- ++i;
- final SqlNode expandedGroupExpr =
- validator.expand(groupExpr, bb.scope);
- aggConverter.addGroupExpr(expandedGroupExpr);
-
- if (expandedGroupExpr instanceof SqlIdentifier) {
- // SQL 2003 does not allow expressions of column references
- SqlIdentifier expr = (SqlIdentifier) expandedGroupExpr;
-
- // column references should be fully qualified.
- assert expr.names.size() == 2;
- String originalRelName = expr.names.get(0);
- String originalFieldName = expr.names.get(1);
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- bb.scope.resolve(
- originalRelName,
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
- int childNamespaceIndex = nsIndexes[0];
-
- int namespaceOffset = 0;
-
- if (childNamespaceIndex > 0) {
- // If not the first child, need to figure out the width of
- // output types from all the preceding namespaces
- assert ancestorScopes[0] instanceof ListScope;
- List<SqlValidatorNamespace> children =
- ((ListScope) ancestorScopes[0]).getChildren();
-
- for (int j = 0; j < childNamespaceIndex; j++) {
- namespaceOffset +=
- children.get(j).getRowType().getFieldCount();
- }
- }
-
- RelDataTypeField field =
- catalogReader.field(foundNs.getRowType(), originalFieldName);
- int origPos = namespaceOffset + field.getIndex();
-
- groupExprProjection.put(origPos, i);
- }
- }
-
- RexNode havingExpr = null;
- List<RexNode> selectExprs = new ArrayList<RexNode>();
- List<String> selectNames = new ArrayList<String>();
-
- try {
- Util.permAssert(bb.agg == null, "already in agg mode");
- bb.agg = aggConverter;
-
- // convert the select and having expressions, so that the
- // agg converter knows which aggregations are required
-
- selectList.accept(aggConverter);
- for (SqlNode expr : orderExprList) {
- expr.accept(aggConverter);
- }
- if (having != null) {
- having.accept(aggConverter);
- }
-
- // compute inputs to the aggregator
- List<RexNode> preExprs = aggConverter.getPreExprs();
- List<String> preNames = aggConverter.getPreNames();
-
- if (preExprs.size() == 0) {
- // Special case for COUNT(*), where we can end up with no inputs
- // at all. The rest of the system doesn't like 0-tuples, so we
- // select a dummy constant here.
- preExprs =
- Collections.singletonList(
- (RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO));
- preNames = Collections.singletonList(null);
- }
-
- final RelNode inputRel = bb.root;
-
- // Project the expressions required by agg and having.
- bb.setRoot(
- RelOptUtil.createProject(
- inputRel,
- preExprs,
- preNames,
- true),
- false);
- bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection);
-
- // REVIEW jvs 31-Oct-2007: doesn't the declaration of
- // monotonicity here assume sort-based aggregation at
- // the physical level?
-
- // Tell bb which of group columns are sorted.
- bb.columnMonotonicities.clear();
- for (SqlNode groupItem : groupList) {
- bb.columnMonotonicities.add(
- bb.scope.getMonotonicity(groupItem));
- }
-
- // Add the aggregator
- bb.setRoot(
- createAggregate(
- bb,
- BitSets.range(aggConverter.groupExprs.size()),
- aggConverter.getAggCalls()),
- false);
-
- bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection);
-
- // Replace subqueries in having here and modify having to use
- // the replaced expressions
- if (having
<TRUNCATED>
[35/50] [abbrv] incubator-kylin git commit: KYLIN-780 JDBC upgrade,
test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
deleted file mode 100644
index 67652cb..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.ColumnMetaData.Rep;
-import org.apache.calcite.avatica.ColumnMetaData.ScalarType;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethodBase;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.StringRequestEntity;
-import org.apache.commons.httpclient.protocol.Protocol;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-import org.apache.kylin.jdbc.KylinConnectionImpl;
-import org.apache.kylin.jdbc.KylinEnumerator;
-import org.apache.kylin.jdbc.KylinMetaImpl;
-import org.apache.kylin.jdbc.util.DefaultSslProtocolSocketFactory;
-import org.apache.kylin.jdbc.util.SQLTypeMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.HashMultimap;
-import org.apache.kylin.jdbc.KylinJdbc41Factory.KylinJdbc41PreparedStatement;
-import org.apache.kylin.jdbc.stub.TableMetaStub.ColumnMetaStub;
-
-/**
- * @author xduo
- *
- */
-public class KylinClient implements RemoteClient {
- private static final Logger logger = LoggerFactory.getLogger(KylinClient.class);
-
- private final KylinConnectionImpl conn;
-
- public KylinClient(KylinConnectionImpl conn) {
- this.conn = conn;
- }
-
- @Override
- public void connect() throws ConnectionException {
- PostMethod post = new PostMethod(conn.getConnectUrl());
- HttpClient httpClient = new HttpClient();
-
- if (conn.getConnectUrl().toLowerCase().startsWith("https://")) {
- registerSsl();
- }
- addPostHeaders(post);
-
- try {
- StringRequestEntity requestEntity = new StringRequestEntity("{}", "application/json", "UTF-8");
- post.setRequestEntity(requestEntity);
- httpClient.executeMethod(post);
-
- if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
- logger.error("Connect failed with error code " + post.getStatusCode() + " and message:\n" + post.getResponseBodyAsString());
-
- throw new ConnectionException("Connect failed, error code " + post.getStatusCode() + " and message: " + post.getResponseBodyAsString());
- }
- } catch (HttpException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new ConnectionException(e.getLocalizedMessage());
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new ConnectionException(e.getLocalizedMessage());
- }
- }
-
- @Override
- public KylinMetaImpl.MetaProject getMetadata(String project) throws ConnectionException {
- GetMethod get = new GetMethod(conn.getMetaProjectUrl(project));
- HttpClient httpClient = new HttpClient();
-
- if (conn.getConnectUrl().toLowerCase().startsWith("https://")) {
- registerSsl();
- }
- addPostHeaders(get);
-
- List<TableMetaStub> tableMetaStubs = null;
- try {
- httpClient.executeMethod(get);
-
- if (get.getStatusCode() != 200 && get.getStatusCode() != 201) {
- logger.error("Connect failed with error code " + get.getStatusCode() + " and message:\n" + get.getResponseBodyAsString());
-
- throw new ConnectionException("Connect failed, error code " + get.getStatusCode() + " and message: " + get.getResponseBodyAsString());
- }
-
- tableMetaStubs = new ObjectMapper().readValue(get.getResponseBodyAsString(), new TypeReference<List<TableMetaStub>>() {
- });
-
- List<KylinMetaImpl.MetaTable> tables = new ArrayList<KylinMetaImpl.MetaTable>();
- HashMultimap<String, KylinMetaImpl.MetaTable> schemasMap = HashMultimap.create();
-
- for (TableMetaStub tableMetaStub : tableMetaStubs) {
- List<KylinMetaImpl.MetaColumn> columns = new ArrayList<KylinMetaImpl.MetaColumn>();
-
- for (ColumnMetaStub columnMetaStub : tableMetaStub.getColumns()) {
- KylinMetaImpl.MetaColumn column = createNewColumn(columnMetaStub);
- columns.add(column);
- }
-
- KylinMetaImpl.MetaTable table = createNewTable(tableMetaStub, columns);
- tables.add(table);
- schemasMap.put(tableMetaStub.getTABLE_CAT() + "#" + tableMetaStub.getTABLE_SCHEM(), table);
- }
-
- HashMultimap<String, KylinMetaImpl.MetaSchema> catalogMap = HashMultimap.create();
- List<KylinMetaImpl.MetaSchema> schemas = new ArrayList<KylinMetaImpl.MetaSchema>();
- for (String key : schemasMap.keySet()) {
- String cat = key.split("#")[0];
- String schema = key.split("#")[1];
- KylinMetaImpl.MetaSchema metaSchema = new KylinMetaImpl.MetaSchema(cat, schema, new ArrayList<KylinMetaImpl.MetaTable>(schemasMap.get(key)));
- schemas.add(metaSchema);
- catalogMap.put(cat, metaSchema);
- }
-
- List<KylinMetaImpl.MetaCatalog> catalogs = new ArrayList<KylinMetaImpl.MetaCatalog>();
- for (String key : catalogMap.keySet()) {
- KylinMetaImpl.MetaCatalog metaCatalog = new KylinMetaImpl.MetaCatalog(key, new ArrayList<KylinMetaImpl.MetaSchema>(catalogMap.get(key)));
- catalogs.add(metaCatalog);
- }
-
- return new KylinMetaImpl.MetaProject(project, catalogs);
- } catch (HttpException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new ConnectionException(e.getLocalizedMessage());
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new ConnectionException(e.getLocalizedMessage());
- }
- }
-
- private KylinMetaImpl.MetaTable createNewTable(TableMetaStub tableMetaStub, List<KylinMetaImpl.MetaColumn> columns) {
- KylinMetaImpl.MetaTable table = new KylinMetaImpl.MetaTable(tableMetaStub.getTABLE_CAT(), tableMetaStub.getTABLE_SCHEM(), tableMetaStub.getTABLE_NAME(), tableMetaStub.getTABLE_TYPE(), tableMetaStub.getREMARKS(), tableMetaStub.getTYPE_CAT(), tableMetaStub.getTYPE_SCHEM(), tableMetaStub.getTYPE_NAME(), tableMetaStub.getSELF_REFERENCING_COL_NAME(), tableMetaStub.getREF_GENERATION(), columns);
- return table;
- }
-
- private KylinMetaImpl.MetaColumn createNewColumn(ColumnMetaStub columnMetaStub) {
- KylinMetaImpl.MetaColumn column = new KylinMetaImpl.MetaColumn(columnMetaStub.getTABLE_CAT(), columnMetaStub.getTABLE_SCHEM(), columnMetaStub.getTABLE_NAME(), columnMetaStub.getCOLUMN_NAME(), columnMetaStub.getDATA_TYPE(), columnMetaStub.getTYPE_NAME(), columnMetaStub.getCOLUMN_SIZE(), columnMetaStub.getBUFFER_LENGTH(), columnMetaStub.getDECIMAL_DIGITS(), columnMetaStub.getNUM_PREC_RADIX(), columnMetaStub.getNULLABLE(), columnMetaStub.getREMARKS(), columnMetaStub.getCOLUMN_DEF(), columnMetaStub.getSQL_DATA_TYPE(), columnMetaStub.getSQL_DATETIME_SUB(), columnMetaStub.getCHAR_OCTET_LENGTH(), columnMetaStub.getORDINAL_POSITION(), columnMetaStub.getIS_NULLABLE(), columnMetaStub.getSCOPE_CATLOG(), columnMetaStub.getSCOPE_TABLE(), columnMetaStub.getSOURCE_DATA_TYPE(), columnMetaStub.getIS_AUTOINCREMENT(), columnMetaStub.getSCOPE_SCHEMA());
- return column;
- }
-
- @Override
- public DataSet<Object[]> query(AvaticaStatement statement, String sql) throws SQLException {
- SQLResponseStub queryRes = null;
-
- List<StateParam> params = null;
- if (statement instanceof KylinJdbc41PreparedStatement) {
- params = genPrestateStates(statement);
- }
-
- queryRes = runKylinQuery(sql, params);
-
- List<ColumnMetaData> metas = genColumnMeta(queryRes);
- List<Object[]> data = genResultData(queryRes, metas);
-
- return new DataSet<Object[]>(metas, new KylinEnumerator<Object[]>(data));
- }
-
- /**
- * @param queryRes
- * @param metas
- * @return
- */
- private List<Object[]> genResultData(SQLResponseStub queryRes, List<ColumnMetaData> metas) {
- List<Object[]> data = new ArrayList<Object[]>();
- for (String[] result : queryRes.getResults()) {
- Object[] row = new Object[result.length];
-
- for (int i = 0; i < result.length; i++) {
- ColumnMetaData meta = metas.get(i);
- row[i] = SQLTypeMap.wrapObject(result[i], meta.type.type);
- }
-
- data.add(row);
- }
- return data;
- }
-
- /**
- * @param statement
- * @param params
- */
- private List<StateParam> genPrestateStates(AvaticaStatement statement) {
- List<StateParam> params = new ArrayList<StateParam>();
- List<Object> values = ((KylinJdbc41PreparedStatement) statement).getParameterValues();
-
- for (int i = 0; i < values.size(); i++) {
- Object value = values.get(i);
- params.add(new StateParam(value.getClass().getCanonicalName(), String.valueOf(value)));
- }
-
- return params;
- }
-
- /**
- * @param queryRes
- * @return
- */
- private List<ColumnMetaData> genColumnMeta(SQLResponseStub queryRes) {
- List<ColumnMetaData> metas = new ArrayList<ColumnMetaData>();
- for (int i = 0; i < queryRes.getColumnMetas().size(); i++) {
- SQLResponseStub.ColumnMetaStub scm = queryRes.getColumnMetas().get(i);
- ScalarType type = ColumnMetaData.scalar(scm.getColumnType(), scm.getColumnTypeName(), Rep.of(SQLTypeMap.convert(scm.getColumnType())));
-
- ColumnMetaData meta = new ColumnMetaData(i, scm.isAutoIncrement(), scm.isCaseSensitive(), scm.isSearchable(), scm.isCurrency(), scm.getIsNullable(), scm.isSigned(), scm.getDisplaySize(), scm.getLabel(), scm.getName(), scm.getSchemaName(), scm.getPrecision(), scm.getScale(), scm.getTableName(), scm.getSchemaName(), type, scm.isReadOnly(), scm.isWritable(), scm.isWritable(), null);
-
- metas.add(meta);
- }
-
- return metas;
- }
-
- /**
- * @param sql
- * @return
- * @throws IOException
- */
- private SQLResponseStub runKylinQuery(String sql, List<StateParam> params) throws SQLException {
- String url = conn.getQueryUrl();
- String project = conn.getProject();
- QueryRequest request = null;
-
- if (null != params) {
- request = new PreQueryRequest();
- ((PreQueryRequest) request).setParams(params);
- url += "/prestate";
- } else {
- request = new QueryRequest();
- }
- request.setSql(sql);
- request.setProject(project);
-
- PostMethod post = new PostMethod(url);
- addPostHeaders(post);
- HttpClient httpClient = new HttpClient();
- if (conn.getQueryUrl().toLowerCase().startsWith("https://")) {
- registerSsl();
- }
-
- String postBody = null;
- ObjectMapper mapper = new ObjectMapper();
- try {
- postBody = mapper.writeValueAsString(request);
- logger.debug("Post body:\n " + postBody);
- } catch (JsonProcessingException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- String response = null;
- SQLResponseStub queryRes = null;
-
- try {
- StringRequestEntity requestEntity = new StringRequestEntity(postBody, "application/json", "UTF-8");
- post.setRequestEntity(requestEntity);
-
- httpClient.executeMethod(post);
- response = post.getResponseBodyAsString();
-
- if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
- logger.error("Failed to query", response);
- throw new SQLException(response);
- }
-
- queryRes = new ObjectMapper().readValue(response, SQLResponseStub.class);
-
- } catch (HttpException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new SQLException(e.getLocalizedMessage());
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- throw new SQLException(e.getLocalizedMessage());
- }
-
- return queryRes;
- }
-
- private void addPostHeaders(HttpMethodBase method) {
- method.addRequestHeader("Accept", "application/json, text/plain, */*");
- method.addRequestHeader("Content-Type", "application/json");
- method.addRequestHeader("Authorization", "Basic " + conn.getBasicAuthHeader());
- }
-
- private void registerSsl() {
- Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new DefaultSslProtocolSocketFactory(), 443));
- }
-
- public class QueryRequest {
- private String sql;
- private String project;
-
- public String getSql() {
- return sql;
- }
-
- public void setSql(String sql) {
- this.sql = sql;
- }
-
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
- }
-
- public class PreQueryRequest extends QueryRequest {
- private List<StateParam> params;
-
- public List<StateParam> getParams() {
- return params;
- }
-
- public void setParams(List<StateParam> params) {
- this.params = params;
- }
- }
-
- public class StateParam {
- private String className;
- private String value;
-
- public StateParam(String className, String value) {
- super();
- this.className = className;
- this.value = value;
- }
-
- public String getClassName() {
- return className;
- }
-
- public void setClazz(String className) {
- this.className = className;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinColumnMetaData.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinColumnMetaData.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinColumnMetaData.java
deleted file mode 100644
index e0e14e7..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinColumnMetaData.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.sql.DatabaseMetaData;
-
-import net.hydromatic.avatica.ColumnMetaData;
-
-/**
- * @author xduo
- *
- */
-public class KylinColumnMetaData extends ColumnMetaData {
-
- public KylinColumnMetaData(int ordinal, boolean autoIncrement, boolean caseSensitive, boolean searchable, boolean currency, int nullable, boolean signed, int displaySize, String label, String columnName, String schemaName, int precision, int scale, String tableName, String catalogName, AvaticaType type, boolean readOnly, boolean writable, boolean definitelyWritable, String columnClassName) {
- super(ordinal, autoIncrement, caseSensitive, searchable, currency, nullable, signed, displaySize, label, columnName, schemaName, precision, scale, tableName, catalogName, type, readOnly, writable, definitelyWritable, columnClassName);
- }
-
- public static ColumnMetaData dummy(int ordinal, String label, String columnName, AvaticaType type, boolean nullable) {
- return new ColumnMetaData(ordinal, false, true, false, false, nullable ? DatabaseMetaData.columnNullable : DatabaseMetaData.columnNoNulls, true, -1, label, columnName, null, -1, -1, null, null, type, true, false, false, null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
deleted file mode 100644
index 093f150..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.sql.SQLException;
-
-import org.apache.calcite.avatica.AvaticaStatement;
-
-import org.apache.kylin.jdbc.KylinMetaImpl.MetaProject;
-
-/**
- * Remote query stub of kylin restful service
- *
- * @author xduo
- *
- */
-public interface RemoteClient {
-
- /**
- * Connect to kylin restful service. ConnectionException will be thrown if
- * authentication failed.
- *
- * @throws ConnectionException
- */
- public void connect() throws ConnectionException;
-
- /**
- * @param project
- * @return
- */
- public MetaProject getMetadata(String project) throws ConnectionException;
-
- /**
- * Run query
- *
- * @param statement
- * @param sql
- * @return
- * @throws SQLException
- */
- public DataSet<Object[]> query(AvaticaStatement statement, String sql) throws SQLException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/SQLResponseStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/SQLResponseStub.java
deleted file mode 100644
index a9c6c02..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/SQLResponseStub.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * @author xduo
- *
- */
-public class SQLResponseStub implements Serializable {
- private static final long serialVersionUID = 1L;
-
- // private static final Logger logger =
- // LoggerFactory.getLogger(SQLResponse.class);
-
- // the data type for each column
- private List<ColumnMetaStub> columnMetas;
-
- // the results rows, each row contains several columns
- private List<String[]> results;
-
- private String cube;
-
- // if not select query, only return affected row count
- private int affectedRowCount;
-
- // if isException, the detailed exception message
- private String exceptionMessage;
-
- private boolean isException;
-
- private long duration;
-
- private boolean isPartial = false;
-
- private long totalScanCount;
-
- private boolean hitCache = false;
-
- public SQLResponseStub() {
- }
-
- public List<ColumnMetaStub> getColumnMetas() {
- return columnMetas;
- }
-
- public void setColumnMetas(List<ColumnMetaStub> columnMetas) {
- this.columnMetas = columnMetas;
- }
-
- public List<String[]> getResults() {
- return results;
- }
-
- public void setResults(List<String[]> results) {
- this.results = results;
- }
-
- public String getCube() {
- return cube;
- }
-
- public void setCube(String cube) {
- this.cube = cube;
- }
-
- public int getAffectedRowCount() {
- return affectedRowCount;
- }
-
- public void setAffectedRowCount(int affectedRowCount) {
- this.affectedRowCount = affectedRowCount;
- }
-
- public boolean getIsException() {
- return isException;
- }
-
- public void setIsException(boolean isException) {
- this.isException = isException;
- }
-
- public String getExceptionMessage() {
- return exceptionMessage;
- }
-
- public void setExceptionMessage(String exceptionMessage) {
- this.exceptionMessage = exceptionMessage;
- }
-
- public long getDuration() {
- return duration;
- }
-
- public void setDuration(long duration) {
- this.duration = duration;
- }
-
- public boolean isPartial() {
- return isPartial;
- }
-
- public void setPartial(boolean isPartial) {
- this.isPartial = isPartial;
- }
-
- public long getTotalScanCount() {
- return totalScanCount;
- }
-
- public void setTotalScanCount(long totalScanCount) {
- this.totalScanCount = totalScanCount;
- }
-
- public boolean isHitCache() {
- return hitCache;
- }
-
- public void setHitCache(boolean hitCache) {
- this.hitCache = hitCache;
- }
-
- public static long getSerialversionuid() {
- return serialVersionUID;
- }
-
- public static class ColumnMetaStub {
-
- private boolean isAutoIncrement;
- private boolean isCaseSensitive;
- private boolean isSearchable;
- private boolean isCurrency;
- private int isNullable;// 0:nonull, 1:nullable, 2: nullableunknown
- private boolean isSigned;
- private int displaySize;
- private String label;// AS keyword
- private String name;
- private String schemaName;
- private String catelogName;
- private String tableName;
- private int precision;
- private int scale;
- private int columnType;// as defined in java.sql.Types
- private String columnTypeName;
- private boolean isReadOnly;
- private boolean isWritable;
- private boolean isDefinitelyWritable;
-
- public ColumnMetaStub() {
- }
-
- public boolean isAutoIncrement() {
- return isAutoIncrement;
- }
-
- public void setAutoIncrement(boolean isAutoIncrement) {
- this.isAutoIncrement = isAutoIncrement;
- }
-
- public boolean isCaseSensitive() {
- return isCaseSensitive;
- }
-
- public void setCaseSensitive(boolean isCaseSensitive) {
- this.isCaseSensitive = isCaseSensitive;
- }
-
- public boolean isSearchable() {
- return isSearchable;
- }
-
- public void setSearchable(boolean isSearchable) {
- this.isSearchable = isSearchable;
- }
-
- public boolean isCurrency() {
- return isCurrency;
- }
-
- public void setCurrency(boolean isCurrency) {
- this.isCurrency = isCurrency;
- }
-
- public int getIsNullable() {
- return isNullable;
- }
-
- public void setIsNullable(int isNullable) {
- this.isNullable = isNullable;
- }
-
- public boolean isSigned() {
- return isSigned;
- }
-
- public void setSigned(boolean isSigned) {
- this.isSigned = isSigned;
- }
-
- public int getDisplaySize() {
- return displaySize;
- }
-
- public void setDisplaySize(int displaySize) {
- this.displaySize = displaySize;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getSchemaName() {
- return schemaName;
- }
-
- public void setSchemaName(String schemaName) {
- this.schemaName = schemaName;
- }
-
- public String getCatelogName() {
- return catelogName;
- }
-
- public void setCatelogName(String catelogName) {
- this.catelogName = catelogName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public void setPrecision(int precision) {
- this.precision = precision;
- }
-
- public int getScale() {
- return scale;
- }
-
- public void setScale(int scale) {
- this.scale = scale;
- }
-
- public int getColumnType() {
- return columnType;
- }
-
- public void setColumnType(int columnType) {
- this.columnType = columnType;
- }
-
- public String getColumnTypeName() {
- return columnTypeName;
- }
-
- public void setColumnTypeName(String columnTypeName) {
- this.columnTypeName = columnTypeName;
- }
-
- public boolean isReadOnly() {
- return isReadOnly;
- }
-
- public void setReadOnly(boolean isReadOnly) {
- this.isReadOnly = isReadOnly;
- }
-
- public boolean isWritable() {
- return isWritable;
- }
-
- public void setWritable(boolean isWritable) {
- this.isWritable = isWritable;
- }
-
- public boolean isDefinitelyWritable() {
- return isDefinitelyWritable;
- }
-
- public void setDefinitelyWritable(boolean isDefinitelyWritable) {
- this.isDefinitelyWritable = isDefinitelyWritable;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/TableMetaStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/TableMetaStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/TableMetaStub.java
deleted file mode 100644
index 524fa81..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/TableMetaStub.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.stub;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author xduo
- *
- */
-public class TableMetaStub {
-
- private static final long serialVersionUID = 1L;
- private String TABLE_CAT;
- private String TABLE_SCHEM;
- private String TABLE_NAME;
- private String TABLE_TYPE;
- private String REMARKS;
- private String TYPE_CAT;
- private String TYPE_SCHEM;
- private String TYPE_NAME;
- private String SELF_REFERENCING_COL_NAME;
- private String REF_GENERATION;
- @JsonProperty("columns")
- private List<ColumnMetaStub> columns = new ArrayList<ColumnMetaStub>();
-
- public String getTABLE_CAT() {
- return TABLE_CAT;
- }
-
- public void setTABLE_CAT(String tABLE_CAT) {
- TABLE_CAT = tABLE_CAT;
- }
-
- public String getTABLE_SCHEM() {
- return TABLE_SCHEM;
- }
-
- public void setTABLE_SCHEM(String tABLE_SCHEM) {
- TABLE_SCHEM = tABLE_SCHEM;
- }
-
- public String getTABLE_NAME() {
- return TABLE_NAME;
- }
-
- public void setTABLE_NAME(String tABLE_NAME) {
- TABLE_NAME = tABLE_NAME;
- }
-
- public String getTABLE_TYPE() {
- return TABLE_TYPE;
- }
-
- public void setTABLE_TYPE(String tABLE_TYPE) {
- TABLE_TYPE = tABLE_TYPE;
- }
-
- public String getREMARKS() {
- return REMARKS;
- }
-
- public void setREMARKS(String rEMARKS) {
- REMARKS = rEMARKS;
- }
-
- public String getTYPE_CAT() {
- return TYPE_CAT;
- }
-
- public void setTYPE_CAT(String tYPE_CAT) {
- TYPE_CAT = tYPE_CAT;
- }
-
- public String getTYPE_SCHEM() {
- return TYPE_SCHEM;
- }
-
- public void setTYPE_SCHEM(String tYPE_SCHEM) {
- TYPE_SCHEM = tYPE_SCHEM;
- }
-
- public String getTYPE_NAME() {
- return TYPE_NAME;
- }
-
- public void setTYPE_NAME(String tYPE_NAME) {
- TYPE_NAME = tYPE_NAME;
- }
-
- public String getSELF_REFERENCING_COL_NAME() {
- return SELF_REFERENCING_COL_NAME;
- }
-
- public void setSELF_REFERENCING_COL_NAME(String sELF_REFERENCING_COL_NAME) {
- SELF_REFERENCING_COL_NAME = sELF_REFERENCING_COL_NAME;
- }
-
- public String getREF_GENERATION() {
- return REF_GENERATION;
- }
-
- public void setREF_GENERATION(String rEF_GENERATION) {
- REF_GENERATION = rEF_GENERATION;
- }
-
- public List<ColumnMetaStub> getColumns() {
- return columns;
- }
-
- public void setColumns(List<ColumnMetaStub> columns) {
- this.columns = columns;
- }
-
- public static long getSerialversionuid() {
- return serialVersionUID;
- }
-
- public static class ColumnMetaStub {
- private String TABLE_CAT;
- private String TABLE_SCHEM;
- private String TABLE_NAME;
- private String COLUMN_NAME;
- private int DATA_TYPE;
- private String TYPE_NAME;
- private int COLUMN_SIZE;
- private int BUFFER_LENGTH;
- private int DECIMAL_DIGITS;
- private int NUM_PREC_RADIX;
- private int NULLABLE;
- private String REMARKS;
- private String COLUMN_DEF;
- private int SQL_DATA_TYPE;
- private int SQL_DATETIME_SUB;
- private int CHAR_OCTET_LENGTH;
- private int ORDINAL_POSITION;
- private String IS_NULLABLE;
- private String SCOPE_CATLOG;
- private String SCOPE_SCHEMA;
- private String SCOPE_TABLE;
- private short SOURCE_DATA_TYPE;
- private String IS_AUTOINCREMENT;
-
- public String getTABLE_CAT() {
- return TABLE_CAT;
- }
-
- public void setTABLE_CAT(String tABLE_CAT) {
- TABLE_CAT = tABLE_CAT;
- }
-
- public String getTABLE_SCHEM() {
- return TABLE_SCHEM;
- }
-
- public void setTABLE_SCHEM(String tABLE_SCHEM) {
- TABLE_SCHEM = tABLE_SCHEM;
- }
-
- public String getTABLE_NAME() {
- return TABLE_NAME;
- }
-
- public void setTABLE_NAME(String tABLE_NAME) {
- TABLE_NAME = tABLE_NAME;
- }
-
- public String getCOLUMN_NAME() {
- return COLUMN_NAME;
- }
-
- public void setCOLUMN_NAME(String cOLUMN_NAME) {
- COLUMN_NAME = cOLUMN_NAME;
- }
-
- public int getDATA_TYPE() {
- return DATA_TYPE;
- }
-
- public void setDATA_TYPE(int dATA_TYPE) {
- DATA_TYPE = dATA_TYPE;
- }
-
- public String getTYPE_NAME() {
- return TYPE_NAME;
- }
-
- public void setTYPE_NAME(String tYPE_NAME) {
- TYPE_NAME = tYPE_NAME;
- }
-
- public int getCOLUMN_SIZE() {
- return COLUMN_SIZE;
- }
-
- public void setCOLUMN_SIZE(int cOLUMN_SIZE) {
- COLUMN_SIZE = cOLUMN_SIZE;
- }
-
- public int getBUFFER_LENGTH() {
- return BUFFER_LENGTH;
- }
-
- public void setBUFFER_LENGTH(int bUFFER_LENGTH) {
- BUFFER_LENGTH = bUFFER_LENGTH;
- }
-
- public int getDECIMAL_DIGITS() {
- return DECIMAL_DIGITS;
- }
-
- public void setDECIMAL_DIGITS(int dECIMAL_DIGITS) {
- DECIMAL_DIGITS = dECIMAL_DIGITS;
- }
-
- public int getNUM_PREC_RADIX() {
- return NUM_PREC_RADIX;
- }
-
- public void setNUM_PREC_RADIX(int nUM_PREC_RADIX) {
- NUM_PREC_RADIX = nUM_PREC_RADIX;
- }
-
- public int getNULLABLE() {
- return NULLABLE;
- }
-
- public void setNULLABLE(int nULLABLE) {
- NULLABLE = nULLABLE;
- }
-
- public String getREMARKS() {
- return REMARKS;
- }
-
- public void setREMARKS(String rEMARKS) {
- REMARKS = rEMARKS;
- }
-
- public String getCOLUMN_DEF() {
- return COLUMN_DEF;
- }
-
- public void setCOLUMN_DEF(String cOLUMN_DEF) {
- COLUMN_DEF = cOLUMN_DEF;
- }
-
- public int getSQL_DATA_TYPE() {
- return SQL_DATA_TYPE;
- }
-
- public void setSQL_DATA_TYPE(int sQL_DATA_TYPE) {
- SQL_DATA_TYPE = sQL_DATA_TYPE;
- }
-
- public int getSQL_DATETIME_SUB() {
- return SQL_DATETIME_SUB;
- }
-
- public void setSQL_DATETIME_SUB(int sQL_DATETIME_SUB) {
- SQL_DATETIME_SUB = sQL_DATETIME_SUB;
- }
-
- public int getCHAR_OCTET_LENGTH() {
- return CHAR_OCTET_LENGTH;
- }
-
- public void setCHAR_OCTET_LENGTH(int cHAR_OCTET_LENGTH) {
- CHAR_OCTET_LENGTH = cHAR_OCTET_LENGTH;
- }
-
- public int getORDINAL_POSITION() {
- return ORDINAL_POSITION;
- }
-
- public void setORDINAL_POSITION(int oRDINAL_POSITION) {
- ORDINAL_POSITION = oRDINAL_POSITION;
- }
-
- public String getIS_NULLABLE() {
- return IS_NULLABLE;
- }
-
- public void setIS_NULLABLE(String iS_NULLABLE) {
- IS_NULLABLE = iS_NULLABLE;
- }
-
- public String getSCOPE_CATLOG() {
- return SCOPE_CATLOG;
- }
-
- public void setSCOPE_CATLOG(String sCOPE_CATLOG) {
- SCOPE_CATLOG = sCOPE_CATLOG;
- }
-
- public String getSCOPE_SCHEMA() {
- return SCOPE_SCHEMA;
- }
-
- public void setSCOPE_SCHEMA(String sCOPE_SCHEMA) {
- SCOPE_SCHEMA = sCOPE_SCHEMA;
- }
-
- public String getSCOPE_TABLE() {
- return SCOPE_TABLE;
- }
-
- public void setSCOPE_TABLE(String sCOPE_TABLE) {
- SCOPE_TABLE = sCOPE_TABLE;
- }
-
- public short getSOURCE_DATA_TYPE() {
- return SOURCE_DATA_TYPE;
- }
-
- public void setSOURCE_DATA_TYPE(short sOURCE_DATA_TYPE) {
- SOURCE_DATA_TYPE = sOURCE_DATA_TYPE;
- }
-
- public String getIS_AUTOINCREMENT() {
- return IS_AUTOINCREMENT;
- }
-
- public void setIS_AUTOINCREMENT(String iS_AUTOINCREMENT) {
- this.IS_AUTOINCREMENT = iS_AUTOINCREMENT;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
index 9ed24cc..d607ac5 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
@@ -35,8 +35,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * @author xduo
- *
*/
public class DefaultSslProtocolSocketFactory implements SecureProtocolSocketFactory {
/** Log object for this class. */
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
deleted file mode 100644
index c4090ab..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/SQLTypeMap.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc.util;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.ColumnMetaData.Rep;
-
-import org.apache.kylin.jdbc.stub.KylinColumnMetaData;
-
-/**
- * Util class to handle type gap between sql types and java types.
- *
- * @author xduo
- *
- */
-public class SQLTypeMap {
-
- public final static Map<String, ColumnMetaData> schemaMetaTypeMapping = new LinkedHashMap<String, ColumnMetaData>();
-
- public final static Map<String, ColumnMetaData> columnMetaTypeMapping = new LinkedHashMap<String, ColumnMetaData>();
-
- public final static Map<String, ColumnMetaData> tableMetaTypeMapping = new LinkedHashMap<String, ColumnMetaData>();
-
- static {
- schemaMetaTypeMapping.put("TABLE_CAT", KylinColumnMetaData.dummy(0, "TABLE_SCHEM", "TABLE_SCHEM", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- schemaMetaTypeMapping.put("TABLE_SCHEM", KylinColumnMetaData.dummy(1, "TABLE_CATALOG", "TABLE_CATALOG", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
-
- tableMetaTypeMapping.put("TABLE_CAT", KylinColumnMetaData.dummy(0, "TABLE_CAT", "TABLE_CAT", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TABLE_SCHEM", KylinColumnMetaData.dummy(1, "TABLE_SCHEM", "TABLE_SCHEM", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TABLE_NAME", KylinColumnMetaData.dummy(2, "TABLE_NAME", "TABLE_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TABLE_TYPE", KylinColumnMetaData.dummy(3, "TABLE_TYPE", "TABLE_TYPE", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("REMARKS", KylinColumnMetaData.dummy(4, "REMARKS", "REMARKS", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TYPE_CAT", KylinColumnMetaData.dummy(5, "TYPE_CAT", "TYPE_CAT", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TYPE_SCHEM", KylinColumnMetaData.dummy(6, "TYPE_SCHEM", "TYPE_SCHEM", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("TYPE_NAME", KylinColumnMetaData.dummy(7, "TYPE_NAME", "TYPE_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("SELF_REFERENCING_COL_NAME", KylinColumnMetaData.dummy(8, "SELF_REFERENCING_COL_NAME", "SELF_REFERENCING_COL_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- tableMetaTypeMapping.put("REF_GENERATION", KylinColumnMetaData.dummy(9, "REF_GENERATION", "REF_GENERATION", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
-
- columnMetaTypeMapping.put("TABLE_CAT", KylinColumnMetaData.dummy(0, "TABLE_CAT", "TABLE_CAT", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("TABLE_SCHEM", KylinColumnMetaData.dummy(1, "TABLE_SCHEM", "TABLE_SCHEM", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("TABLE_NAME", KylinColumnMetaData.dummy(2, "TABLE_NAME", "TABLE_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("COLUMN_NAME", KylinColumnMetaData.dummy(3, "COLUMN_NAME", "COLUMN_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("DATA_TYPE", KylinColumnMetaData.dummy(4, "DATA_TYPE", "DATA_TYPE", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("TYPE_NAME", KylinColumnMetaData.dummy(5, "TYPE_NAME", "TYPE_NAME", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("COLUMN_SIZE", KylinColumnMetaData.dummy(6, "COLUMN_SIZE", "COLUMN_SIZE", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("BUFFER_LENGTH", KylinColumnMetaData.dummy(7, "BUFFER_LENGTH", "BUFFER_LENGTH", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("DECIMAL_DIGITS", KylinColumnMetaData.dummy(8, "DECIMAL_DIGITS", "DECIMAL_DIGITS", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("NUM_PREC_RADIX", KylinColumnMetaData.dummy(9, "NUM_PREC_RADIX", "NUM_PREC_RADIX", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("NULLABLE", KylinColumnMetaData.dummy(10, "NULLABLE", "NULLABLE", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("REMARKS", KylinColumnMetaData.dummy(11, "REMARKS", "REMARKS", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("COLUMN_DEF", KylinColumnMetaData.dummy(12, "COLUMN_DEF", "COLUMN_DEF", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("SQL_DATA_TYPE", KylinColumnMetaData.dummy(13, "SQL_DATA_TYPE", "SQL_DATA_TYPE", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("SQL_DATETIME_SUB", KylinColumnMetaData.dummy(14, "SQL_DATETIME_SUB", "SQL_DATETIME_SUB", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("CHAR_OCTET_LENGTH", KylinColumnMetaData.dummy(15, "CHAR_OCTET_LENGTH", "CHAR_OCTET_LENGTH", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("ORDINAL_POSITION", KylinColumnMetaData.dummy(16, "ORDINAL_POSITION", "ORDINAL_POSITION", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("IS_NULLABLE", KylinColumnMetaData.dummy(17, "IS_NULLABLE", "IS_NULLABLE", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("SCOPE_CATALOG", KylinColumnMetaData.dummy(18, "SCOPE_CATALOG", "SCOPE_CATALOG", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("SCOPE_TABLE", KylinColumnMetaData.dummy(19, "SCOPE_TABLE", "SCOPE_TABLE", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("SOURCE_DATA_TYPE", KylinColumnMetaData.dummy(20, "SOURCE_DATA_TYPE", "SOURCE_DATA_TYPE", ColumnMetaData.scalar(Types.INTEGER, "integer", Rep.INTEGER), true));
- columnMetaTypeMapping.put("IS_AUTOINCREMENT", KylinColumnMetaData.dummy(21, "IS_AUTOINCREMENT", "IS_AUTOINCREMENT", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- columnMetaTypeMapping.put("IS_GENERATEDCOLUMN", KylinColumnMetaData.dummy(22, "IS_GENERATEDCOLUMN", "IS_GENERATEDCOLUMN", ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- }
-
- @SuppressWarnings("rawtypes")
- public static Class convert(int sqlType) {
- Class result = Object.class;
-
- switch (sqlType) {
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGVARCHAR:
- result = String.class;
- break;
- case Types.NUMERIC:
- case Types.DECIMAL:
- result = BigDecimal.class;
- break;
- case Types.BIT:
- result = Boolean.class;
- break;
- case Types.TINYINT:
- result = Byte.class;
- break;
- case Types.SMALLINT:
- result = Short.class;
- break;
- case Types.INTEGER:
- result = Integer.class;
- break;
- case Types.BIGINT:
- result = Long.class;
- break;
- case Types.REAL:
- case Types.FLOAT:
- case Types.DOUBLE:
- result = Double.class;
- break;
- case Types.BINARY:
- case Types.VARBINARY:
- case Types.LONGVARBINARY:
- result = Byte[].class;
- break;
- case Types.DATE:
- result = Date.class;
- break;
- case Types.TIME:
- result = Time.class;
- break;
- case Types.TIMESTAMP:
- result = Timestamp.class;
- break;
- }
-
- return result;
- }
-
- public static Object wrapObject(String value, int sqlType) {
- if (null == value) {
- return null;
- }
-
- switch (sqlType) {
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGVARCHAR:
- return value;
- case Types.NUMERIC:
- case Types.DECIMAL:
- return new BigDecimal(value);
- case Types.BIT:
- return Boolean.parseBoolean(value);
- case Types.TINYINT:
- return Byte.valueOf(value);
- case Types.SMALLINT:
- return Short.valueOf(value);
- case Types.INTEGER:
- return Integer.parseInt(value);
- case Types.BIGINT:
- return Long.parseLong(value);
- case Types.FLOAT:
- return Float.parseFloat(value);
- case Types.REAL:
- case Types.DOUBLE:
- return Double.parseDouble(value);
- case Types.BINARY:
- case Types.VARBINARY:
- case Types.LONGVARBINARY:
- return value.getBytes();
- case Types.DATE:
- return Date.valueOf(value);
- case Types.TIME:
- return Time.valueOf(value);
- case Types.TIMESTAMP:
- return Timestamp.valueOf(value);
- }
-
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
index 4594e1a..c17efb1 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
@@ -1,174 +1,165 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import static org.junit.Assert.*;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Properties;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Unit test for Driver.
- *
- * @author xduo
- *
- */
-public class DriverTest {
-
- @Test
- public void testStatementWithMockData() throws SQLException {
- Driver driver = new DummyDriver();
-
- Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
-
- ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
- while (tables.next()) {
- for (int i = 0; i < 10; i++) {
- assertEquals("dummy", tables.getString(i + 1));
- }
- }
-
- Statement state = conn.createStatement();
- ResultSet resultSet = state.executeQuery("select * from test_table");
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- assertEquals(12, metadata.getColumnType(1));
- assertEquals("varchar", metadata.getColumnTypeName(1));
- assertEquals(1, metadata.isNullable(1));
-
- while (resultSet.next()) {
- assertEquals("foo", resultSet.getString(1));
- assertEquals("bar", resultSet.getString(2));
- assertEquals("tool", resultSet.getString(3));
- }
- }
-
- @Test
- public void testPreStatementWithMockData() throws SQLException {
- Driver driver = new DummyDriver();
-
- Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
- PreparedStatement state = conn.prepareStatement("select * from test_table where id=?");
- state.setInt(1, 10);
- ResultSet resultSet = state.executeQuery();
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- assertEquals(12, metadata.getColumnType(1));
- assertEquals("varchar", metadata.getColumnTypeName(1));
- assertEquals(1, metadata.isNullable(1));
-
- while (resultSet.next()) {
- assertEquals("foo", resultSet.getString(1));
- assertEquals("bar", resultSet.getString(2));
- assertEquals("tool", resultSet.getString(3));
- }
- }
-
- @Ignore("not maintaining")
- @Test
- public void testWithCubeData() throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException {
- Driver driver = (Driver) Class.forName("org.apache.kylin.kylin.jdbc.Driver").newInstance();
- Properties info = new Properties();
- info.put("user", "");
- info.put("password", "");
- Connection conn = driver.connect("jdbc:kylin://localhost/default", info);
-
- ResultSet catalogs = conn.getMetaData().getCatalogs();
- while (catalogs.next()) {
- System.out.println(catalogs.getString("TABLE_CAT"));
- }
-
- ResultSet schemas = conn.getMetaData().getSchemas();
- while (schemas.next()) {
- System.out.println(schemas.getString(1));
- System.out.println(schemas.getString(2));
- }
-
- ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
- while (tables.next()) {
- String tableName = tables.getString(3);
- assertEquals(tables.getString("TABLE_NAME"), tableName);
- ResultSet columns = conn.getMetaData().getColumns(null, null, tableName, null);
-
- while (columns.next()) {
- System.out.println(columns.getString("COLUMN_NAME"));
- String column = "";
- for (int i = 0; i < 23; i++) {
- column += columns.getString(i + 1) + ", ";
- }
-
- System.out.println("Column in table " + tableName + ": " + column);
- }
- }
-
- for (int j = 0; j < 3; j++) {
- Statement state = conn.createStatement();
- ResultSet resultSet = state.executeQuery("select * from test_kylin_fact");
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- System.out.println("Metadata:");
-
- for (int i = 0; i < metadata.getColumnCount(); i++) {
- String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
- System.out.println(metaStr);
- }
-
- System.out.println("Data:");
- while (resultSet.next()) {
- String dataStr = resultSet.getFloat(1) + " " + resultSet.getInt(2) + " " + resultSet.getInt(3) + " " + resultSet.getLong(4) + " " + resultSet.getDate(5) + " " + resultSet.getString(6);
- System.out.println(dataStr);
- }
- }
- }
-
- @Ignore("not maintaining")
- @Test
- public void testPreStatementWithCubeData() throws SQLException {
- Driver driver = new Driver();
- Properties info = new Properties();
- info.put("user", "");
- info.put("password", "");
- Connection conn = driver.connect("jdbc:kylin://localhost/default", info);
- PreparedStatement state = conn.prepareStatement("select * from test_kylin_fact where seller_id=?");
- state.setLong(1, 10000001);
- ResultSet resultSet = state.executeQuery();
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- System.out.println("Metadata:");
-
- for (int i = 0; i < metadata.getColumnCount(); i++) {
- String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
- System.out.println(metaStr);
- }
-
- System.out.println("Data:");
- while (resultSet.next()) {
- String dataStr = resultSet.getFloat(1) + " " + resultSet.getInt(2) + " " + resultSet.getInt(3) + " " + resultSet.getLong(4) + " " + resultSet.getDate(5) + " " + resultSet.getString(6);
- System.out.println(dataStr);
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import static org.junit.Assert.*;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import org.apache.kylin.jdbc.Driver;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Unit test for Driver.
+ */
+public class DriverTest {
+
+ @Test
+ public void testStatementWithMockData() throws SQLException {
+ Driver driver = new DummyDriver();
+
+ Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
+
+ ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
+ while (tables.next()) {
+ for (int i = 0; i < 4; i++) {
+ assertEquals("dummy", tables.getString(i + 1));
+ }
+ for (int i = 4; i < 10; i++) {
+ assertEquals(null, tables.getString(i + 1));
+ }
+ }
+
+ Statement state = conn.createStatement();
+ ResultSet resultSet = state.executeQuery("select * from test_table");
+
+ ResultSetMetaData metadata = resultSet.getMetaData();
+ assertEquals(12, metadata.getColumnType(1));
+ assertEquals("varchar", metadata.getColumnTypeName(1));
+ assertEquals(1, metadata.isNullable(1));
+
+ while (resultSet.next()) {
+ assertEquals("foo", resultSet.getString(1));
+ assertEquals("bar", resultSet.getString(2));
+ assertEquals("tool", resultSet.getString(3));
+ }
+ }
+
+ @Test
+ public void testPreparedStatementWithMockData() throws SQLException {
+ Driver driver = new DummyDriver();
+
+ Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
+ PreparedStatement state = conn.prepareStatement("select * from test_table where id=?");
+ state.setInt(1, 10);
+ ResultSet resultSet = state.executeQuery();
+
+ ResultSetMetaData metadata = resultSet.getMetaData();
+ assertEquals(12, metadata.getColumnType(1));
+ assertEquals("varchar", metadata.getColumnTypeName(1));
+ assertEquals(1, metadata.isNullable(1));
+
+ while (resultSet.next()) {
+ assertEquals("foo", resultSet.getString(1));
+ assertEquals("bar", resultSet.getString(2));
+ assertEquals("tool", resultSet.getString(3));
+ }
+ }
+
+ @Ignore("require dev sandbox")
+ @Test
+ public void testWithCubeData() throws Exception {
+ Driver driver = new Driver();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
+
+ ResultSet catalogs = conn.getMetaData().getCatalogs();
+ System.out.println("CATALOGS");
+ printResultSet(catalogs);
+
+ ResultSet schemas = conn.getMetaData().getSchemas();
+ System.out.println("SCHEMAS");
+ printResultSet(schemas);
+
+ ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
+ System.out.println("TABLES");
+ printResultSet(tables);
+
+ for (int j = 0; j < 3; j++) {
+ Statement state = conn.createStatement();
+ ResultSet resultSet = state.executeQuery("select * from test_kylin_fact");
+
+ printResultSetMetaData(resultSet);
+ printResultSet(resultSet);
+ }
+ }
+
+ @Ignore("require dev sandbox")
+ @Test
+ public void testPreparedStatementWithCubeData() throws SQLException {
+ Driver driver = new Driver();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
+
+ PreparedStatement state = conn.prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
+ state.setLong(1, 10000001);
+ ResultSet resultSet = state.executeQuery();
+
+ printResultSetMetaData(resultSet);
+ printResultSet(resultSet);
+ }
+
+ private void printResultSet(ResultSet rs) throws SQLException {
+ ResultSetMetaData meta = rs.getMetaData();
+ System.out.println("Data:");
+
+ while (rs.next()) {
+ StringBuilder buf = new StringBuilder();
+ buf.append("[");
+ for (int i = 0; i < meta.getColumnCount(); i++) {
+ if (i > 0)
+ buf.append(", ");
+ buf.append(rs.getString(i + 1));
+ }
+ buf.append("]");
+ System.out.println(buf);
+ }
+ }
+
+ private void printResultSetMetaData(ResultSet rs) throws SQLException {
+ ResultSetMetaData metadata = rs.getMetaData();
+ System.out.println("Metadata:");
+
+ for (int i = 0; i < metadata.getColumnCount(); i++) {
+ String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
+ System.out.println(metaStr);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
index 7af3cb7..0dcd624 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
@@ -1,80 +1,85 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.List;
-
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.ColumnMetaData.Rep;
-import net.hydromatic.linq4j.Enumerator;
-
-import org.apache.kylin.jdbc.stub.ConnectionException;
-import org.apache.kylin.jdbc.stub.DataSet;
-import org.apache.kylin.jdbc.stub.RemoteClient;
-
-/**
- * @author xduo
- *
- */
-public class DummyClient implements RemoteClient {
-
- public DummyClient(KylinConnectionImpl conn) {
- }
-
- @Override
- public void connect() throws ConnectionException {
- }
-
- @Override
- public KylinMetaImpl.MetaProject getMetadata(String project) throws ConnectionException {
- List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- for (int i = 0; i < 10; i++) {
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- }
-
- List<KylinMetaImpl.MetaTable> tables = new ArrayList<KylinMetaImpl.MetaTable>();
- KylinMetaImpl.MetaTable table = new KylinMetaImpl.MetaTable("dummy", "dummy", "dummy", "dummy", "dummy", "dummy", "dummy", "dummy", "dummy", "dummy", new ArrayList<KylinMetaImpl.MetaColumn>());
- tables.add(table);
-
- List<KylinMetaImpl.MetaSchema> schemas = new ArrayList<KylinMetaImpl.MetaSchema>();
- schemas.add(new KylinMetaImpl.MetaSchema("dummy", "dummy", tables));
- List<KylinMetaImpl.MetaCatalog> catalogs = new ArrayList<KylinMetaImpl.MetaCatalog>();
- catalogs.add(new KylinMetaImpl.MetaCatalog("dummy", schemas));
-
- return new KylinMetaImpl.MetaProject(null, catalogs);
- }
-
- @Override
- public DataSet<Object[]> query(AvaticaStatement statement, String sql) {
- List<Object[]> data = new ArrayList<Object[]>();
- Object[] row = new Object[] { "foo", "bar", "tool" };
- data.add(row);
- Enumerator<Object[]> enumerator = new KylinEnumerator<Object[]>(data);
- List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
-
- return new DataSet<Object[]>(meta, enumerator);
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import java.io.IOException;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.kylin.jdbc.IRemoteClient;
+import org.apache.kylin.jdbc.KylinConnection;
+import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog;
+import org.apache.kylin.jdbc.KylinMeta.KMetaColumn;
+import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
+import org.apache.kylin.jdbc.KylinMeta.KMetaSchema;
+import org.apache.kylin.jdbc.KylinMeta.KMetaTable;
+
+/**
+ */
+public class DummyClient implements IRemoteClient {
+
+ public DummyClient(KylinConnection conn) {
+ }
+
+ @Override
+ public void connect() throws IOException {
+ }
+
+ @Override
+ public KMetaProject retrieveMetaData(String project) throws IOException {
+ List<KMetaColumn> columns = new ArrayList<KMetaColumn>();
+
+ KMetaTable table = new KMetaTable("dummy", "dummy", "dummy", "dummy", columns);
+ List<KMetaTable> tables = new ArrayList<KMetaTable>();
+ tables.add(table);
+
+ KMetaSchema schema = new KMetaSchema("dummy", "dummy", tables);
+ List<KMetaSchema> schemas = new ArrayList<KMetaSchema>();
+ schemas.add(schema);
+
+ KMetaCatalog catalog = new KMetaCatalog("dummay", schemas);
+ List<KMetaCatalog> catalogs = new ArrayList<KMetaCatalog>();
+ catalogs.add(catalog);
+
+ return new KMetaProject(project, catalogs);
+ }
+
+ @Override
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
+ List<Object> data = new ArrayList<Object>();
+ Object[] row = new Object[] { "foo", "bar", "tool" };
+ data.add(row);
+
+ List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
+ meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
+ meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
+ meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
+
+ return new QueryResult(meta, data);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
index a5f5383..43c7b75 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
@@ -1,32 +1,32 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-/**
- * @author xduo
- *
- */
-public class DummyDriver extends Driver {
-
- @Override
- protected String getFactoryClassName(JdbcVersion jdbcVersion) {
- return "org.apache.kylin.jdbc.DummyJdbc41Factory";
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import org.apache.kylin.jdbc.Driver;
+
+/**
+ */
+public class DummyDriver extends Driver {
+
+ @Override
+ protected String getFactoryClassName(JdbcVersion jdbcVersion) {
+ return DummyJdbcFactory.class.getName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbc41Factory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbc41Factory.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbc41Factory.java
deleted file mode 100644
index df0b49c..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbc41Factory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import org.apache.kylin.jdbc.stub.RemoteClient;
-
-/**
- * @author xduo
- *
- */
-public class DummyJdbc41Factory extends KylinJdbc41Factory {
-
- // ~ kylin sepcified
- @Override
- public RemoteClient newRemoteClient(KylinConnectionImpl connection) {
- return new DummyClient(connection);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
new file mode 100644
index 0000000..578e594
--- /dev/null
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.jdbc;
+
+import org.apache.kylin.jdbc.IRemoteClient;
+import org.apache.kylin.jdbc.KylinConnection;
+import org.apache.kylin.jdbc.KylinJdbcFactory;
+
+/**
+ */
+public class DummyJdbcFactory extends KylinJdbcFactory {
+
+ public DummyJdbcFactory() {
+ super(4, 1);
+ }
+
+ @Override
+ public IRemoteClient newRemoteClient(KylinConnection conn) {
+ return new DummyClient(conn);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/query/src/main/java/org/apache/kylin/query/QueryCli.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/QueryCli.java b/query/src/main/java/org/apache/kylin/query/QueryCli.java
index fe02b2f..898cb22 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryCli.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryCli.java
@@ -25,13 +25,13 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
+import org.apache.calcite.jdbc.Driver;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
@@ -54,7 +54,7 @@ public class QueryCli {
KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt()));
String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());
- Class.forName("net.hydromatic.optiq.jdbc.Driver");
+ Class.forName(Driver.class.getName());
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(null, config);
Connection conn = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 53795ec..1eea415 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -36,7 +36,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.model.SelectedColumnMeta;
@@ -208,20 +207,24 @@ public class QueryController extends BasicController {
}
checkQueryAuth(sqlResponse);
-
- return sqlResponse;
- } catch (AccessDeniedException ade) {
- // Access exception is bind with each user, it will not be cached
- logger.error("Exception when execute sql", ade);
- throw new ForbiddenException(ade.getLocalizedMessage());
+
} catch (Throwable e) { // calcite may throw AssertError
- SQLResponse exceptionRes = new SQLResponse(null, null, 0, true, e.getMessage());
- Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
- exceptionCache.put(new Element(sqlRequest, exceptionRes));
-
logger.error("Exception when execute sql", e);
- throw new InternalErrorException(QueryUtil.makeErrorMsgUserFriendly(e.getLocalizedMessage()));
+ String errMsg = QueryUtil.makeErrorMsgUserFriendly(e);
+
+ sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
+
+ // Access exception is bind with each user, it will not be cached
+ if ((e instanceof AccessDeniedException) == false) {
+ Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
+ exceptionCache.put(new Element(sqlRequest, sqlResponse));
+ }
}
+
+ if (sqlResponse.getIsException())
+ throw new InternalErrorException(sqlResponse.getExceptionMessage());
+ else
+ return sqlResponse;
}
private SQLResponse searchQueryInCache(SQLRequest sqlRequest) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index f248e95..764dd7b 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -43,6 +43,7 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
+import org.apache.calcite.jdbc.Driver;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,7 +115,7 @@ public abstract class BasicService {
Properties props = new Properties();
props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, String.valueOf(KylinConfig.getInstanceFromEnv().getScanThreshold()));
ds.setConnectionProperties(props);
- ds.setDriverClassName("net.hydromatic.optiq.jdbc.Driver");
+ ds.setDriverClassName(Driver.class.getName());
ds.setUrl("jdbc:calcite:model=" + modelJson.getAbsolutePath());
ret = olapDataSources.putIfAbsent(project, ds);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b28e9f4a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index dd3ffa9..994fe03 100644
--- a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -149,12 +149,22 @@ public class QueryUtil {
return null;
}
- /**
- * adjust error message order
- *
- * @param errorMsg
- * @return
- */
+ public static String makeErrorMsgUserFriendly(Throwable e) {
+ String msg = e.getMessage();
+
+ // pick ParseException error message if possible
+ Throwable cause = e;
+ while (cause != null) {
+ if (cause.getClass().getName().contains("ParseException")) {
+ msg = cause.getMessage();
+ break;
+ }
+ cause = cause.getCause();
+ }
+
+ return makeErrorMsgUserFriendly(msg);
+ }
+
public static String makeErrorMsgUserFriendly(String errorMsg) {
try {
// make one line
[07/50] [abbrv] incubator-kylin git commit: KYLIN-857 backport the
new AggrKey from 0.8 to 0.7
Posted by li...@apache.org.
KYLIN-857 backport the new AggrKey from 0.8 to 0.7
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cc25a042
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cc25a042
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cc25a042
Branch: refs/heads/0.7
Commit: cc25a04243a32bf60cf794e87211661c0dc16080
Parents: 269d922
Author: honma <ho...@ebay.com>
Authored: Thu Jun 25 12:22:15 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Jun 25 12:22:15 2015 +0800
----------------------------------------------------------------------
.../storage/hbase/coprocessor/AggrKey.java | 116 +++++++++++++++++++
.../hbase/coprocessor/AggregationCache.java | 4 +-
.../hbase/coprocessor/CoprocessorProjector.java | 85 ++------------
.../endpoint/EndpointAggregationCache.java | 4 +-
.../hbase/coprocessor/endpoint/IIEndpoint.java | 30 ++---
.../observer/AggregationScanner.java | 3 +-
.../observer/ObserverAggregationCache.java | 25 ++--
.../hbase/coprocessor/RowProjectorTest.java | 14 +--
.../endpoint/EndpointAggregationTest.java | 13 ++-
9 files changed, 166 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
new file mode 100644
index 0000000..7309f61
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
@@ -0,0 +1,116 @@
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.cube.kv.RowConstants;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+
+/**
+ * Created by qianzhou on 4/20/15.
+ */
+public class AggrKey implements Comparable<AggrKey> {
+
+ final byte[] groupByMask;
+ final transient int[] groupByMaskSet;
+ transient int hashcode;
+ byte[] data;
+ int offset;
+
+ AggrKey(byte[] groupByMask) {
+ this.groupByMask = groupByMask;
+ LinkedList<Integer> list = Lists.newLinkedList();
+ for (int i = 0; i < groupByMask.length; i++) {
+ if (groupByMask[i] != 0) {
+ list.add(i);
+ }
+ }
+ groupByMaskSet = new int[list.size()];
+ for (int i = 0; i < list.size(); i++) {
+ groupByMaskSet[i] = list.get(i);
+ }
+ }
+
+ private AggrKey(byte[] groupByMask, int[] groupByMaskSet, byte[] data, int offset, int hashcode) {
+ this.groupByMask = groupByMask;
+ this.groupByMaskSet = groupByMaskSet;
+ this.data = data;
+ this.offset = offset;
+ this.hashcode = hashcode;
+ }
+
+ private int calculateHash()
+ {
+ int hash = 1;
+ for (int i = 0; i < groupByMaskSet.length; i++) {
+ byte t = data[offset + groupByMaskSet[i]];
+ if(t!= RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
+ hash = (31 * hash) + t;
+ }
+ }
+ return hash;
+ }
+
+ public byte[] get() {
+ return data;
+ }
+
+ public int offset() {
+ return offset;
+ }
+
+ public int length() {
+ return groupByMask.length;
+ }
+
+ void set(byte[] data, int offset) {
+ this.data = data;
+ this.offset = offset;
+ this.hashcode = calculateHash();
+ }
+
+ public byte[] getGroupByMask() {
+ return this.groupByMask;
+ }
+
+ public byte[] copyBytes() {
+ return Arrays.copyOfRange(data, offset, offset + length());
+ }
+
+ AggrKey copy() {
+ AggrKey copy = new AggrKey(this.groupByMask, this.groupByMaskSet, copyBytes(), 0, this.hashcode);
+ return copy;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashcode;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ AggrKey other = (AggrKey) obj;
+ return compareTo(other) == 0;
+ }
+
+ @Override
+ public int compareTo(AggrKey o) {
+ int comp = this.length() - o.length();
+ if (comp != 0)
+ return comp;
+
+ for (int i = 0; i < groupByMaskSet.length; i++) {
+ comp = BytesUtil.compareByteUnsigned(this.data[this.offset + groupByMaskSet[i]], o.data[o.offset + groupByMaskSet[i]]);
+ if (comp != 0)
+ return comp;
+ }
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
index 0c48487..483c72c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
@@ -30,7 +30,7 @@ import java.util.SortedMap;
public abstract class AggregationCache {
transient int rowMemBytes;
static final int MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB
- protected final SortedMap<CoprocessorProjector.AggrKey, MeasureAggregator[]> aggBufMap;
+ protected final SortedMap<AggrKey, MeasureAggregator[]> aggBufMap;
public AggregationCache() {
this.aggBufMap = Maps.newTreeMap();
@@ -38,7 +38,7 @@ public abstract class AggregationCache {
public abstract MeasureAggregator[] createBuffer();
- public MeasureAggregator[] getBuffer(CoprocessorProjector.AggrKey aggkey) {
+ public MeasureAggregator[] getBuffer(AggrKey aggkey) {
MeasureAggregator[] aggBuf = aggBufMap.get(aggkey);
if (aggBuf == null) {
aggBuf = createBuffer();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
index b959383..1f0a920 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
@@ -18,11 +18,6 @@
package org.apache.kylin.storage.hbase.coprocessor;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -32,6 +27,11 @@ import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.metadata.model.TblColRef;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
/**
* @author yangli9
*/
@@ -102,10 +102,11 @@ public class CoprocessorProjector {
// ============================================================================
final byte[] groupByMask; // mask out columns that are not needed (by group by)
- final AggrKey aggrKey = new AggrKey();
+ final AggrKey aggrKey;
public CoprocessorProjector(byte[] groupByMask) {
this.groupByMask = groupByMask;
+ this.aggrKey = new AggrKey(this.groupByMask);
}
public AggrKey getAggrKey(List<Cell> rowCells) {
@@ -123,76 +124,4 @@ public class CoprocessorProjector {
aggrKey.set(row, 0);
return aggrKey;
}
-
- public class AggrKey implements Comparable<AggrKey> {
- byte[] data;
- int offset;
-
- public byte[] get() {
- return data;
- }
-
- public int offset() {
- return offset;
- }
-
- public int length() {
- return groupByMask.length;
- }
-
- void set(byte[] data, int offset) {
- this.data = data;
- this.offset = offset;
- }
-
- public AggrKey copy() {
- AggrKey copy = new AggrKey();
- copy.set(new byte[length()], 0);
- System.arraycopy(this.data, this.offset, copy.data, copy.offset, length());
- return copy;
- }
-
- @Override
- public int hashCode() {
- int hash = 1;
- for (int i = 0, j = offset, n = length(); i < n; i++, j++) {
- if (groupByMask[i] != 0)
- hash = (31 * hash) + (int) data[j];
- }
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- AggrKey other = (AggrKey) obj;
- if (this.length() != other.length())
- return false;
-
- return compareTo(other) == 0;
- }
-
- @Override
- public int compareTo(AggrKey o) {
- int comp = this.length() - o.length();
- if (comp != 0)
- return comp;
-
- int n = this.length();
- for (int i = 0, j = offset, k = o.offset; i < n; i++, j++, k++) {
- if (groupByMask[i] != 0) {
- comp = BytesUtil.compareByteUnsigned(this.data[j], o.data[k]);
- if (comp != 0)
- return comp;
- }
- }
- return 0;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
index b6722dc..ac6fa70 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
@@ -19,8 +19,8 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import java.util.Map;
import java.util.Set;
@@ -41,7 +41,7 @@ public class EndpointAggregationCache extends AggregationCache {
return this.aggregators.createBuffer();
}
- public Set<Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]>> getAllEntries() {
+ public Set<Map.Entry<AggrKey, MeasureAggregator[]>> getAllEntries() {
return aggBufMap.entrySet();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index 77bed3b..2e85eac 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -22,22 +22,7 @@ import com.google.protobuf.ByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
-
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
-
import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -49,6 +34,15 @@ import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
+import org.apache.kylin.storage.hbase.coprocessor.*;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
import java.io.IOException;
import java.util.Iterator;
@@ -137,7 +131,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
Iterator<RawTableRecord> iterator = slice.iterateWithBitmap(result);
while (iterator.hasNext()) {
byte[] data = iterator.next().getBytes();
- CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(data);
+ AggrKey aggKey = projector.getAggrKey(data);
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, data);
aggCache.checkMemoryUsage();
@@ -145,8 +139,8 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
}
byte[] metricBuffer = new byte[CoprocessorConstants.METRIC_SERIALIZE_BUFFER_SIZE];
- for (Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
- CoprocessorProjector.AggrKey aggrKey = entry.getKey();
+ for (Map.Entry<AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
+ AggrKey aggrKey = entry.getKey();
IIProtos.IIResponse.IIRow.Builder rowBuilder = IIProtos.IIResponse.IIRow.newBuilder().setColumns(ByteString.copyFrom(aggrKey.get(), aggrKey.offset(), aggrKey.length()));
int length = aggregators.serializeMetricValues(entry.getValue(), metricBuffer);
rowBuilder.setMeasures(ByteString.copyFrom(metricBuffer, 0, length));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
index 85f3f6e..158b08f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
@@ -80,7 +81,7 @@ public class AggregationScanner implements RegionScanner {
continue;
if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR.ordinal()) {
- CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(results);
+ AggrKey aggKey = projector.getAggrKey(results);
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, results);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java
index 6dce289..28feda7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java
@@ -18,22 +18,21 @@
package org.apache.kylin.storage.hbase.coprocessor.observer;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-
import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
+import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
/**
* @author yangli9
@@ -59,7 +58,7 @@ public class ObserverAggregationCache extends AggregationCache {
private class AggregationRegionScanner implements RegionScanner {
private final RegionScanner innerScanner;
- private final Iterator<Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]>> iterator;
+ private final Iterator<Entry<AggrKey, MeasureAggregator[]>> iterator;
public AggregationRegionScanner(RegionScanner innerScanner) {
this.innerScanner = innerScanner;
@@ -71,7 +70,7 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner next()");
boolean hasMore = false;
if (iterator.hasNext()) {
- Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry = iterator.next();
+ Entry<AggrKey, MeasureAggregator[]> entry = iterator.next();
makeCells(entry, results);
hasMore = iterator.hasNext();
}
@@ -79,12 +78,12 @@ public class ObserverAggregationCache extends AggregationCache {
return hasMore;
}
- private void makeCells(Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry, List<Cell> results) {
+ private void makeCells(Entry<AggrKey, MeasureAggregator[]> entry, List<Cell> results) {
byte[][] families = aggregators.getHColFamilies();
byte[][] qualifiers = aggregators.getHColQualifiers();
int nHCols = aggregators.getHColsNum();
- CoprocessorProjector.AggrKey rowKey = entry.getKey();
+ AggrKey rowKey = entry.getKey();
MeasureAggregator[] aggBuf = entry.getValue();
ByteBuffer[] rowValues = aggregators.getHColValues(aggBuf);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
index 6a22d74..f597c7e 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/RowProjectorTest.java
@@ -18,19 +18,17 @@
package org.apache.kylin.storage.hbase.coprocessor;
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
+import com.google.common.collect.Lists;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.kylin.common.util.Bytes;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector.AggrKey;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc25a042/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
index 24291ac..87c6445 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
@@ -18,23 +18,22 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-import static org.junit.Assert.assertEquals;
-
+import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.hadoop.io.LongWritable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -42,6 +41,8 @@ import org.junit.Test;
import java.io.IOException;
import java.util.*;
+import static org.junit.Assert.assertEquals;
+
/**
* Created by Hongbin Ma(Binmahone) on 11/27/14.
*
@@ -173,7 +174,7 @@ public class EndpointAggregationTest extends LocalFileMetadataTestCase {
for (int i = 0; i < tableData.size(); ++i) {
byte[] data = tableData.get(i).getBytes();
- CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(data);
+ AggrKey aggKey = projector.getAggrKey(data);
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, data);
aggCache.checkMemoryUsage();
@@ -183,7 +184,7 @@ public class EndpointAggregationTest extends LocalFileMetadataTestCase {
long sumTotal = 0;
long minTotal = 0;
- for (Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
+ for (Map.Entry<AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
sumTotal += ((LongWritable) entry.getValue()[0].getState()).get();
minTotal += ((LongWritable) entry.getValue()[1].getState()).get();
[10/50] [abbrv] incubator-kylin git commit: minor,
add configuration to disable run remote coprocessor
Posted by li...@apache.org.
minor, add configuration to disable run remote coprocessor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/faeb8a96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/faeb8a96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/faeb8a96
Branch: refs/heads/0.7
Commit: faeb8a9689e39f983599f36e5681fff040345e19
Parents: 613106c
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 25 14:03:58 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 25 14:04:42 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 27 +++--------------
.../kylin/common/debug/BackdoorToggles.java | 32 ++++++++++++++++++--
.../kylin/storage/hbase/HBaseKeyRange.java | 11 +++----
.../coprocessor/observer/ObserverEnabler.java | 15 +++++----
4 files changed, 48 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index dde3ba5..5ae18a6 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -40,8 +40,6 @@ public class KylinConfig {
public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
- public static final String PROP_SCAN_THRESHOLD = "PROP_SCAN_THRESHOLD";
-
public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
@@ -92,8 +90,6 @@ public class KylinConfig {
public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
-
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
@@ -345,10 +341,6 @@ public class KylinConfig {
System.setProperty(COPROCESSOR_LOCAL_JAR, path);
}
- public int getCoprocessorScanBitsThreshold() {
- return Integer.parseInt(getOptional(COPROCESSOR_SCAN_BITS_THRESHOLD, "32"));
- }
-
public double getDefaultHadoopJobReducerInputMB() {
return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
}
@@ -428,10 +420,14 @@ public class KylinConfig {
public int getDictionaryMaxCardinality() {
return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
}
-
+
public int getScanThreshold() {
return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
}
+
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
+ }
public Long getQueryDurationCacheThreshold() {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
@@ -445,10 +441,6 @@ public class KylinConfig {
return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
}
- public int getConcurrentScanThreadCount() {
- return Integer.parseInt(this.getOptional("kylin.query.scan.thread.count", "40"));
- }
-
public boolean isQueryCacheEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
}
@@ -637,15 +629,6 @@ public class KylinConfig {
kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v);
}
- /**
- * return -1 if there is no setting
- *
- * @return
- */
- public int getPropScanThreshold() {
- return kylinConfig.getInt(PROP_SCAN_THRESHOLD, -1);
- }
-
public String getProperty(String key, String defaultValue) {
return kylinConfig.getString(key, defaultValue);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 315543f..5760798 100644
--- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -6,13 +6,25 @@ import java.util.Map;
*/
public class BackdoorToggles {
- private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal();
+ private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
public static void setToggles(Map<String, String> toggles) {
_backdoorToggles.set(toggles);
}
- public static String getToggle(String key) {
+ public static String getObserverBehavior() {
+ return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ }
+
+ public static boolean getDisableFuzzyKey() {
+ return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
+ }
+
+ public static boolean getRunLocalCoprocessor() {
+ return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
+ }
+
+ private static String getString(String key) {
Map<String, String> toggles = _backdoorToggles.get();
if (toggles == null) {
return null;
@@ -20,6 +32,10 @@ public class BackdoorToggles {
return toggles.get(key);
}
}
+
+ private static boolean getBoolean(String key) {
+ return "true".equals(getString(key));
+ }
public static void cleanToggles() {
_backdoorToggles.remove();
@@ -50,4 +66,16 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+
+ /**
+ * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
+ *
+ example:
+
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
+ }
+
+ */
+ public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index 278c50c..9b05dc7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -168,13 +168,10 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
- //debug/profiling purpose
- String toggle;
- if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_DISABLE_FUZZY_KEY)) != null) {
- if (Boolean.valueOf(toggle)) {
- logger.info("The execution of this query will not use fuzzy key");
- return result;
- }
+ // debug/profiling purpose
+ if (BackdoorToggles.getDisableFuzzyKey()) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
}
FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faeb8a96/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index 6acc40e..84688a1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -20,10 +20,12 @@ package org.apache.kylin.storage.hbase.coprocessor.observer;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
@@ -53,7 +55,6 @@ public class ObserverEnabler {
private static final Logger logger = LoggerFactory.getLogger(ObserverEnabler.class);
static final String FORCE_COPROCESSOR = "forceObserver";
- static final boolean DEBUG_LOCAL_COPROCESSOR = false;
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
@@ -67,17 +68,19 @@ public class ObserverEnabler {
CoprocessorFilter filter = CoprocessorFilter.fromFilter(segment, tupleFiler);
CoprocessorProjector projector = CoprocessorProjector.makeForObserver(segment, cuboid, groupBy);
ObserverAggregators aggrs = ObserverAggregators.fromValueDecoders(rowValueDecoders);
+
+ boolean localCoprocessor = KylinConfig.getInstanceFromEnv().getQueryRunLocalCoprocessor() || BackdoorToggles.getRunLocalCoprocessor();
- if (DEBUG_LOCAL_COPROCESSOR) {
+ if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
return new ResultScannerAdapter(aggrScanner);
} else {
- //debug/profiling purpose
- String toggle;
- if ((toggle = BackdoorToggles.getToggle(BackdoorToggles.DEBUG_TOGGLE_OBSERVER_BEHAVIOR)) == null) {
- toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString();//default behavior
+ // debug/profiling purpose
+ String toggle = BackdoorToggles.getObserverBehavior();
+ if (toggle == null) {
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
[48/50] [abbrv] incubator-kylin git commit: KYLIN-875 minor refactor,
add 'addTask()' to interface ChainedExecutable
Posted by li...@apache.org.
KYLIN-875 minor refactor, add 'addTask()' to interface ChainedExecutable
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/44c09a26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/44c09a26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/44c09a26
Branch: refs/heads/0.7
Commit: 44c09a269e642bbc511fba197a8ae83db86cc3a8
Parents: 340a175
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Jul 8 11:15:49 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Jul 8 11:16:20 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/execution/ChainedExecutable.java | 4 +++-
.../java/org/apache/kylin/job/manager/ExecutableManager.java | 8 ++++----
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/44c09a26/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
index 3eddf1d..8ceb23e 100644
--- a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
@@ -25,6 +25,8 @@ import java.util.List;
*/
public interface ChainedExecutable extends Executable {
- List<? extends Executable> getTasks();
+ List<? extends AbstractExecutable> getTasks();
+
+ void addTask(AbstractExecutable executable);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/44c09a26/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index a2b310b..05f8c8e 100644
--- a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -280,9 +280,9 @@ public class ExecutableManager {
result.setUuid(executable.getId());
result.setType(executable.getClass().getName());
result.setParams(executable.getParams());
- if (executable instanceof DefaultChainedExecutable) {
+ if (executable instanceof ChainedExecutable) {
List<ExecutablePO> tasks = Lists.newArrayList();
- for (AbstractExecutable task : ((DefaultChainedExecutable) executable).getTasks()) {
+ for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
tasks.add(parse(task));
}
result.setTasks(tasks);
@@ -304,9 +304,9 @@ public class ExecutableManager {
result.setParams(executablePO.getParams());
List<ExecutablePO> tasks = executablePO.getTasks();
if (tasks != null && !tasks.isEmpty()) {
- Preconditions.checkArgument(result instanceof DefaultChainedExecutable);
+ Preconditions.checkArgument(result instanceof ChainedExecutable);
for (ExecutablePO subTask: tasks) {
- ((DefaultChainedExecutable) result).addTask(parseTo(subTask));
+ ((ChainedExecutable) result).addTask(parseTo(subTask));
}
}
return result;
[50/50] [abbrv] incubator-kylin git commit: hotfix: query duration
for deciding query cache is not properly set
Posted by li...@apache.org.
hotfix: query duration for deciding query cache is not properly set
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/83d05800
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/83d05800
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/83d05800
Branch: refs/heads/0.7
Commit: 83d05800d07a7895dfe62ccff7289772a2e0bfc2
Parents: 7362154
Author: honma <ho...@ebay.com>
Authored: Thu Jul 9 13:18:01 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Jul 9 13:18:11 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/QueryController.java | 29 ++++++++------------
1 file changed, 12 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/83d05800/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 1eea415..7e07cc1 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -18,19 +18,10 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import javax.servlet.http.HttpServletResponse;
-
+import com.codahale.metrics.annotation.Timed;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
-
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -53,16 +44,18 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import com.codahale.metrics.annotation.Timed;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
/**
* Handle query requests.
@@ -197,11 +190,13 @@ public class QueryController extends BasicController {
SQLResponse sqlResponse = searchQueryInCache(sqlRequest);
try {
if (null == sqlResponse) {
+ long start = System.currentTimeMillis();
sqlResponse = queryService.query(sqlRequest);
+ long duration = System.currentTimeMillis() - start;
long durationThreshold = KylinConfig.getInstanceFromEnv().getQueryDurationCacheThreshold();
long scancountThreshold = KylinConfig.getInstanceFromEnv().getQueryScanCountCacheThreshold();
- if (!sqlResponse.getIsException() && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
+ if (!sqlResponse.getIsException() && (duration > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
}
[19/50] [abbrv] incubator-kylin git commit: minor,
refactor QueryController move dup code into doQuery()
Posted by li...@apache.org.
minor, refactor QueryController move dup code into doQuery()
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e7fcc20d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e7fcc20d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e7fcc20d
Branch: refs/heads/0.7
Commit: e7fcc20d8ab6f824eb6b88b8f8fa133e284dc34f
Parents: 231fcc4
Author: Yang Li <li...@apache.org>
Authored: Tue Jun 30 18:27:15 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Jun 30 18:27:38 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/QueryController.java | 65 ++++++++++----------
.../org/apache/kylin/rest/util/QueryUtil.java | 6 +-
2 files changed, 36 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e7fcc20d/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 7cd14ff..53795ec 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -18,10 +18,19 @@
package org.apache.kylin.rest.controller;
-import com.codahale.metrics.annotation.Timed;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletResponse;
+
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
+
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -45,18 +54,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
+import com.codahale.metrics.annotation.Timed;
/**
* Handle query requests.
@@ -81,35 +88,15 @@ public class QueryController extends BasicController {
@ResponseBody
@Timed(name = "query")
public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
- initDebugToggles(sqlRequest);
-
- long startTimestamp = System.currentTimeMillis();
- SQLResponse response = doQuery(sqlRequest);
- response.setDuration(System.currentTimeMillis() - startTimestamp);
- queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
-
- cleanupDebugToggles();
-
- return response;
+ return doQuery(sqlRequest);
}
+ // TODO should be just "prepare" a statement, get back expected ResultSetMetaData
@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
@Timed(name = "query")
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
- long startTimestamp = System.currentTimeMillis();
-
- SQLResponse response = doQuery(sqlRequest);
- response.setDuration(System.currentTimeMillis() - startTimestamp);
-
- queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
-
- if (response.getIsException()) {
- String errorMsg = response.getExceptionMessage();
- throw new InternalErrorException(QueryUtil.makeErrorMsgUserFriendly(errorMsg));
- }
-
- return response;
+ return doQuery(sqlRequest);
}
@RequestMapping(value = "/saved_queries", method = RequestMethod.POST)
@@ -181,6 +168,18 @@ public class QueryController extends BasicController {
}
private SQLResponse doQuery(SQLRequest sqlRequest) {
+ initDebugToggles(sqlRequest);
+
+ long startTimestamp = System.currentTimeMillis();
+ SQLResponse response = doQueryInternal(sqlRequest);
+ response.setDuration(System.currentTimeMillis() - startTimestamp);
+ queryService.logQuery(sqlRequest, response, new Date(startTimestamp), new Date(System.currentTimeMillis()));
+
+ cleanupDebugToggles();
+ return response;
+ }
+
+ private SQLResponse doQueryInternal(SQLRequest sqlRequest) {
String sql = sqlRequest.getSql();
String project = sqlRequest.getProject();
logger.info("Using project: " + project);
@@ -193,7 +192,7 @@ public class QueryController extends BasicController {
if (sql.toLowerCase().contains("select") == false) {
logger.debug("Directly return expection as not supported");
- throw new InternalErrorException(QueryUtil.makeErrorMsgUserFriendly("Not Supported SQL."));
+ throw new InternalErrorException("Not Supported SQL.");
}
SQLResponse sqlResponse = searchQueryInCache(sqlRequest);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e7fcc20d/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 7de4415..dd3ffa9 100644
--- a/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -157,8 +157,10 @@ public class QueryUtil {
*/
public static String makeErrorMsgUserFriendly(String errorMsg) {
try {
- errorMsg = errorMsg.replaceAll("\\s", " ");// replace all invisible
- // characters
+ // make one line
+ errorMsg = errorMsg.replaceAll("\\s", " ");
+
+ // move cause to be ahead of sql, calcite creates the message pattern below
Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)");
Matcher matcher = pattern.matcher(errorMsg);
if (matcher.find()) {
[28/50] [abbrv] incubator-kylin git commit: KYLIN-873,
fix cancel button does not work when [resume][discard] job
Posted by li...@apache.org.
KYLIN-873, fix cancel button does not work when [resume][discard] job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/725bde52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/725bde52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/725bde52
Branch: refs/heads/0.7
Commit: 725bde52771e64688aa8ca54ee31c86609d8b45b
Parents: cc1ab7b
Author: jiazhong <ji...@ebay.com>
Authored: Mon Jul 6 11:23:30 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Jul 6 11:25:07 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/access.js | 39 +++++++--------
webapp/app/js/controllers/job.js | 82 +++++++++++++++++---------------
2 files changed, 63 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/725bde52/webapp/app/js/controllers/access.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/access.js b/webapp/app/js/controllers/access.js
index 4cfd833..57dd0f9 100644
--- a/webapp/app/js/controllers/access.js
+++ b/webapp/app/js/controllers/access.js
@@ -102,25 +102,26 @@ KylinApp.controller('AccessCtrl', function ($scope, AccessService, MessageServic
confirmButtonColor: '#DD6B55',
confirmButtonText: "Yes",
closeOnConfirm: true
- }, function () {
- var revokeRequst = {
- type: type,
- uuid: entity.uuid,
- accessEntryId: access.id
- };
- AccessService.revoke(revokeRequst, function (accessEntities) {
- entity.accessEntities = accessEntities.accessEntryResponseList;
- SweetAlert.swal('Success!', 'The access has been revoked.', 'success');
- }, function (e) {
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- });
-
+ }, function (isConfirm) {
+ if (isConfirm) {
+ var revokeRequst = {
+ type: type,
+ uuid: entity.uuid,
+ accessEntryId: access.id
+ };
+ AccessService.revoke(revokeRequst, function (accessEntities) {
+ entity.accessEntities = accessEntities.accessEntryResponseList;
+ SweetAlert.swal('Success!', 'The access has been revoked.', 'success');
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
+ }
});
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/725bde52/webapp/app/js/controllers/job.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js
index 45961b7..812316e 100644
--- a/webapp/app/js/controllers/job.js
+++ b/webapp/app/js/controllers/job.js
@@ -111,25 +111,27 @@ KylinApp
confirmButtonColor: '#DD6B55',
confirmButtonText: "Yes",
closeOnConfirm: true
- }, function () {
- loadingRequest.show();
- JobService.resume({jobId: job.uuid}, {}, function (job) {
- loadingRequest.hide();
- JobList.jobs[job.uuid] = job;
- if (angular.isDefined($scope.state.selectedJob)) {
- $scope.state.selectedJob = JobList.jobs[$scope.state.selectedJob.uuid];
- }
- SweetAlert.swal('Success!', 'Job has been resumed successfully!', 'success');
- }, function (e) {
- loadingRequest.hide();
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- });
+ }, function (isConfirm) {
+ if (isConfirm) {
+ loadingRequest.show();
+ JobService.resume({jobId: job.uuid}, {}, function (job) {
+ loadingRequest.hide();
+ JobList.jobs[job.uuid] = job;
+ if (angular.isDefined($scope.state.selectedJob)) {
+ $scope.state.selectedJob = JobList.jobs[$scope.state.selectedJob.uuid];
+ }
+ SweetAlert.swal('Success!', 'Job has been resumed successfully!', 'success');
+ }, function (e) {
+ loadingRequest.hide();
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
+ }
});
}
@@ -143,28 +145,30 @@ KylinApp
confirmButtonColor: '#DD6B55',
confirmButtonText: "Yes",
closeOnConfirm: true
- }, function () {
- loadingRequest.show();
- JobService.cancel({jobId: job.uuid}, {}, function (job) {
- loadingRequest.hide();
- $scope.safeApply(function () {
- JobList.jobs[job.uuid] = job;
- if (angular.isDefined($scope.state.selectedJob)) {
- $scope.state.selectedJob = JobList.jobs[$scope.state.selectedJob.uuid];
+ }, function (isConfirm) {
+ if (isConfirm) {
+ loadingRequest.show();
+ JobService.cancel({jobId: job.uuid}, {}, function (job) {
+ loadingRequest.hide();
+ $scope.safeApply(function () {
+ JobList.jobs[job.uuid] = job;
+ if (angular.isDefined($scope.state.selectedJob)) {
+ $scope.state.selectedJob = JobList.jobs[$scope.state.selectedJob.uuid];
+ }
+
+ });
+ SweetAlert.swal('Success!', 'Job has been discarded successfully!', 'success');
+ }, function (e) {
+ loadingRequest.hide();
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
}
-
});
- SweetAlert.swal('Success!', 'Job has been discarded successfully!', 'success');
- }, function (e) {
- loadingRequest.hide();
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- });
+ }
});
}
[26/50] [abbrv] incubator-kylin git commit: kylin-792 ,
add sh to run kylin performance monitor
Posted by li...@apache.org.
kylin-792 ,add sh to run kylin performance monitor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5cc130d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5cc130d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5cc130d3
Branch: refs/heads/0.7
Commit: 5cc130d3c77b4eca50aa4110413180079277ae84
Parents: a935c7b
Author: jiazhong <ji...@ebay.com>
Authored: Fri Jul 3 17:42:50 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Fri Jul 3 17:42:50 2015 +0800
----------------------------------------------------------------------
bin/performance-monitor.sh | 38 ++++++++++++++++++++
.../org/apache/kylin/monitor/QueryParser.java | 2 +-
2 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5cc130d3/bin/performance-monitor.sh
----------------------------------------------------------------------
diff --git a/bin/performance-monitor.sh b/bin/performance-monitor.sh
new file mode 100644
index 0000000..9004c02
--- /dev/null
+++ b/bin/performance-monitor.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ if [ -z "$KYLIN_HOME" ]
+ then
+ echo 'please make sure KYLIN_HOME has been set'
+ exit 1
+ else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+ fi
+
+ dir=$(dirname ${0})
+
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+
+ export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+ _monitorjar=`ls ${KYLIN_HOME}/lib |grep kylin-monitor`
+
+ hbase \
+ org.apache.hadoop.util.RunJar $KYLIN_HOME/lib/${_monitorjar} org.apache.kylin.monitor.Client
+ exit 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5cc130d3/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
----------------------------------------------------------------------
diff --git a/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java b/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
index fc8a403..8b910a8 100644
--- a/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
+++ b/monitor/src/main/java/org/apache/kylin/monitor/QueryParser.java
@@ -235,7 +235,7 @@ public class QueryParser {
*/
public FileSystem getHdfsFileSystem() throws IOException {
Configuration conf = new Configuration();
- conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
+// conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
FileSystem fs = null;
try {
fs = FileSystem.get(conf);
[42/50] [abbrv] incubator-kylin git commit: KYLIN-867 a bug fix in
LookupTableEnumerator
Posted by li...@apache.org.
KYLIN-867 a bug fix in LookupTableEnumerator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab3b07df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab3b07df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab3b07df
Branch: refs/heads/0.7
Commit: ab3b07dfb77f40f99cf5cbb143345464772bd586
Parents: 4f742cd
Author: shaofengshi <sh...@apache.org>
Authored: Sun Jul 5 13:53:33 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 6 16:19:17 2015 +0800
----------------------------------------------------------------------
.../query/enumerator/LookupTableEnumerator.java | 19 ++++++-
.../kylin/storage/hybrid/HybridInstance.java | 55 ++++++++++++++++----
2 files changed, 63 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab3b07df/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 3ae19dd..733e8fe 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -29,9 +29,13 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPTable;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import org.apache.kylin.storage.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
@@ -42,10 +46,23 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
private final Object[] current;
private Iterator<String[]> iterator;
+ private final static Logger logger = LoggerFactory.getLogger(LookupTableEnumerator.class);
public LookupTableEnumerator(OLAPContext olapContext) {
//TODO: assuming LookupTableEnumerator is handled by a cube
- CubeInstance cube = (CubeInstance) olapContext.realization;
+ CubeInstance cube = null;
+
+ if (olapContext.realization instanceof CubeInstance)
+ cube = (CubeInstance) olapContext.realization;
+ else if (olapContext.realization instanceof HybridInstance) {
+ final HybridInstance hybridInstance = (HybridInstance)olapContext.realization;
+ final IRealization latestRealization = hybridInstance.getLatestRealization();
+ if (latestRealization instanceof CubeInstance) {
+ cube = (CubeInstance) latestRealization;
+ } else {
+ throw new IllegalStateException();
+ }
+ }
String lookupTableName = olapContext.firstTableScan.getTableName();
DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab3b07df/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 68984cd..e7071f9 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -14,9 +14,10 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.LinkedHashSet;
-import java.util.List;
+import java.util.*;
/**
*/
@@ -46,6 +47,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private String projectName;
private boolean initiated = false;
+ private final static Logger logger = LoggerFactory.getLogger(HybridInstance.class);
public List<RealizationEntry> getRealizationEntries() {
return realizationEntries;
@@ -63,12 +65,18 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
throw new IllegalArgumentException();
RealizationRegistry registry = RealizationRegistry.getInstance(config);
- realizations = new IRealization[realizationEntries.size()];
+ List<IRealization> realizationList = Lists.newArrayList();
for (int i = 0; i < realizationEntries.size(); i++) {
IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization());
- if (realization == null)
- throw new IllegalArgumentException("Realization '" + realizationEntries.get(i) + "' not loaded.");
- realizations[i] = realization;
+ if (realization == null) {
+ logger.error("Realization '" + realization.getName() + " is not found, remove from Hybrid '" + this.getName() + "'");
+ continue;
+ }
+ if (realization.isReady() == false) {
+ logger.error("Realization '" + realization.getName() + " is disabled, remove from Hybrid '" + this.getName() + "'");
+ continue;
+ }
+ realizationList.add(realization);
}
LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>();
@@ -76,10 +84,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>();
dateRangeStart = 0;
dateRangeEnd = Long.MAX_VALUE;
- for (IRealization realization : realizations) {
- if (realization.isReady() == false)
- continue;
-
+ for (IRealization realization : realizationList) {
columns.addAll(realization.getAllColumns());
dimensions.addAll(realization.getAllDimensions());
measures.addAll(realization.getMeasures());
@@ -98,6 +103,29 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
allColumns = Lists.newArrayList(columns);
allMeasures = Lists.newArrayList(measures);
+ Collections.sort(realizationList, new Comparator<IRealization>() {
+ @Override
+ public int compare(IRealization o1, IRealization o2) {
+
+ long i1 = o1.getDateRangeStart();
+ long i2 = o2.getDateRangeStart();
+ long comp = i1 - i2;
+ if (comp != 0) {
+ return comp > 0 ? 1 : -1;
+ }
+
+ i1 = o1.getDateRangeEnd();
+ i2 = o2.getDateRangeEnd();
+ comp = i1 - i2;
+ if (comp != 0) {
+ return comp > 0 ? 1 : -1;
+ }
+
+ return 0;
+ }
+ });
+
+ this.realizations = realizationList.toArray(new IRealization[realizationList.size()]);
initiated = true;
}
}
@@ -208,4 +236,11 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
public void setCost(int cost) {
this.cost = cost;
}
+
+ public IRealization getLatestRealization() {
+ if (realizations.length > 0) {
+ return realizations[realizations.length - 1];
+ }
+ return null;
+ }
}
[21/50] [abbrv] incubator-kylin git commit: KYLIN-868 add a metadata
backup/restore script in bin folder
Posted by li...@apache.org.
KYLIN-868 add a metadata backup/restore script in bin folder
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cbcfd59d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cbcfd59d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cbcfd59d
Branch: refs/heads/0.7
Commit: cbcfd59df85ecd417583c4d4471acc7ed2f21071
Parents: 0975139
Author: honma <ho...@ebay.com>
Authored: Wed Jul 1 15:53:10 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jul 1 15:53:10 2015 +0800
----------------------------------------------------------------------
bin/metastore.sh | 37 +++++++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cbcfd59d/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
new file mode 100644
index 0000000..f08de31
--- /dev/null
+++ b/bin/metastore.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
+_jobjar=`ls ${KYLIN_HOME}/lib |grep kylin-job`
+_fulljobjar="${KYLIN_HOME}/lib/${_jobjar}"
+
+if [ $1 == "backup" ]
+then
+
+ mkdir -p ${KYLIN_HOME}/meta_backups
+
+ _now=$(date +"%Y_%m_%d_%H_%M_%S")
+ _file="${KYLIN_HOME}/meta_backups/meta_${_now}"
+ echo "Starting backup to ${_file}"
+ mkdir -p ${_file}
+
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy ${KYLIN_HOME}/conf/kylin.properties ${_file}
+ echo "metadata store backed up to ${_file}"
+
+elif [ $1 == "restore" ]
+then
+
+ _file=$2
+ echo "Starting restoring $_file"
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy $_file ${KYLIN_HOME}/conf/kylin.properties
+
+elif [ $1 == "reset" ]
+then
+
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool reset
+
+else
+ echo "usage: metastore.sh backup or metastore.sh restore PATH_TO_LOCAL_META"
+ exit 1
+fi
[41/50] [abbrv] incubator-kylin git commit: KYLIN-867 initial commit,
backport Hybrid model to 0.7
Posted by li...@apache.org.
KYLIN-867 initial commit, backport Hybrid model to 0.7
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9491e92e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9491e92e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9491e92e
Branch: refs/heads/0.7
Commit: 9491e92ef2a0fda1ba34ee94a41359fd94da0376
Parents: a2bab5f
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jul 2 19:04:25 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 6 16:19:17 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/lock/JobLock.java | 9 +
.../apache/kylin/common/lock/MockJobLock.java | 15 ++
.../kylin/common/lock/ZookeeperJobLock.java | 83 ++++++++
.../kylin/common/persistence/ResourceStore.java | 1 +
.../kylin/common/restclient/Broadcaster.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 39 +++-
.../apache/kylin/cube/cuboid/CuboidTest.java | 1 -
.../hybrid/test_kylin_hybrid_inner_join.json | 14 ++
.../hybrid/test_kylin_hybrid_left_join.json | 14 ++
.../invertedindex_desc/test_kylin_ii_desc.json | 7 +-
.../localmeta/project/default.json | 6 -
.../localmeta/project/onlyinner.json | 4 -
.../localmeta/project/onlyleft.json | 4 -
.../apache/kylin/invertedindex/IIInstance.java | 36 ++++
.../kylin/invertedindex/index/SliceBuilder.java | 3 -
.../kylin/invertedindex/model/IIDesc.java | 27 ++-
.../kylin/invertedindex/model/IIDimension.java | 9 +-
.../invertedindex/IIDescManagerTest.java | 2 +-
.../invertedindex/IIInstanceTest.java | 4 +-
.../invertedindex/InvertedIndexLocalTest.java | 6 +-
.../java/org/apache/kylin/job/Scheduler.java | 3 +-
.../job/impl/threadpool/DefaultScheduler.java | 49 +----
.../kylin/job/BuildCubeWithEngineTest.java | 5 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 5 +-
.../job/impl/threadpool/BaseSchedulerTest.java | 3 +-
.../apache/kylin/metadata/model/TblColRef.java | 4 +
.../metadata/realization/IRealization.java | 8 +
.../realization/RealizationRegistry.java | 1 +
.../metadata/realization/RealizationType.java | 2 +-
.../metadata/realization/SQLDigestUtil.java | 114 +++++++++++
.../metadata/tuple/CompoundTupleIterator.java | 45 +++++
.../kylin/metadata/tuple/ITupleIterator.java | 8 +-
.../apache/kylin/query/routing/RoutingRule.java | 4 +-
.../AdjustForWeeklyMatchedRealization.java | 25 ++-
.../RoutingRules/RealizationPriorityRule.java | 5 +-
.../RoutingRules/RealizationSortRule.java | 66 +++++++
.../apache/kylin/query/test/IIQueryTest.java | 3 +
.../kylin/rest/controller/JobController.java | 3 +-
.../kylin/storage/StorageEngineFactory.java | 6 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 5 +
.../hbase/SerializedHBaseTupleIterator.java | 5 +
.../endpoint/EndpointTupleIterator.java | 10 +
.../kylin/storage/hybrid/HybridInstance.java | 188 +++++++++++++++++++
.../kylin/storage/hybrid/HybridManager.java | 127 +++++++++++++
.../storage/hybrid/HybridStorageEngine.java | 69 +++++++
.../endpoint/EndpointAggregationTest.java | 1 +
.../endpoint/TableRecordInfoTest.java | 1 +
.../kylin/storage/hybrid/HybridManagerTest.java | 45 +++++
48 files changed, 979 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/JobLock.java b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
new file mode 100644
index 0000000..7fdb64c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
@@ -0,0 +1,9 @@
+package org.apache.kylin.common.lock;
+
+/**
+ */
+public interface JobLock {
+ boolean lock();
+
+ void unlock();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
new file mode 100644
index 0000000..230d4d8
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.common.lock;
+
+/**
+ */
+public class MockJobLock implements JobLock {
+ @Override
+ public boolean lock() {
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
new file mode 100644
index 0000000..603894c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
@@ -0,0 +1,83 @@
+package org.apache.kylin.common.lock;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ */
+public class ZookeeperJobLock implements JobLock {
+ private Logger logger = LoggerFactory.getLogger(ZookeeperJobLock.class);
+
+ private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+
+ private String scheduleID;
+ private InterProcessMutex sharedLock;
+ private CuratorFramework zkClient;
+
+ @Override
+ public boolean lock() {
+ this.scheduleID = schedulerId();
+ String ZKConnectString = getZKConnectString();
+ if (StringUtils.isEmpty(ZKConnectString)) {
+ throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
+ }
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
+ this.zkClient.start();
+ this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
+ boolean hasLock = false;
+ try {
+ hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ logger.warn("error acquire lock", e);
+ }
+ if (!hasLock) {
+ logger.warn("fail to acquire lock, scheduler has not been started");
+ zkClient.close();
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ releaseLock();
+ }
+
+ private String getZKConnectString() {
+ Configuration conf = HadoopUtil.newHBaseConfiguration(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ }
+
+ private void releaseLock() {
+ try {
+ if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
+ // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
+ if (zkClient.checkExists().forPath(scheduleID) != null) {
+ zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(scheduleID);
+ }
+ }
+ } catch (Exception e) {
+ logger.error("error release lock:" + scheduleID);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String schedulerId() {
+ return ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 7021915..a49706d 100644
--- a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -51,6 +51,7 @@ abstract public class ResourceStore {
public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
public static final String TABLE_RESOURCE_ROOT = "/table";
+ public static final String HYBRID_RESOURCE_ROOT = "/hybrid";
private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
index b636e89..68e68c3 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
@@ -139,7 +139,7 @@ public class Broadcaster {
}
public static enum TYPE {
- ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model");
+ ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model"), HYBRID("hybrid");
private String text;
private TYPE(String text) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index d978887..898925a 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -327,7 +327,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
@Override
public int getCost(SQLDigest digest) {
- return 0;
+ return cost;
}
@Override
@@ -348,4 +348,41 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
this.projectName = projectName;
}
+
+ @Override
+ public long getDateRangeStart() {
+ List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long startTime = Long.MAX_VALUE;
+ for (CubeSegment seg : readySegs) {
+ if (seg.getDateRangeStart() < startTime)
+ startTime = seg.getDateRangeStart();
+ }
+
+ return startTime;
+ }
+
+ @Override
+ public long getDateRangeEnd() {
+
+ List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long endTime = 0;
+ for (CubeSegment seg : readySegs) {
+ if (seg.getDateRangeEnd() > endTime)
+ endTime = seg.getDateRangeEnd();
+ }
+
+ return endTime;
+ }
+
+ @Override
+ public String getModelName() {
+ return this.getDescriptor().getModelName();
+ }
+
+ @Override
+ public List<TblColRef> getAllDimensions() {
+ return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 9f7cbc6..4914448 100644
--- a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -173,7 +173,6 @@ public class CuboidTest extends LocalFileMetadataTestCase {
assertEquals(toLong("111111111"), cuboid.getId());
}
- //@Test
public void testII() {
CubeDesc cube = getTestKylinCubeII();
assertEquals(toLong("111111111"), Cuboid.getBaseCuboidId(cube));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
new file mode 100644
index 0000000..5582841
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
@@ -0,0 +1,14 @@
+{
+ "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
+ "name": "test_kylin_hybrid_inner_join",
+ "historyRealization": {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_empty"
+ },
+ "realTimeRealization": {
+ "type": "INVERTED_INDEX",
+ "realization": "test_kylin_ii"
+ },
+ "last_modified": 1420016227424,
+ "create_time": null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
new file mode 100644
index 0000000..37f59d6
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
@@ -0,0 +1,14 @@
+{
+ "uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
+ "name": "test_kylin_hybrid_left_join",
+ "historyRealization": {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_left_join_empty"
+ },
+ "realTimeRealization": {
+ "type": "INVERTED_INDEX",
+ "realization": "test_kylin_ii"
+ },
+ "last_modified": 1420016227424,
+ "create_time": null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
index 962d887..a5ee5e5 100644
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
+++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
@@ -1,12 +1,11 @@
{
"uuid": "74bf87b5-c7b5-4420-a12a-07f6b37b3187",
-
"last_modified": 0,
"name": "test_kylin_ii_desc",
"fact_table": "default.test_kylin_fact",
"model_name": "test_kylin_ii_model_desc",
"timestamp_dimension": "cal_dt",
- "bitmap_dimensions": [
+ "value_dimensions": [
{
"table": "default.test_kylin_fact",
"columns": [
@@ -43,9 +42,7 @@
"seller_type_cd",
"seller_type_desc"
]
- }
- ],
- "value_dimensions": [
+ },
{
"table": "edw.test_cal_dt",
"columns": [
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index c6b8b74..853a10a 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -27,11 +27,5 @@
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii"
}
- ],
- "cubes": [
- "test_kylin_cube_with_slr_empty",
- "test_kylin_cube_without_slr_empty",
- "test_kylin_cube_with_slr_left_join_empty",
- "test_kylin_cube_without_slr_left_join_empty"
]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/project/onlyinner.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/onlyinner.json b/examples/test_case_data/localmeta/project/onlyinner.json
index b98d762..a3ab387 100644
--- a/examples/test_case_data/localmeta/project/onlyinner.json
+++ b/examples/test_case_data/localmeta/project/onlyinner.json
@@ -12,9 +12,5 @@
"type": "CUBE",
"realization": "test_kylin_cube_without_slr_empty"
}
- ],
- "cubes": [
- "test_kylin_cube_with_slr_empty",
- "test_kylin_cube_without_slr_empty"
]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/examples/test_case_data/localmeta/project/onlyleft.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/onlyleft.json b/examples/test_case_data/localmeta/project/onlyleft.json
index 9569eea..014ff4b 100644
--- a/examples/test_case_data/localmeta/project/onlyleft.json
+++ b/examples/test_case_data/localmeta/project/onlyleft.json
@@ -12,9 +12,5 @@
"type": "CUBE",
"realization": "test_kylin_cube_without_slr_left_join_empty"
}
- ],
- "cubes": [
- "test_kylin_cube_with_slr_left_join_empty",
- "test_kylin_cube_without_slr_left_join_empty"
]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index eeb06ed..9c46fbb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -310,4 +310,40 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
this.projectName = projectName;
}
+ @Override
+ public long getDateRangeStart() {
+ List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long startTime = Long.MAX_VALUE;
+ for (IISegment seg : readySegs) {
+ if (seg.getDateRangeStart() < startTime)
+ startTime = seg.getDateRangeStart();
+ }
+
+ return startTime;
+ }
+
+ @Override
+ public long getDateRangeEnd() {
+
+ List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long endTime = 0;
+ for (IISegment seg : readySegs) {
+ if (seg.getDateRangeEnd() > endTime)
+ endTime = seg.getDateRangeEnd();
+ }
+
+ return endTime;
+ }
+
+ @Override
+ public String getModelName() {
+ return this.getDescriptor().getModelName();
+ }
+
+ @Override
+ public List<TblColRef> getAllDimensions() {
+ return getDescriptor().listAllDimensions();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index e217df0..72050a1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,9 +61,6 @@ public class SliceBuilder {
// reset for next slice
nRecords = 0;
containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getBitmapColumns()) {
- containers[i] = new BitMapContainer(info.getDigest(), i);
- }
for (int i : info.getDescriptor().getValueColumns()) {
containers[i] = new CompressedValueContainer(info.getDigest(), i,
nRecordsCap);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
index 6db6082..ab1438b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
@@ -71,8 +71,6 @@ public class IIDesc extends RootPersistentEntity {
private String modelName;
@JsonProperty("timestamp_dimension")
private String timestampDimension;
- @JsonProperty("bitmap_dimensions")
- private List<IIDimension> bitmapDimensions;
@JsonProperty("value_dimensions")
private List<IIDimension> valueDimensions;
@JsonProperty("metrics")
@@ -87,8 +85,8 @@ public class IIDesc extends RootPersistentEntity {
// computed
private List<TableDesc> allTables = Lists.newArrayList();
private List<TblColRef> allColumns = Lists.newArrayList();
+ private List<TblColRef> allDimensions = Lists.newArrayList();
private int tsCol;
- private int[] bitmapCols;
private int[] valueCols;
private int[] metricsCols;
private BitSet metricsColSet;
@@ -111,7 +109,6 @@ public class IIDesc extends RootPersistentEntity {
timestampDimension = timestampDimension.toUpperCase();
// capitalize
- IIDimension.capicalizeStrings(bitmapDimensions);
IIDimension.capicalizeStrings(valueDimensions);
StringUtil.toUpperCaseArray(metricNames, metricNames);
@@ -119,11 +116,13 @@ public class IIDesc extends RootPersistentEntity {
HashSet<String> allTableNames = Sets.newHashSet();
measureDescs = Lists.newArrayList();
measureDescs.add(makeCountMeasure());
- for (IIDimension iiDimension : Iterables.concat(bitmapDimensions, valueDimensions)) {
+ for (IIDimension iiDimension : valueDimensions) {
TableDesc tableDesc = this.getTableDesc(iiDimension.getTable());
for (String column : iiDimension.getColumns()) {
ColumnDesc columnDesc = tableDesc.findColumnByName(column);
- allColumns.add(new TblColRef(columnDesc));
+ TblColRef tcr = new TblColRef(columnDesc);
+ allColumns.add(tcr);
+ allDimensions.add(tcr);
measureDescs.add(makeHLLMeasure(columnDesc, null));
}
@@ -146,15 +145,11 @@ public class IIDesc extends RootPersistentEntity {
}
// indexing for each type of columns
- bitmapCols = new int[IIDimension.getColumnCount(bitmapDimensions)];
valueCols = new int[IIDimension.getColumnCount(valueDimensions)];
metricsCols = new int[metricNames.length];
metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
int totalIndex = 0;
- for (int i = 0; i < bitmapCols.length; ++i, ++totalIndex) {
- bitmapCols[i] = totalIndex;
- }
for (int i = 0; i < valueCols.length; ++i, ++totalIndex) {
valueCols[i] = totalIndex;
}
@@ -174,7 +169,7 @@ public class IIDesc extends RootPersistentEntity {
}
}
if (tsCol < 0)
- throw new RuntimeException("timestamp_dimension is not in bitmapDimensions or valueDimensions");
+ throw new RuntimeException("timestamp_dimension is not in valueDimensions");
}
private TableDesc getTableDesc(String tableName) {
@@ -262,6 +257,10 @@ public class IIDesc extends RootPersistentEntity {
return allColumns;
}
+ public List<TblColRef> listAllDimensions() {
+ return allDimensions;
+ }
+
public TblColRef findColumnRef(String table, String column) {
ColumnDesc columnDesc = this.getTableDesc(table).findColumnByName(column);
return new TblColRef(columnDesc);
@@ -299,10 +298,6 @@ public class IIDesc extends RootPersistentEntity {
return tsCol;
}
- public int[] getBitmapColumns() {
- return bitmapCols;
- }
-
public int[] getValueColumns() {
return valueCols;
}
@@ -359,7 +354,7 @@ public class IIDesc extends RootPersistentEntity {
try {
md = MessageDigest.getInstance("MD5");
StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTableName()).append("|").append(timestampDimension).append("|").append(JsonUtil.writeValueAsString(this.bitmapDimensions)).append("|").append(JsonUtil.writeValueAsString(valueDimensions)).append("|").append(JsonUtil.writeValueAsString(this.metricNames)).append("|").append(sharding).append("|").append(sliceSize);
+ sigString.append(this.name).append("|").append(this.getFactTableName()).append("|").append(timestampDimension).append("|").append("|").append(JsonUtil.writeValueAsString(valueDimensions)).append("|").append(JsonUtil.writeValueAsString(this.metricNames)).append("|").append(sharding).append("|").append(sliceSize);
byte[] signature = md.digest(sigString.toString().getBytes());
return new String(Base64.encodeBase64(signature));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
index ee8b6b9..d2d377d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
@@ -18,6 +18,7 @@
package org.apache.kylin.invertedindex.model;
+import java.util.Arrays;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -52,9 +53,11 @@ public class IIDimension {
public static void capicalizeStrings(List<IIDimension> dimensions) {
- for (IIDimension iiDimension : dimensions) {
- iiDimension.setTable(iiDimension.getTable().toUpperCase());
- StringUtil.toUpperCaseArray(iiDimension.getColumns(), iiDimension.getColumns());
+ if (dimensions != null) {
+ for (IIDimension iiDimension : dimensions) {
+ iiDimension.setTable(iiDimension.getTable().toUpperCase());
+ StringUtil.toUpperCaseArray(iiDimension.getColumns(), iiDimension.getColumns());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
index 5782a82..bd01b2c 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
@@ -54,7 +54,7 @@ public class IIDescManagerTest extends LocalFileMetadataTestCase {
public void testCRUD() throws IOException {
IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
- String newDescName = "Copy of " + TEST_II_DESC_NAME;
+ String newDescName = "Copy_of_" + TEST_II_DESC_NAME;
try {
IIDesc testRecord = mgr.getIIDesc(newDescName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
index 792d674..4eb11be 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
@@ -70,7 +70,7 @@ public class IIInstanceTest extends LocalFileMetadataTestCase {
IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_desc");
- IIInstance ii = IIInstance.create("new ii", "default", iiDesc);
+ IIInstance ii = IIInstance.create("new_ii", "default", iiDesc);
IIManager iiMgr = IIManager.getInstance(getTestConfig());
@@ -78,7 +78,7 @@ public class IIInstanceTest extends LocalFileMetadataTestCase {
iiMgr.createII(ii);
- Assert.assertNotNull(iiMgr.getII("new ii"));
+ Assert.assertNotNull(iiMgr.getII("new_ii"));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index 42de647..45eb9b5 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Pair;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@@ -62,6 +63,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
}
@Test
+ @Ignore
public void testBitMapContainer() {
// create container
BitMapContainer container = new BitMapContainer(info.getDigest(), 0);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/job/src/main/java/org/apache/kylin/job/Scheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/Scheduler.java b/job/src/main/java/org/apache/kylin/job/Scheduler.java
index 120b86f..87ce072 100644
--- a/job/src/main/java/org/apache/kylin/job/Scheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/Scheduler.java
@@ -18,6 +18,7 @@
package org.apache.kylin.job;
+import org.apache.kylin.common.lock.JobLock;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.SchedulerException;
import org.apache.kylin.job.execution.Executable;
@@ -27,7 +28,7 @@ import org.apache.kylin.job.execution.Executable;
*/
public interface Scheduler<T extends Executable> {
- void init(JobEngineConfig jobEngineConfig) throws SchedulerException;
+ void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException;
void shutdown() throws SchedulerException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 6483035..d69e74d 100644
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -26,6 +26,7 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.kylin.common.lock.JobLock;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.RetryPolicy;
@@ -143,29 +144,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
}
}
- private void releaseLock() {
- try {
- if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
- // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
- if (zkClient.checkExists().forPath(schedulerId()) != null) {
- zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(schedulerId());
- }
- }
- } catch (Exception e) {
- logger.error("error release lock:" + schedulerId());
- throw new RuntimeException(e);
- }
- }
-
- private String schedulerId() {
- return ZOOKEEPER_LOCK_PATH + "/" + jobEngineConfig.getConfig().getMetadataUrlPrefix();
- }
-
- private String getZKConnectString(JobEngineConfig context) {
- Configuration conf = HadoopUtil.newHBaseConfiguration(context.getConfig().getStorageUrl());
- return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- }
-
public static DefaultScheduler getInstance() {
return INSTANCE;
}
@@ -182,33 +160,16 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
}
@Override
- public synchronized void init(JobEngineConfig jobEngineConfig) throws SchedulerException {
+ public synchronized void init(JobEngineConfig jobEngineConfig, final JobLock jobLock) throws SchedulerException {
if (!initialized) {
initialized = true;
} else {
return;
}
- String ZKConnectString = getZKConnectString(jobEngineConfig);
- if (StringUtils.isEmpty(ZKConnectString)) {
- throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
- }
this.jobEngineConfig = jobEngineConfig;
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
- this.zkClient.start();
- this.sharedLock = new InterProcessMutex(zkClient, schedulerId());
- boolean hasLock = false;
- try {
- hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
- } catch (Exception e) {
- logger.warn("error acquire lock", e);
- }
- if (!hasLock) {
- logger.warn("fail to acquire lock, scheduler has not been started");
- zkClient.close();
- return;
- }
+ jobLock.lock();
+
executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
//load all executable, set them to a consistent status
fetcherPool = Executors.newScheduledThreadPool(1);
@@ -228,6 +189,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
logger.debug("Closing zk connection");
try {
shutdown();
+ jobLock.unlock();
} catch (SchedulerException e) {
logger.error("error shutdown scheduler", e);
}
@@ -243,7 +205,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
public void shutdown() throws SchedulerException {
fetcherPool.shutdown();
jobPool.shutdown();
- releaseLock();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index a5a1156..7fd4246 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -96,7 +97,7 @@ public class BuildCubeWithEngineTest {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
jobService = ExecutableManager.getInstance(kylinConfig);
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
@@ -212,7 +213,7 @@ public class BuildCubeWithEngineTest {
// this cube's start date is 0, end date is 20501112000000
long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
+ long date2 = f.parse("2022-01-01").getTime();
// this cube doesn't support incremental build, always do full build
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 194a8d7..6dacc08 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -35,6 +35,7 @@ import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
import org.junit.After;
import org.junit.Before;
@@ -108,7 +109,7 @@ public class BuildIIWithEngineTest {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
jobService = ExecutableManager.getInstance(kylinConfig);
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
@@ -134,7 +135,7 @@ public class BuildIIWithEngineTest {
ii.setStatus(RealizationStatusEnum.READY);
iiManager.updateII(ii);
}
- backup();
+ // backup();
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 9da0f73..29ef7ad 100644
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.job.impl.threadpool;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -87,7 +88,7 @@ public abstract class BaseSchedulerTest extends HBaseMetadataTestCase {
setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"), 10);
jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 76c3911..ca932e9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -67,6 +67,10 @@ public class TblColRef {
this.column = column;
}
+ public ColumnDesc getColumnDesc() {
+ return column;
+ }
+
public ColumnDesc getColumn() {
return column;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index d1571ce..43f5e9a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -51,6 +51,8 @@ public interface IRealization {
public List<TblColRef> getAllColumns();
+ public List<TblColRef> getAllDimensions();
+
public List<MeasureDesc> getMeasures();
public boolean isReady();
@@ -63,4 +65,10 @@ public interface IRealization {
public void setProjectName(String prjName);
+ public long getDateRangeStart();
+
+ public long getDateRangeEnd();
+
+ public String getModelName();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
index c05497e..2c7ba42 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
@@ -111,6 +111,7 @@ public class RealizationRegistry {
IRealizationProvider p = providers.get(type);
if (p == null) {
logger.warn("No provider for realization type " + type);
+ return null;
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
index 64e3ab7..1b5dbc5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
@@ -24,5 +24,5 @@ package org.apache.kylin.metadata.realization;
//TODO: change to String for plugin
public enum RealizationType {
- CUBE, INVERTED_INDEX
+ CUBE, INVERTED_INDEX, HYBRID
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
new file mode 100644
index 0000000..72e3b10
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -0,0 +1,114 @@
+package org.apache.kylin.metadata.realization;
+
+import com.google.common.base.Function;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ */
+public class SQLDigestUtil {
+
+ public static <F, T> T appendTsFilterToExecute(SQLDigest sqlDigest, TblColRef partitionColRef, Range<Long> tsRange, Function<F, T> action) {
+
+ // add the boundary condition to query real-time
+ TupleFilter originalFilter = sqlDigest.filter;
+ sqlDigest.filter = createFilterForRealtime(originalFilter, partitionColRef, tsRange);
+
+ boolean addFilterColumn = false, addAllColumn = false;
+
+ if (!sqlDigest.filterColumns.contains(partitionColRef)) {
+ sqlDigest.filterColumns.add(partitionColRef);
+ addFilterColumn = true;
+ }
+
+ if (!sqlDigest.allColumns.contains(partitionColRef)) {
+ sqlDigest.allColumns.add(partitionColRef);
+ addAllColumn = true;
+ }
+
+ T ret = action.apply(null);
+
+ // restore the sqlDigest
+ sqlDigest.filter = originalFilter;
+
+ if (addFilterColumn)
+ sqlDigest.filterColumns.remove(partitionColRef);
+
+ if (addAllColumn)
+ sqlDigest.allColumns.remove(partitionColRef);
+
+ return ret;
+ }
+
+ //ts column type differentiate
+ private static String formatTimeStr(DataType type, long ts) {
+ String ret;
+ if (type == DataType.getInstance("date")) {
+ ret = DateFormat.formatToDateStr(ts);
+ } else if (type == DataType.getInstance("long")) {
+ ret = String.valueOf(ts);
+ } else {
+ throw new IllegalArgumentException("Illegal type for partition column " + type);
+ }
+ return ret;
+ }
+
+ private static TupleFilter createFilterForRealtime(TupleFilter originFilter, TblColRef partitionColRef, Range<Long> tsRange) {
+ DataType type = partitionColRef.getColumnDesc().getType();
+
+ String startTimeStr, endTimeStr;
+ CompareTupleFilter startFilter = null, endFilter = null;
+ if (tsRange.hasLowerBound()) {
+ startTimeStr = formatTimeStr(type, tsRange.lowerEndpoint());
+ if (tsRange.lowerBoundType() == BoundType.CLOSED) {
+ startFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
+ } else {
+ startFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ }
+ ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
+ ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(startTimeStr);
+ startFilter.addChild(columnTupleFilter);
+ startFilter.addChild(constantTupleFilter);
+ }
+
+ if (tsRange.hasUpperBound()) {
+ endTimeStr = formatTimeStr(type, tsRange.upperEndpoint());
+ if (tsRange.upperBoundType() == BoundType.CLOSED) {
+ endFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ } else {
+ endFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LT);
+ }
+ ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
+ ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(endTimeStr);
+ endFilter.addChild(columnTupleFilter);
+ endFilter.addChild(constantTupleFilter);
+ }
+
+ if (originFilter == null) {
+ if (endFilter == null) {
+ return startFilter;
+ }
+ if (startFilter == null) {
+ return endFilter;
+ }
+ }
+
+ LogicalTupleFilter logicalTupleFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+
+ if (originFilter != null) {
+ logicalTupleFilter.addChild(originFilter);
+ }
+ if (startFilter != null) {
+ logicalTupleFilter.addChild(startFilter);
+ }
+ if (endFilter != null) {
+ logicalTupleFilter.addChild(endFilter);
+ }
+
+ return logicalTupleFilter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
new file mode 100644
index 0000000..f5d8dd6
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.metadata.tuple;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ */
+public class CompoundTupleIterator implements ITupleIterator {
+ private static final Logger logger = LoggerFactory.getLogger(CompoundTupleIterator.class);
+ private List<ITupleIterator> backends;
+ private Iterator<ITuple> compoundIterator;
+
+ public CompoundTupleIterator(List<ITupleIterator> backends) {
+ Preconditions.checkArgument(backends != null && backends.size() != 0, "backends not exists");
+ this.backends = backends;
+ this.compoundIterator = Iterators.concat(backends.iterator());
+ }
+
+ @Override
+ public void close() {
+ for (ITupleIterator i : backends) {
+ i.close();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.compoundIterator.hasNext();
+ }
+
+ @Override
+ public ITuple next() {
+ return this.compoundIterator.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
index 6966751..5235387 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
@@ -18,10 +18,12 @@
package org.apache.kylin.metadata.tuple;
+import java.util.Iterator;
+
/**
* @author xjiang
*/
-public interface ITupleIterator {
+public interface ITupleIterator extends Iterator<ITuple> {
public static final ITupleIterator EMPTY_TUPLE_ITERATOR = new ITupleIterator() {
@Override
public boolean hasNext() {
@@ -34,6 +36,10 @@ public interface ITupleIterator {
}
@Override
+ public void remove() {
+ }
+
+ @Override
public void close() {
}
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index aaad516..04972fa 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -41,10 +41,8 @@ public abstract class RoutingRule {
//1. simple query use II prior to cube
//2. exact match prior to week match
static {
- rules.add(new RealizationPriorityRule());
rules.add(new RemoveUncapableRealizationsRule());
- rules.add(new SimpleQueryMoreColumnsCubeFirstRule());
- rules.add(new CubesSortRule());
+ rules.add(new RealizationSortRule());
rules.add(new AdjustForWeeklyMatchedRealization());//this rule might modify olapcontext content, better put it at last
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
index 465d162..3402556 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,12 +18,6 @@
package org.apache.kylin.query.routing.RoutingRules;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.kylin.cube.CubeCapabilityChecker;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
@@ -34,9 +28,15 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RoutingRule;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
*/
public class AdjustForWeeklyMatchedRealization extends RoutingRule {
private static final Logger logger = LoggerFactory.getLogger(AdjustForWeeklyMatchedRealization.class);
@@ -46,6 +46,13 @@ public class AdjustForWeeklyMatchedRealization extends RoutingRule {
if (realizations.size() > 0) {
IRealization first = realizations.get(0);
+ if (first instanceof HybridInstance) {
+ HybridInstance hybrid = (HybridInstance) first;
+
+ if (hybrid.getHistoryRealizationInstance() instanceof CubeInstance)
+ first = hybrid.getHistoryRealizationInstance();
+ }
+
if (first instanceof CubeInstance) {
CubeInstance cube = (CubeInstance) first;
adjustOLAPContextIfNecessary(cube, olapContext);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
index 6a67140..7425d62 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
@@ -33,8 +33,9 @@ public class RealizationPriorityRule extends RoutingRule {
static Map<RealizationType, Integer> priorities = Maps.newHashMap();
static {
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.INVERTED_INDEX, 1);
+ priorities.put(RealizationType.HYBRID, 2);
+ priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
}
public static void setPriorities(Map<RealizationType, Integer> priorities) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
new file mode 100644
index 0000000..f9a250e
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing.RoutingRules;
+
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.routing.RoutingRule;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ */
+public class RealizationSortRule extends RoutingRule {
+ @Override
+ public void apply(List<IRealization> realizations, final OLAPContext olapContext) {
+
+ // sort cube candidates, 0) the priority 1) the cost indicator, 2) the lesser header
+ // columns the better, 3) the lesser body columns the better 4) the larger date range the better
+
+ Collections.sort(realizations, new Comparator<IRealization>() {
+ @Override
+ public int compare(IRealization o1, IRealization o2) {
+ int i1 = RealizationPriorityRule.priorities.get(o1.getType());
+ int i2 = RealizationPriorityRule.priorities.get(o2.getType());
+ int comp = i1 - i2;
+ if (comp != 0) {
+ return comp;
+ }
+
+ comp = o1.getCost(olapContext.getSQLDigest()) - o2.getCost(olapContext.getSQLDigest());
+ if (comp != 0) {
+ return comp;
+ }
+
+ if (o1 instanceof HybridInstance)
+ return -1;
+ else if (o2 instanceof HybridInstance)
+ return 1;
+
+ return 0;
+ }
+ });
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index f5b42d5..b58e439 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -24,6 +24,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Maps;
@@ -40,6 +41,7 @@ public class IIQueryTest extends KylinQueryTest {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.INVERTED_INDEX, 0);
priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.HYBRID, 1);
RealizationPriorityRule.setPriorities(priorities);
}
@@ -51,6 +53,7 @@ public class IIQueryTest extends KylinQueryTest {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.INVERTED_INDEX, 1);
priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.HYBRID, 0);
RealizationPriorityRule.setPriorities(priorities);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 2f2d9cf..0e0a0b1 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.TimeZone;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +83,7 @@ public class JobController extends BasicController implements InitializingBean {
public void run() {
try {
DefaultScheduler scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
logger.error("scheduler has not been started");
System.exit(1);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
index 5eb4ea3..b81bad2 100644
--- a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
+++ b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
@@ -24,6 +24,8 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.storage.hbase.CubeStorageEngine;
import org.apache.kylin.storage.hbase.InvertedIndexStorageEngine;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridStorageEngine;
/**
* @author xjiang
@@ -33,8 +35,10 @@ public class StorageEngineFactory {
public static IStorageEngine getStorageEngine(IRealization realization) {
if (realization.getType() == RealizationType.INVERTED_INDEX) {
return new InvertedIndexStorageEngine((IIInstance) realization);
- } else {
+ } else if (realization.getType() == RealizationType.CUBE) {
return new CubeStorageEngine((CubeInstance) realization);
+ } else {
+ return new HybridStorageEngine((HybridInstance) realization);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 268c098..ad35972 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -180,6 +180,11 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
return this.tuple;
}
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
private void scanNextRange() {
if (this.rangeIterator.hasNext()) {
closeScanner();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
index 8a22579..97ba981 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
@@ -122,6 +122,11 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
@Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void close() {
context.setTotalScanCount(scanCount);
segmentIterator.close();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index f865dff..9e5c804 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -192,6 +192,11 @@ public class EndpointTupleIterator implements ITupleIterator {
}
@Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void close() {
IOUtils.closeQuietly(table);
logger.info("Closed after " + rowsInAllMetric + " rows are fetched");
@@ -310,6 +315,11 @@ public class EndpointTupleIterator implements ITupleIterator {
}
@Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void close() {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
new file mode 100644
index 0000000..1694719
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -0,0 +1,188 @@
+package org.apache.kylin.storage.hybrid;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metadata.realization.SQLDigest;
+
+import java.util.List;
+
+/**
+ */
+
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class HybridInstance extends RootPersistentEntity implements IRealization {
+
+ @JsonIgnore
+ private KylinConfig config;
+
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("historyRealization")
+ private RealizationEntry historyRealization;
+
+ @JsonProperty("realTimeRealization")
+ private RealizationEntry realTimeRealization;
+
+ private IRealization historyRealizationInstance;
+ private IRealization realTimeRealizationInstance;
+ private String projectName;
+
+ public void init() {
+ RealizationRegistry registry = RealizationRegistry.getInstance(config);
+ historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization());
+ realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization());
+
+ if (historyRealizationInstance == null) {
+ throw new IllegalArgumentException("Didn't find realization '" + historyRealization.getType() + "'" + " with name '" + historyRealization.getRealization() + "' in '" + name + "'");
+ }
+
+ if (realTimeRealizationInstance == null) {
+ throw new IllegalArgumentException("Didn't find realization '" + realTimeRealization.getType() + "'" + " with name '" + realTimeRealization.getRealization() + "' in '" + name + "'");
+ }
+
+ }
+
+ @Override
+ public boolean isCapable(SQLDigest digest) {
+ return getHistoryRealizationInstance().isCapable(digest) && getRealTimeRealizationInstance().isCapable(digest);
+ }
+
+ @Override
+ public int getCost(SQLDigest digest) {
+ return historyRealizationInstance.getCost(digest);
+ //return Math.min(historyRealizationInstance.getCost(digest), realTimeRealizationInstance.getCost(digest));
+ }
+
+ @Override
+ public RealizationType getType() {
+ return RealizationType.HYBRID;
+ }
+
+ @Override
+ public String getFactTable() {
+ return getHistoryRealizationInstance().getFactTable();
+ }
+
+ @Override
+ public List<TblColRef> getAllColumns() {
+ return getHistoryRealizationInstance().getAllColumns();
+ }
+
+ @Override
+ public List<MeasureDesc> getMeasures() {
+ return getHistoryRealizationInstance().getMeasures();
+ }
+
+ @Override
+ public boolean isReady() {
+ return historyRealizationInstance.isReady() || realTimeRealizationInstance.isReady();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return getCanonicalName();
+ }
+
+ @Override
+ public String getCanonicalName() {
+ return getType() + "[name=" + name + "]";
+ }
+
+ @Override
+ public String getProjectName() {
+ return projectName;
+ }
+
+ @Override
+ public void setProjectName(String prjName) {
+ projectName = prjName;
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public void setConfig(KylinConfig config) {
+ this.config = config;
+ }
+
+ public RealizationEntry getHistoryRealization() {
+ return historyRealization;
+ }
+
+ public RealizationEntry getRealTimeRealization() {
+ return realTimeRealization;
+ }
+
+ public IRealization getHistoryRealizationInstance() {
+ if (historyRealizationInstance == null) {
+ this.init();
+ }
+ return historyRealizationInstance;
+ }
+
+ public IRealization getRealTimeRealizationInstance() {
+ if (realTimeRealizationInstance == null) {
+ this.init();
+ }
+ return realTimeRealizationInstance;
+ }
+
+ @Override
+ public long getDateRangeStart() {
+ return Math.min(getHistoryRealizationInstance().getDateRangeStart(), getRealTimeRealizationInstance().getDateRangeStart());
+ }
+
+ @Override
+ public long getDateRangeEnd() {
+ return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1;
+ }
+
+
+
+ public DataModelDesc getDataModelDesc(){
+ if (getHistoryRealizationInstance() instanceof CubeInstance) {
+ return ((CubeInstance) historyRealizationInstance).getDescriptor().getModel();
+ }
+
+ return ((IIInstance) historyRealizationInstance).getDescriptor().getModel();
+ }
+
+ @Override
+ public String getModelName() {
+ if (getHistoryRealizationInstance() instanceof CubeInstance) {
+ return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName();
+ }
+
+ return ((IIInstance) historyRealizationInstance).getDescriptor().getModelName();
+ }
+
+ @Override
+ public List<TblColRef> getAllDimensions(){
+
+ return this.getHistoryRealizationInstance().getAllDimensions();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
new file mode 100644
index 0000000..df3fae3
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -0,0 +1,127 @@
+package org.apache.kylin.storage.hybrid;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.restclient.Broadcaster;
+import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.IRealizationProvider;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ */
+public class HybridManager implements IRealizationProvider {
+ public static final Serializer<HybridInstance> HYBRID_SERIALIZER = new JsonSerializer<HybridInstance>(HybridInstance.class);
+
+ private static final Logger logger = LoggerFactory.getLogger(HybridManager.class);
+
+ // static cached instances
+ private static final ConcurrentHashMap<KylinConfig, HybridManager> CACHE = new ConcurrentHashMap<KylinConfig, HybridManager>();
+
+ public static HybridManager getInstance(KylinConfig config) {
+ HybridManager r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+
+ synchronized (HybridManager.class) {
+ r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+ try {
+ r = new HybridManager(config);
+ CACHE.put(config, r);
+ if (CACHE.size() > 1) {
+ logger.warn("More than one Hybrid Manager singleton exist");
+ }
+ return r;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to init Hybrid Manager from " + config, e);
+ }
+ }
+ }
+
+ public static void clearCache() {
+ CACHE.clear();
+ }
+
+ // ============================================================================
+
+ private KylinConfig config;
+
+ private CaseInsensitiveStringCache<HybridInstance> hybridMap = new CaseInsensitiveStringCache<HybridInstance>(Broadcaster.TYPE.HYBRID);
+
+ private HybridManager(KylinConfig config) throws IOException {
+ logger.info("Initializing HybridManager with config " + config);
+ this.config = config;
+
+ loadAllHybridInstance();
+ }
+
+ private void loadAllHybridInstance() throws IOException {
+ ResourceStore store = getStore();
+ List<String> paths = store.collectResourceRecursively(ResourceStore.HYBRID_RESOURCE_ROOT, ".json");
+
+ logger.debug("Loading Hybrid from folder " + store.getReadableResourcePath(ResourceStore.HYBRID_RESOURCE_ROOT));
+
+ for (String path : paths) {
+ loadHybridInstance(path);
+ }
+
+ logger.debug("Loaded " + paths.size() + " Hybrid(s)");
+ }
+
+ private synchronized HybridInstance loadHybridInstance(String path) throws IOException {
+ ResourceStore store = getStore();
+
+ HybridInstance hybridInstance = null;
+ try {
+ hybridInstance = store.getResource(path, HybridInstance.class, HYBRID_SERIALIZER);
+ hybridInstance.setConfig(config);
+
+ if (StringUtils.isBlank(hybridInstance.getName()))
+ throw new IllegalStateException("HybridInstance name must not be blank, at " + path);
+
+ if (hybridInstance.getHistoryRealization() == null || hybridInstance.getRealTimeRealization() == null) {
+
+ throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path);
+ }
+
+ final String name = hybridInstance.getName();
+ hybridMap.putLocal(name, hybridInstance);
+
+ return hybridInstance;
+ } catch (Exception e) {
+ logger.error("Error during load hybrid instance " + path, e);
+ return null;
+ }
+ }
+
+ @Override
+ public RealizationType getRealizationType() {
+ return RealizationType.HYBRID;
+ }
+
+ @Override
+ public IRealization getRealization(String name) {
+ return getHybridInstance(name);
+ }
+
+ public HybridInstance getHybridInstance(String name) {
+ return hybridMap.get(name);
+ }
+
+ private ResourceStore getStore() {
+ return ResourceStore.getStore(this.config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
new file mode 100644
index 0000000..7ad5ed0
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -0,0 +1,69 @@
+package org.apache.kylin.storage.hybrid;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ranges;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.SQLDigestUtil;
+import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.tuple.TupleInfo;
+
+import javax.annotation.Nullable;
+
+/**
+ */
+public class HybridStorageEngine implements IStorageEngine {
+
+ private HybridInstance hybridInstance;
+ private IStorageEngine historicalStorageEngine;
+ private IStorageEngine realtimeStorageEngine;
+
+ public HybridStorageEngine(HybridInstance hybridInstance) {
+ this.hybridInstance = hybridInstance;
+ this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance());
+ this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance());
+ }
+
+ @Override
+ public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) {
+
+ // search the historic realization
+ ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest);
+
+ String modelName = hybridInstance.getModelName();
+ MetadataManager metaMgr = getMetadataManager();
+ DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
+
+ // if the model isn't partitioned, only query the history
+ if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
+ return historicalDataIterator;
+
+ TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
+
+ ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
+ Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
+ new Function<Void, ITupleIterator>() {
+ @Nullable
+ @Override
+ public ITupleIterator apply(@Nullable Void input) {
+ return realtimeStorageEngine.search(context, sqlDigest);
+ }
+ });
+
+ // combine history and real-time tuple iterator
+ return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
+ }
+
+
+ private MetadataManager getMetadataManager() {
+ return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
index 87c6445..b98b09b 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationTest.java
@@ -36,6 +36,7 @@ import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
index c4f785a..6ec9385 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TableRecordInfoTest.java
@@ -25,6 +25,7 @@ import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9491e92e/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
new file mode 100644
index 0000000..3ad7ca6
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.storage.hybrid;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class HybridManagerTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBasics() throws Exception {
+ HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
+ cube.init();
+ System.out.println(JsonUtil.writeValueAsIndentString(cube));
+
+ IRealization history = cube.getHistoryRealizationInstance();
+ IRealization realTime = cube.getRealTimeRealizationInstance();
+
+ Assert.assertTrue(history instanceof CubeInstance);
+ Assert.assertTrue(realTime instanceof IIInstance);
+
+ }
+
+
+ public HybridManager getHybridManager() {
+ return HybridManager.getInstance(getTestConfig());
+ }
+}
[02/50] [abbrv] incubator-kylin git commit: KYLIN-848 Couldn't resume
or discard a cube job
Posted by li...@apache.org.
KYLIN-848 Couldn't resume or discard a cube job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/198be9e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/198be9e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/198be9e0
Branch: refs/heads/0.7
Commit: 198be9e067742b7493a912cb84e09d618f54a8fd
Parents: 6ab0903
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 23 21:00:40 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 23 21:00:40 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/JobController.java | 6 +++--
.../apache/kylin/rest/service/JobService.java | 28 +++++++++++++-------
2 files changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/198be9e0/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 2b79155..2f2d9cf 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -169,7 +169,8 @@ public class JobController extends BasicController implements InitializingBean {
@ResponseBody
public JobInstance resume(@PathVariable String jobId) {
try {
- jobService.resumeJob(jobId);
+ final JobInstance jobInstance = jobService.getJobInstance(jobId);
+ jobService.resumeJob(jobInstance);
return jobService.getJobInstance(jobId);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
@@ -188,7 +189,8 @@ public class JobController extends BasicController implements InitializingBean {
public JobInstance cancel(@PathVariable String jobId) {
try {
- return jobService.cancelJob(jobId);
+ final JobInstance jobInstance = jobService.getJobInstance(jobId);
+ return jobService.cancelJob(jobInstance);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/198be9e0/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index b3aa4ee..c3a5f8c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -44,6 +44,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.rest.constant.Constant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
@@ -59,6 +60,9 @@ public class JobService extends BasicService {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(JobService.class);
+ @Autowired
+ private AccessService accessService;
+
public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue) throws IOException, JobException {
Integer limit = (null == limitValue) ? 30 : limitValue;
Integer offset = (null == offsetValue) ? 0 : offsetValue;
@@ -150,7 +154,13 @@ public class JobService extends BasicService {
throw new JobException("invalid build type:" + buildType);
}
getExecutableManager().addJob(job);
- return getSingleJobInstance(job);
+ JobInstance jobInstance = getSingleJobInstance(job);
+
+ accessService.init(jobInstance, null);
+ accessService.inherit(jobInstance, cube);
+
+ return jobInstance;
+
}
public JobInstance getJobInstance(String uuid) throws IOException, JobException {
@@ -274,26 +284,26 @@ public class JobService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
- public void resumeJob(String jobId) throws IOException, JobException {
- getExecutableManager().resumeJob(jobId);
+ public void resumeJob(JobInstance job) throws IOException, JobException {
+ getExecutableManager().resumeJob(job.getId());
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
- public JobInstance cancelJob(String jobId) throws IOException, JobException {
+ public JobInstance cancelJob(JobInstance job) throws IOException, JobException {
// CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
// for (BuildCubeJob cubeJob: listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING))) {
// getExecutableManager().stopJob(cubeJob.getId());
// }
- final JobInstance jobInstance = getJobInstance(jobId);
- final String segmentId = jobInstance.getRelatedSegment();
- CubeInstance cubeInstance = getCubeManager().getCube(jobInstance.getRelatedCube());
+
+ final String segmentId = job.getRelatedSegment();
+ CubeInstance cubeInstance = getCubeManager().getCube(job.getRelatedCube());
final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
if (segment.getStatus() == SegmentStatusEnum.NEW) {
cubeInstance.getSegments().remove(segment);
getCubeManager().updateCube(cubeInstance);
}
- getExecutableManager().discardJob(jobId);
- return jobInstance;
+ getExecutableManager().discardJob(job.getId());
+ return job;
}
}
[30/50] [abbrv] incubator-kylin git commit: KYLIN-780 Upgrade query
module, all query test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 7f707c7..485ed3d 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -22,7 +22,7 @@
<artifactId>kylin-jdbc</artifactId>
<packaging>jar</packaging>
<name>Kylin:JDBC</name>
- <description>Kylin JDBC Driver on optiq avatica</description>
+ <description>Kylin JDBC Driver on Calcite Avatica</description>
<parent>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
index 5198fd7..cee4de4 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
@@ -21,12 +21,12 @@ package org.apache.kylin.jdbc;
import java.sql.DriverManager;
import java.sql.SQLException;
-import net.hydromatic.avatica.AvaticaConnection;
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.DriverVersion;
-import net.hydromatic.avatica.Handler;
-import net.hydromatic.avatica.HandlerImpl;
-import net.hydromatic.avatica.UnregisteredDriver;
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.DriverVersion;
+import org.apache.calcite.avatica.Handler;
+import org.apache.calcite.avatica.HandlerImpl;
+import org.apache.calcite.avatica.UnregisteredDriver;
import org.apache.kylin.jdbc.stub.RemoteClient;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
index 887bd66..718ef7d 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnectionImpl.java
@@ -27,12 +27,12 @@ import java.util.Properties;
import javax.xml.bind.DatatypeConverter;
-import net.hydromatic.avatica.AvaticaConnection;
-import net.hydromatic.avatica.AvaticaFactory;
-import net.hydromatic.avatica.AvaticaPreparedStatement;
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.Meta;
-import net.hydromatic.avatica.UnregisteredDriver;
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaFactory;
+import org.apache.calcite.avatica.AvaticaPreparedStatement;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.avatica.UnregisteredDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
index 04d9231..2502d4b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinEnumerator.java
@@ -21,7 +21,7 @@ package org.apache.kylin.jdbc;
import java.util.Collection;
import java.util.Iterator;
-import net.hydromatic.linq4j.Enumerator;
+import org.apache.calcite.linq4j.Enumerator;
/**
* Query results enumerator
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
index 90ff2e2..1336f96 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbc41Factory.java
@@ -24,16 +24,16 @@ import java.util.List;
import java.util.Properties;
import java.util.TimeZone;
-import net.hydromatic.avatica.AvaticaConnection;
-import net.hydromatic.avatica.AvaticaDatabaseMetaData;
-import net.hydromatic.avatica.AvaticaFactory;
-import net.hydromatic.avatica.AvaticaPrepareResult;
-import net.hydromatic.avatica.AvaticaPreparedStatement;
-import net.hydromatic.avatica.AvaticaResultSet;
-import net.hydromatic.avatica.AvaticaResultSetMetaData;
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.UnregisteredDriver;
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaDatabaseMetaData;
+import org.apache.calcite.avatica.AvaticaFactory;
+import org.apache.calcite.avatica.AvaticaPrepareResult;
+import org.apache.calcite.avatica.AvaticaPreparedStatement;
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaResultSetMetaData;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.UnregisteredDriver;
import org.apache.kylin.jdbc.stub.KylinClient;
import org.apache.kylin.jdbc.stub.RemoteClient;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
index 3c14c9b..17e8ed4 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMetaImpl.java
@@ -29,25 +29,19 @@ import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
-import net.hydromatic.avatica.AvaticaPrepareResult;
-import net.hydromatic.avatica.AvaticaResultSet;
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.ColumnMetaData.Rep;
-import net.hydromatic.avatica.Cursor;
-import net.hydromatic.avatica.Meta;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.runtime.EnumeratorCursor;
-
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.sql.SqlJdbcFunctionCall;
+import org.apache.calcite.sql.parser.SqlParser;
import org.apache.kylin.jdbc.stub.DataSet;
import org.apache.kylin.jdbc.stub.KylinColumnMetaData;
import org.apache.kylin.jdbc.stub.RemoteClient;
-import org.eigenbase.sql.SqlJdbcFunctionCall;
-import org.eigenbase.sql.parser.SqlParser;
-import org.eigenbase.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.kylin.jdbc.util.SQLTypeMap;
/**
@@ -329,7 +323,7 @@ public class KylinMetaImpl implements Meta {
* Client could request metadata after prepare
*
* (non-Javadoc)
- * @see net.hydromatic.avatica.Meta#prepare(net.hydromatic.avatica.AvaticaStatement, java.lang.String)
+ * @see org.apache.calcite.avatica.Meta#prepare(org.apache.calcite.avatica.AvaticaStatement, java.lang.String)
*/
public AvaticaPrepareResult prepare(AvaticaStatement statement, String sql) {
RemoteClient client = factory.newRemoteClient(conn);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
index 0e4fb35..686e948 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPrepare.java
@@ -21,12 +21,10 @@ package org.apache.kylin.jdbc;
import java.util.List;
import java.util.Map;
-import net.hydromatic.avatica.AvaticaParameter;
-import net.hydromatic.avatica.AvaticaPrepareResult;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.Cursor;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.runtime.EnumeratorCursor;
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.AvaticaPrepareResult;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.Cursor;
/**
* Interface of kylin prepare statement implementation
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 28c3725..7cb9b25 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -21,9 +21,9 @@ package org.apache.kylin.jdbc;
import java.sql.ResultSetMetaData;
import java.util.TimeZone;
-import net.hydromatic.avatica.AvaticaPrepareResult;
-import net.hydromatic.avatica.AvaticaResultSet;
-import net.hydromatic.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.AvaticaPrepareResult;
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaStatement;
import org.apache.kylin.jdbc.KylinPrepare.PrepareResult;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
index b380b00..0d7605d 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatementImpl.java
@@ -18,9 +18,9 @@
package org.apache.kylin.jdbc;
-import net.hydromatic.avatica.AvaticaConnection;
-import net.hydromatic.avatica.AvaticaResultSet;
-import net.hydromatic.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaResultSet;
+import org.apache.calcite.avatica.AvaticaStatement;
/**
* Kylin statement implementation
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
index 3584b31..8b0347a 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/DataSet.java
@@ -20,8 +20,8 @@ package org.apache.kylin.jdbc.stub;
import java.util.List;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.linq4j.Enumerator;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.linq4j.Enumerator;
/**
* Data set wrapper.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
index ccbfe5d..67652cb 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/KylinClient.java
@@ -23,10 +23,10 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import net.hydromatic.avatica.AvaticaStatement;
-import net.hydromatic.avatica.ColumnMetaData;
-import net.hydromatic.avatica.ColumnMetaData.Rep;
-import net.hydromatic.avatica.ColumnMetaData.ScalarType;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.calcite.avatica.ColumnMetaData.ScalarType;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
index dae4126..093f150 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/stub/RemoteClient.java
@@ -20,7 +20,7 @@ package org.apache.kylin.jdbc.stub;
import java.sql.SQLException;
-import net.hydromatic.avatica.AvaticaStatement;
+import org.apache.calcite.avatica.AvaticaStatement;
import org.apache.kylin.jdbc.KylinMetaImpl.MetaProject;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index a4e8db6..9f6d38a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -32,7 +32,7 @@ import java.util.regex.Pattern;
*
*/
public class DataType {
-
+
public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
+ "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
+ "|date|time|datetime|timestamp|byte|int|short|long|string|hllc" //
@@ -89,6 +89,7 @@ public class DataType {
private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
+ public static final DataType ANY = DataType.getInstance("any");
public static DataType getInstance(String type) {
if (type == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 3a9fde6..3c5409b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -130,13 +130,13 @@ public class FunctionDesc {
this.parameter = parameter;
}
- public String getSQLType() {
+ public DataType getSQLType() {
if (isCountDistinct())
- return "any";
+ return DataType.ANY;
else if (isSum() || isMax() || isMin())
- return parameter.getColRefs().get(0).getType().getName();
+ return parameter.getColRefs().get(0).getType();
else
- return returnType;
+ return returnDataType;
}
public String getReturnType() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 10c4472..bc1b3a6 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -256,7 +256,7 @@ class ProjectL2Cache {
private static class TableCache {
private boolean exposed = false;
private TableDesc tableDesc;
- private Set<ColumnDesc> exposedColumns = Sets.newHashSet();
+ private Set<ColumnDesc> exposedColumns = Sets.newLinkedHashSet();
private Set<IRealization> realizations = Sets.newHashSet();
TableCache(TableDesc tableDesc) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cdd8191..c0a4be2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,9 +83,8 @@
<!-- REST Service -->
<spring.framework.version>3.1.2.RELEASE</spring.framework.version>
- <!-- Optiq Version -->
- <optiq.version>0.9.2-incubating</optiq.version>
- <linq4j.version>0.4</linq4j.version>
+ <!-- Calcite Version -->
+ <calcite.version>1.0.0-incubating</calcite.version>
<!-- Metrics Codahale Version -->
<metrics.version>3.0.1</metrics.version>
@@ -102,7 +101,7 @@
<sonar.jacoco.reportPath>${project.basedir}/../target/jacoco.exec</sonar.jacoco.reportPath>
<sonar.language>java</sonar.language>
<sonar.jacoco.excludes>
- org/apache/kylin/**/tools/**:**/*CLI.java:net/hydromatic/optiq/**:org/eigenbase/sql2rel/**
+ org/apache/kylin/**/tools/**:**/*CLI.java
</sonar.jacoco.excludes>
</properties>
@@ -248,21 +247,21 @@
<version>${yarn.version}</version>
</dependency>
- <!-- optiq dependencies -->
+ <!-- Calcite dependencies -->
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
- <version>${optiq.version}</version>
+ <version>${calcite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-avatica</artifactId>
- <version>${optiq.version}</version>
+ <version>${calcite.version}</version>
</dependency>
<dependency>
- <groupId>net.hydromatic</groupId>
- <artifactId>linq4j</artifactId>
- <version>${linq4j.version}</version>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-linq4j</artifactId>
+ <version>${calcite.version}</version>
</dependency>
<!-- Other dependencies -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 3881b4d..86fca54 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -23,7 +23,7 @@
<artifactId>kylin-query</artifactId>
<packaging>jar</packaging>
<name>Kylin:Query</name>
- <description>kylin query engine based on optiq</description>
+ <description>kylin query engine based on Calcite</description>
<parent>
<groupId>org.apache.kylin</groupId>
@@ -54,8 +54,8 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>net.hydromatic</groupId>
- <artifactId>linq4j</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-linq4j</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index a85f47b..b83e449 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -22,14 +22,12 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.DataContext;
-import net.hydromatic.optiq.jdbc.OptiqConnection;
-
-import org.eigenbase.reltype.RelDataTypeField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+import org.apache.calcite.DataContext;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.storage.StorageEngineFactory;
@@ -39,7 +37,6 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
/**
- * @author xjiang
*/
public class CubeEnumerator implements Enumerator<Object[]> {
@@ -104,7 +101,7 @@ public class CubeEnumerator implements Enumerator<Object[]> {
this.fieldIndexes = new int[size];
for (int i = 0; i < size; i++) {
String field = fields.get(i);
- RelDataTypeField relField = olapContext.olapRowType.getField(field, true);
+ RelDataTypeField relField = olapContext.olapRowType.getField(field, true, false);
if (relField != null) {
fieldIndexes[i] = relField.getIndex();
} else {
@@ -135,11 +132,9 @@ public class CubeEnumerator implements Enumerator<Object[]> {
// bind dynamic variables
bindVariable(olapContext.filter);
-
-
// query storage engine
IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization);
- ITupleIterator iterator = storageEngine.search(olapContext.storageContext,olapContext.getSQLDigest());
+ ITupleIterator iterator = storageEngine.search(olapContext.storageContext, olapContext.getSQLDigest());
if (logger.isDebugEnabled()) {
logger.debug("return TupleIterator...");
}
@@ -148,7 +143,6 @@ public class CubeEnumerator implements Enumerator<Object[]> {
return iterator;
}
-
private void bindVariable(TupleFilter filter) {
if (filter == null) {
return;
@@ -172,7 +166,7 @@ public class CubeEnumerator implements Enumerator<Object[]> {
}
private void setConnectionProperties() {
- OptiqConnection conn = (OptiqConnection) optiqContext.getQueryProvider();
+ CalciteConnection conn = (CalciteConnection) optiqContext.getQueryProvider();
Properties connProps = conn.getProperties();
String propThreshold = connProps.getProperty(OLAPQuery.PROP_SCAN_THRESHOLD);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java
index c9cec94..b9fb39c 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java
@@ -24,16 +24,12 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import net.hydromatic.linq4j.Enumerator;
-
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.eigenbase.reltype.RelDataTypeField;
/**
* Hive Query Result Enumerator
- *
- * @author xjiang
- *
*/
public class HiveEnumerator implements Enumerator<Object[]> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 0a3627f..3ae19dd 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -18,12 +18,12 @@
package org.apache.kylin.query.enumerator;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import net.hydromatic.linq4j.Enumerator;
-
+import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.DimensionDesc;
@@ -34,8 +34,6 @@ import org.apache.kylin.query.schema.OLAPTable;
import org.apache.kylin.storage.tuple.Tuple;
/**
- * @author yangli9
- *
*/
public class LookupTableEnumerator implements Enumerator<Object[]> {
@@ -85,7 +83,8 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
@Override
public Object[] current() {
- return current;
+ // NOTE if without the copy, sql_lookup/query03.sql will yields messy result. Very weird coz other lookup queries are all good.
+ return Arrays.copyOf(current, current.length);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
index 090980f..453c898 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
@@ -18,17 +18,13 @@
package org.apache.kylin.query.enumerator;
-import net.hydromatic.linq4j.AbstractEnumerable;
-import net.hydromatic.linq4j.Enumerable;
-import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.DataContext;
-
+import org.apache.calcite.DataContext;
+import org.apache.calcite.linq4j.AbstractEnumerable;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.query.relnode.OLAPContext;
/**
- *
- * @author xjiang
- *
*/
public class OLAPQuery extends AbstractEnumerable<Object[]> implements Enumerable<Object[]> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java
index 5c6b29b..b80a483 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java
@@ -18,35 +18,31 @@
package org.apache.kylin.query.optrule;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.InvalidRelException;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.AggregateRel;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.convert.ConverterRule;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelTraitSet;
-
import org.apache.kylin.query.relnode.OLAPAggregateRel;
/**
- *
- * @author xjiang
- *
*/
public class OLAPAggregateRule extends ConverterRule {
public static final ConverterRule INSTANCE = new OLAPAggregateRule();
public OLAPAggregateRule() {
- super(AggregateRel.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPAggregateRule");
+ super(LogicalAggregate.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPAggregateRule");
}
@Override
public RelNode convert(RelNode rel) {
- AggregateRel agg = (AggregateRel) rel;
+ LogicalAggregate agg = (LogicalAggregate) rel;
RelTraitSet traitSet = agg.getTraitSet().replace(OLAPRel.CONVENTION);
try {
- return new OLAPAggregateRel(agg.getCluster(), traitSet, convert(agg.getChild(), traitSet), agg.getGroupSet(), agg.getAggCallList());
+ return new OLAPAggregateRel(agg.getCluster(), traitSet, convert(agg.getInput(), traitSet), agg.getGroupSet(), agg.getAggCallList());
} catch (InvalidRelException e) {
throw new IllegalStateException("Can't create OLAPAggregateRel!", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
index 619e430..4c34f4e 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPFilterRule.java
@@ -18,33 +18,29 @@
package org.apache.kylin.query.optrule;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.kylin.query.relnode.OLAPFilterRel;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.FilterRel;
-import org.eigenbase.relopt.RelOptRule;
-import org.eigenbase.relopt.RelOptRuleCall;
-import org.eigenbase.relopt.RelTraitSet;
/**
- *
- * @author xjiang
- *
*/
-
public class OLAPFilterRule extends RelOptRule {
public static final RelOptRule INSTANCE = new OLAPFilterRule();
public OLAPFilterRule() {
- super(operand(FilterRel.class, any()));
+ super(operand(LogicalFilter.class, any()));
}
@Override
public void onMatch(RelOptRuleCall call) {
- FilterRel filter = call.rel(0);
+ LogicalFilter filter = call.rel(0);
RelTraitSet traitSet = filter.getTraitSet().replace(OLAPRel.CONVENTION);
- OLAPFilterRel olapFilter = new OLAPFilterRel(filter.getCluster(), traitSet, convert(filter.getChild(), traitSet), filter.getCondition());
+ OLAPFilterRel olapFilter = new OLAPFilterRel(filter.getCluster(), traitSet, convert(filter.getInput(), traitSet), filter.getCondition());
call.transformTo(olapFilter);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
index dd97933..10047e6 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java
@@ -18,35 +18,32 @@
package org.apache.kylin.query.optrule;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.InvalidRelException;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.kylin.query.relnode.OLAPFilterRel;
import org.apache.kylin.query.relnode.OLAPJoinRel;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.JoinInfo;
-import org.eigenbase.rel.JoinRel;
-import org.eigenbase.rel.JoinRelType;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.convert.ConverterRule;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
/**
- *
- * @author xjiang
- *
*/
public class OLAPJoinRule extends ConverterRule {
public static final ConverterRule INSTANCE = new OLAPJoinRule();
public OLAPJoinRule() {
- super(JoinRel.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPJoinRule");
+ super(LogicalJoin.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPJoinRule");
}
@Override
public RelNode convert(RelNode rel) {
- JoinRel join = (JoinRel) rel;
+ LogicalJoin join = (LogicalJoin) rel;
RelNode left = join.getInput(0);
RelNode right = join.getInput(1);
@@ -68,8 +65,7 @@ public class OLAPJoinRule extends ConverterRule {
info.getEquiCondition(left, right, cluster.getRexBuilder()), //
info.leftKeys, info.rightKeys, join.getJoinType(), join.getVariablesStopped());
} catch (InvalidRelException e) {
- // Semantic error not possible. Must be a bug. Convert to
- // internal error.
+ // Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
// LOGGER.fine(e.toString());
// return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
index 359b722..cde934e 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPLimitRule.java
@@ -18,35 +18,32 @@
package org.apache.kylin.query.optrule;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Sort;
import org.apache.kylin.query.relnode.OLAPLimitRel;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.relopt.RelOptRule;
-import org.eigenbase.relopt.RelOptRuleCall;
-import org.eigenbase.relopt.RelTraitSet;
/**
- *
- * @author xjiang
- *
*/
public class OLAPLimitRule extends RelOptRule {
public static final RelOptRule INSTANCE = new OLAPLimitRule();
public OLAPLimitRule() {
- super(operand(SortRel.class, any()), "OLAPLimitRule");
+ super(operand(Sort.class, any()), "OLAPLimitRule");
}
@Override
public void onMatch(RelOptRuleCall call) {
- final SortRel sort = call.rel(0);
+ final Sort sort = call.rel(0);
if (sort.offset == null && sort.fetch == null) {
return;
}
final RelTraitSet traitSet = sort.getTraitSet().replace(OLAPRel.CONVENTION);
- RelNode input = sort.getChild();
+ RelNode input = sort.getInput();
if (!sort.getCollation().getFieldCollations().isEmpty()) {
// Create a sort with the same sort key, but no offset or fetch.
input = sort.copy(sort.getTraitSet(), input, sort.getCollation(), null, null);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
index 7ab9dc3..4867162 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPProjectRule.java
@@ -18,34 +18,30 @@
package org.apache.kylin.query.optrule;
-import org.eigenbase.rel.ProjectRel;
-import org.eigenbase.relopt.RelOptRule;
-import org.eigenbase.relopt.RelOptRuleCall;
-import org.eigenbase.relopt.RelTraitSet;
-
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.logical.LogicalProject;
import org.apache.kylin.query.relnode.OLAPProjectRel;
import org.apache.kylin.query.relnode.OLAPRel;
/**
- *
- * @author xjiang
- *
*/
public class OLAPProjectRule extends RelOptRule {
public static final RelOptRule INSTANCE = new OLAPProjectRule();
public OLAPProjectRule() {
- super(operand(ProjectRel.class, any()));
+ super(operand(LogicalProject.class, any()));
}
@Override
public void onMatch(RelOptRuleCall call) {
- ProjectRel project = call.rel(0);
+ LogicalProject project = call.rel(0);
RelTraitSet traitSet = project.getTraitSet().replace(OLAPRel.CONVENTION);
OLAPProjectRel olapProj = new OLAPProjectRel(project.getCluster(), traitSet, //
- convert(project.getChild(), traitSet), project.getProjects(), project.getRowType(), project.getFlags());
+ convert(project.getInput(), traitSet), project.getProjects(), project.getRowType(), project.getFlags());
call.transformTo(olapProj);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPSortRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPSortRule.java
index c9ea5f4..c020d63 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPSortRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPSortRule.java
@@ -18,35 +18,32 @@
package org.apache.kylin.query.optrule;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
+import org.apache.calcite.rel.core.Sort;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.SortRel;
-import org.eigenbase.rel.convert.ConverterRule;
-import org.eigenbase.relopt.Convention;
-import org.eigenbase.relopt.RelTraitSet;
-
import org.apache.kylin.query.relnode.OLAPSortRel;
/**
- * @author xjiang
- *
*/
public class OLAPSortRule extends ConverterRule {
public static final OLAPSortRule INSTANCE = new OLAPSortRule();
public OLAPSortRule() {
- super(SortRel.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPSortRule");
+ super(Sort.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPSortRule");
}
@Override
public RelNode convert(RelNode rel) {
- final SortRel sort = (SortRel) rel;
+ final Sort sort = (Sort) rel;
if (sort.offset != null || sort.fetch != null) {
return null;
}
final RelTraitSet traitSet = sort.getTraitSet().replace(OLAPRel.CONVENTION);
- final RelNode input = sort.getChild();
+ final RelNode input = sort.getInput();
return new OLAPSortRel(rel.getCluster(), traitSet, convert(input, input.getTraitSet().replace(OLAPRel.CONVENTION)), sort.getCollation(), sort.offset, sort.fetch);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/optrule/OLAPToEnumerableConverterRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPToEnumerableConverterRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPToEnumerableConverterRule.java
index 4edee38..995e3e7 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPToEnumerableConverterRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPToEnumerableConverterRule.java
@@ -18,19 +18,14 @@
package org.apache.kylin.query.optrule;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.convert.ConverterRule;
import org.apache.kylin.query.relnode.OLAPRel;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.convert.ConverterRule;
-import org.eigenbase.relopt.RelTraitSet;
-
import org.apache.kylin.query.relnode.OLAPToEnumerableConverter;
/**
- *
- * @author xjiang
- *
*/
public class OLAPToEnumerableConverterRule extends ConverterRule {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 10d4150..832d4c8 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -20,50 +20,47 @@ package org.apache.kylin.query.relnode;
import java.util.*;
-import net.hydromatic.optiq.AggregateFunction;
-import net.hydromatic.optiq.FunctionParameter;
-import net.hydromatic.optiq.impl.AggregateFunctionImpl;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableAggregateRel;
-
+import org.apache.calcite.adapter.enumerable.EnumerableAggregate;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.InvalidRelException;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.schema.AggregateFunction;
+import org.apache.calcite.schema.FunctionParameter;
+import org.apache.calcite.schema.impl.AggregateFunctionImpl;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.InferTypes;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Util;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.sqlfunc.HLLDistinctCountAggFunc;
-import org.eigenbase.rel.AggregateCall;
-import org.eigenbase.rel.AggregateRelBase;
-import org.eigenbase.rel.Aggregation;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.eigenbase.sql.SqlAggFunction;
-import org.eigenbase.sql.SqlIdentifier;
-import org.eigenbase.sql.fun.SqlStdOperatorTable;
-import org.eigenbase.sql.parser.SqlParserPos;
-import org.eigenbase.sql.type.InferTypes;
-import org.eigenbase.sql.type.OperandTypes;
-import org.eigenbase.sql.type.ReturnTypes;
-import org.eigenbase.sql.type.SqlTypeFamily;
-import org.eigenbase.sql.validate.SqlUserDefinedAggFunction;
-import org.eigenbase.util.Util;
import com.google.common.base.Preconditions;
/**
- * @author xjiang
*/
-public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, EnumerableRel {
+public class OLAPAggregateRel extends Aggregate implements OLAPRel {
private final static Map<String, String> AGGR_FUNC_MAP = new HashMap<String, String>();
@@ -96,16 +93,22 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
private List<TblColRef> groups;
private List<FunctionDesc> aggregations;
- public OLAPAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, BitSet groupSet, List<AggregateCall> aggCalls) throws InvalidRelException {
- super(cluster, traits, child, groupSet, aggCalls);
+ public OLAPAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, ImmutableBitSet groupSet, List<AggregateCall> aggCalls) throws InvalidRelException {
+ super(cluster, traits, child, false, groupSet, asList(groupSet), aggCalls);
Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION);
this.afterAggregate = false;
this.rewriteAggCalls = aggCalls;
this.rowType = getRowType();
}
+ private static List<ImmutableBitSet> asList(ImmutableBitSet groupSet) {
+ ArrayList<ImmutableBitSet> l = new ArrayList<ImmutableBitSet>(1);
+ l.add(groupSet);
+ return l;
+ }
+
@Override
- public AggregateRelBase copy(RelTraitSet traitSet, RelNode input, BitSet groupSet, List<AggregateCall> aggCalls) {
+ public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
try {
return new OLAPAggregateRel(getCluster(), traitSet, input, groupSet, aggCalls);
} catch (InvalidRelException e) {
@@ -115,19 +118,13 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
- double factor = .5;
- for (AggregateCall aggCall : aggCalls) {
- if ("$SUM0".equals(aggCall.getAggregation().getName())) {
- factor = .2;
- }
- }
- return super.computeSelfCost(planner).multiplyBy(factor);
+ return super.computeSelfCost(planner).multiplyBy(.05);
}
@Override
public void implementOLAP(OLAPImplementor implementor) {
- implementor.visitChild(getChild(), this);
+ implementor.visitChild(getInput(), this);
this.context = implementor.getContext();
this.columnRowType = buildColumnRowType();
@@ -153,7 +150,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
buildGroups();
buildAggregations();
- ColumnRowType inputColumnRowType = ((OLAPRel) getChild()).getColumnRowType();
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
List<TblColRef> columns = new ArrayList<TblColRef>(this.rowType.getFieldCount());
columns.addAll(this.groups);
@@ -187,7 +184,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
private void buildGroups() {
- ColumnRowType inputColumnRowType = ((OLAPRel) getChild()).getColumnRowType();
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
this.groups = new ArrayList<TblColRef>();
for (int i = getGroupSet().nextSetBit(0); i >= 0; i = getGroupSet().nextSetBit(i + 1)) {
Set<TblColRef> columns = inputColumnRowType.getSourceColumnsByIndex(i);
@@ -196,7 +193,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
private void buildAggregations() {
- ColumnRowType inputColumnRowType = ((OLAPRel) getChild()).getColumnRowType();
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
this.aggregations = new ArrayList<FunctionDesc>();
for (AggregateCall aggCall : this.rewriteAggCalls) {
ParameterDesc parameter = null;
@@ -223,7 +220,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
private void translateAggregation() {
- ColumnRowType inputColumnRowType = ((OLAPRel) getChild()).getColumnRowType();
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
for (int i = 0; i < this.aggregations.size(); i++) {
FunctionDesc aggFunc = this.aggregations.get(i);
context.aggregations.add(aggFunc);
@@ -247,14 +244,13 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
private void fillbackOptimizedColumn() {
- // some aggcall will be optimized out in sub-query (e.g. tableau generated sql)
- // we need to fill them back
- RelDataType inputAggRow = getChild().getRowType();
+ // some aggcall will be optimized out in sub-query (e.g. tableau generated sql), we need to fill them back
+ RelDataType inputAggRow = getInput().getRowType();
RelDataType outputAggRow = getRowType();
if (inputAggRow.getFieldCount() != outputAggRow.getFieldCount()) {
for (RelDataTypeField inputField : inputAggRow.getFieldList()) {
String inputFieldName = inputField.getName();
- if (outputAggRow.getField(inputFieldName, true) == null) {
+ if (outputAggRow.getField(inputFieldName, true, false) == null) {
TblColRef column = this.columnRowType.getColumnByIndex(inputField.getIndex());
this.context.metricsColumns.add(column);
}
@@ -264,7 +260,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
@Override
public void implementRewrite(RewriteImplementor implementor) {
- implementor.visitChild(this, getChild());
+ implementor.visitChild(this, getInput());
// only rewrite the first aggregation
if (!this.afterAggregate && RewriteImplementor.needRewrite(this.context)) {
@@ -281,7 +277,8 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
// rebuild rowType & columnRowType
- this.rowType = this.deriveRowType();
+ //ClassUtil.updateFinalField(Aggregate.class, "aggCalls", this, rewriteAggCalls);
+ this.rowType = this.deriveRowType(); // this does not work coz super.aggCalls is final
this.columnRowType = this.buildColumnRowType();
}
@@ -291,34 +288,24 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
// rebuild parameters
List<Integer> newArgList = new ArrayList<Integer>(1);
String fieldName = func.getRewriteFieldName();
- RelDataTypeField field = getChild().getRowType().getField(fieldName, true);
+ RelDataTypeField field = getInput().getRowType().getField(fieldName, true, false);
newArgList.add(field.getIndex());
// rebuild function
RelDataType fieldType = aggCall.getType();
- Aggregation newAgg = aggCall.getAggregation();
+ SqlAggFunction newAgg = aggCall.getAggregation();
if (func.isCountDistinct()) {
newAgg = createHyperLogLogAggFunction(fieldType);
} else if (func.isCount()) {
- //newAgg = new SqlSumEmptyIsZeroAggFunction(fieldType);
newAgg = SqlStdOperatorTable.SUM0;
}
// rebuild aggregate call
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, newAgg.getName());
-
- // To make sure specified type matches the inferReturnType, or otherwise
- // there will be assertion failure in optiq
- // The problem is BIGINT != BIGINT NOT NULL
- // Details see https://github.scm.corp.ebay.com/Kylin/Kylin/issues/323
- SqlAggFunction aggFunction = (SqlAggFunction) newAggCall.getAggregation();
- AggCallBinding callBinding = newAggCall.createBinding(this);
- RelDataType inferReturnType = aggFunction.inferReturnType(callBinding);
-
- return new AggregateCall(newAgg, false, newArgList, inferReturnType, newAgg.getName());
+ return newAggCall;
}
- private Aggregation createHyperLogLogAggFunction(RelDataType returnType) {
+ private SqlAggFunction createHyperLogLogAggFunction(RelDataType returnType) {
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
SqlIdentifier sqlIdentifier = new SqlIdentifier("HLL_COUNT", new SqlParserPos(1, 1));
AggregateFunction aggFunction = AggregateFunctionImpl.create(HLLDistinctCountAggFunc.class);
@@ -333,16 +320,13 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
-
- EnumerableAggregateRel enumAggRel;
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
try {
- enumAggRel = new EnumerableAggregateRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), getChild(), this.groupSet, rewriteAggCalls);
+ return new EnumerableAggregate(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ sole(inputs), false, this.groupSet, this.groupSets, rewriteAggCalls);
} catch (InvalidRelException e) {
- throw new IllegalStateException("Can't create EnumerableAggregateRel!", e);
+ throw new IllegalStateException("Can't create EnumerableAggregate!", e);
}
-
- return enumAggRel.implement(implementor, pref);
}
@Override
@@ -357,7 +341,7 @@ public class OLAPAggregateRel extends AggregateRelBase implements OLAPRel, Enume
@Override
public boolean hasSubQuery() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
return olapChild.hasSubQuery();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 8e532a9..86473b5 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -26,10 +26,9 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.schema.OLAPSchema;
-import org.eigenbase.reltype.RelDataType;
-
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -38,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.metadata.filter.TupleFilter;
/**
- * @author xjiang
- *
*/
public class OLAPContext {
@@ -95,8 +92,7 @@ public class OLAPContext {
// query info
public OLAPSchema olapSchema = null;
- public OLAPTableScan firstTableScan = null; // to be fact table scan except
- // "select * from lookupTable"
+ public OLAPTableScan firstTableScan = null; // to be fact table scan except "select * from lookupTable"
public RelDataType olapRowType = null;
public boolean afterAggregate = false;
public boolean afterJoin = false;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0783e31..48c1075 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,41 +18,38 @@
package org.apache.kylin.query.relnode;
-import java.util.*;
-
-import com.google.common.collect.Sets;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableCalcRel;
-import net.hydromatic.optiq.runtime.SqlFunctions;
-
-import org.eigenbase.rel.FilterRelBase;
-import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexBuilder;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexDynamicParam;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexLocalRef;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexProgram;
-import org.eigenbase.rex.RexProgramBuilder;
-import org.eigenbase.rex.RexVisitorImpl;
-import org.eigenbase.sql.SqlKind;
-import org.eigenbase.sql.SqlOperator;
-import org.eigenbase.util.NlsString;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.kylin.metadata.model.TblColRef;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.calcite.adapter.enumerable.EnumerableCalc;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexLocalRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.rex.RexProgramBuilder;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.util.NlsString;
import org.apache.kylin.metadata.filter.CaseTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -62,11 +59,15 @@ import org.apache.kylin.metadata.filter.ExtractTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
/**
- * @author xjiang
*/
-public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableRel {
+public class OLAPFilterRel extends Filter implements OLAPRel {
private static class TupleFilterVisitor extends RexVisitorImpl<TupleFilter> {
@@ -211,9 +212,9 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
} else if (literalValue instanceof GregorianCalendar) {
GregorianCalendar g = (GregorianCalendar) literalValue;
strValue = "" + g.get(Calendar.YEAR) + "-" + normToTwoDigits(g.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(g.get(Calendar.DAY_OF_MONTH));
- } else if (literalValue instanceof SqlFunctions.TimeUnitRange) {
+ } else if (literalValue instanceof TimeUnitRange) {
// Extract(x from y) in where clause
- strValue = ((SqlFunctions.TimeUnitRange) literalValue).name();
+ strValue = ((TimeUnitRange) literalValue).name();
} else if (literalValue == null) {
strValue = null;
} else {
@@ -247,13 +248,13 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
}
@Override
- public FilterRelBase copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
+ public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
return new OLAPFilterRel(getCluster(), traitSet, input, condition);
}
@Override
public void implementOLAP(OLAPImplementor implementor) {
- implementor.visitChild(getChild(), this);
+ implementor.visitChild(getInput(), this);
this.columnRowType = buildColumnRowType();
this.context = implementor.getContext();
@@ -265,7 +266,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
}
private ColumnRowType buildColumnRowType() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
return inputColumnRowType;
}
@@ -296,23 +297,22 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
// keep it for having clause
RexBuilder rexBuilder = getCluster().getRexBuilder();
- RelDataType inputRowType = getChild().getRowType();
+ RelDataType inputRowType = getInput().getRowType();
RexProgramBuilder programBuilder = new RexProgramBuilder(inputRowType, rexBuilder);
programBuilder.addIdentity();
programBuilder.addCondition(this.condition);
RexProgram program = programBuilder.getProgram();
- EnumerableCalcRel enumCalcRel = new EnumerableCalcRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), getChild(), this.rowType, program, ImmutableList.<RelCollation> of());
-
- return enumCalcRel.implement(implementor, pref);
+ return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ sole(inputs), program, ImmutableList.<RelCollation> of());
}
@Override
public void implementRewrite(RewriteImplementor implementor) {
- implementor.visitChild(this, getChild());
+ implementor.visitChild(this, getInput());
this.rowType = this.deriveRowType();
this.columnRowType = buildColumnRowType();
@@ -330,7 +330,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
@Override
public boolean hasSubQuery() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
return olapChild.hasSubQuery();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index c51196a..8de155c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -25,42 +25,43 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import net.hydromatic.linq4j.expressions.Blocks;
-import net.hydromatic.linq4j.expressions.Expressions;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableJoinRel;
-import net.hydromatic.optiq.rules.java.PhysType;
-import net.hydromatic.optiq.rules.java.PhysTypeImpl;
-
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableJoin;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.adapter.enumerable.PhysType;
+import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.InvalidRelException;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.schema.OLAPTable;
-import org.eigenbase.rel.InvalidRelException;
-import org.eigenbase.rel.JoinInfo;
-import org.eigenbase.rel.JoinRelType;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory.FieldInfoBuilder;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.eigenbase.reltype.RelDataTypeFieldImpl;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.sql.SqlKind;
-import org.eigenbase.util.ImmutableIntList;
import com.google.common.base.Preconditions;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TblColRef;
/**
- * @author xjiang
*/
-public class OLAPJoinRel extends EnumerableJoinRel implements OLAPRel {
+public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
private final static String[] COLUMN_ARRAY_MARKER = new String[0];
@@ -79,7 +80,7 @@ public class OLAPJoinRel extends EnumerableJoinRel implements OLAPRel {
}
@Override
- public EnumerableJoinRel copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, //
+ public EnumerableJoin copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, //
JoinRelType joinType, boolean semiJoinDone) {
final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
@@ -87,8 +88,7 @@ public class OLAPJoinRel extends EnumerableJoinRel implements OLAPRel {
try {
return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped);
} catch (InvalidRelException e) {
- // Semantic error not possible. Must be a bug. Convert to
- // internal error.
+ // Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
}
}
@@ -227,18 +227,25 @@ public class OLAPJoinRel extends EnumerableJoinRel implements OLAPRel {
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- Result result = null;
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
if (this.hasSubQuery) {
- result = super.implement(implementor, pref);
+ try {
+ return new EnumerableJoin(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, joinType, variablesStopped);
+ } catch (InvalidRelException e) {
+ throw new IllegalStateException("Can't create EnumerableJoin!", e);
+ }
} else {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- RelOptTable factTable = context.firstTableScan.getTable();
- result = implementor.result(physType, Blocks.toBlock(Expressions.call(factTable.getExpression(OLAPTable.class), "executeIndexQuery", implementor.getRootExpression(), Expressions.constant(context.id))));
+ return this;
}
+ }
- return result;
+ @Override
+ public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
+ PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
+ RelOptTable factTable = context.firstTableScan.getTable();
+ MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeIndexQuery", implementor.getRootExpression(), Expressions.constant(context.id));
+ return implementor.result(physType, Blocks.toBlock(exprCall));
}
@Override
@@ -259,7 +266,7 @@ public class OLAPJoinRel extends EnumerableJoinRel implements OLAPRel {
List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
String fieldName = rewriteField.getKey();
- if (this.rowType.getField(fieldName, true) == null) {
+ if (this.rowType.getField(fieldName, true, false) == null) {
RelDataType fieldType = rewriteField.getValue();
RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, paramIndex++, fieldType);
newFieldList.add(newField);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index e638fe7..a31cc21 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -20,30 +20,25 @@ package org.apache.kylin.query.relnode;
import java.util.List;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableLimitRel;
-
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.rel.RelWriter;
-import org.eigenbase.rel.SingleRel;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexNode;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableLimit;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.SingleRel;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
import com.google.common.base.Preconditions;
/**
- *
- * @author xjiang
- *
*/
-public class OLAPLimitRel extends SingleRel implements OLAPRel, EnumerableRel {
+public class OLAPLimitRel extends SingleRel implements OLAPRel {
private final RexNode localOffset; // avoid same name in parent class
private final RexNode localFetch; // avoid same name in parent class
@@ -75,7 +70,7 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel, EnumerableRel {
@Override
public void implementOLAP(OLAPImplementor implementor) {
- implementor.visitChild(getChild(), this);
+ implementor.visitChild(getInput(), this);
this.columnRowType = buildColumnRowType();
@@ -86,30 +81,23 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel, EnumerableRel {
}
private ColumnRowType buildColumnRowType() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
return inputColumnRowType;
}
@Override
public void implementRewrite(RewriteImplementor implementor) {
- implementor.visitChild(this, getChild());
+ implementor.visitChild(this, getInput());
this.rowType = this.deriveRowType();
this.columnRowType = buildColumnRowType();
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- OLAPRel childRel = (OLAPRel) getChild();
- childRel.replaceTraitSet(EnumerableConvention.INSTANCE);
-
- EnumerableLimitRel enumLimit = new EnumerableLimitRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), getChild(), localOffset, localFetch);
- Result res = enumLimit.implement(implementor, pref);
-
- childRel.replaceTraitSet(CONVENTION);
-
- return res;
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
+ return new EnumerableLimit(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ sole(inputs), localOffset, localFetch);
}
@Override
@@ -124,7 +112,7 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel, EnumerableRel {
@Override
public boolean hasSubQuery() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
return olapChild.hasSubQuery();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f4be1526/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 1215f9e..8f80962 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -25,44 +25,39 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import net.hydromatic.optiq.rules.java.EnumerableConvention;
-import net.hydromatic.optiq.rules.java.EnumerableRel;
-import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
-import net.hydromatic.optiq.rules.java.JavaRules.EnumerableCalcRel;
-
+import org.apache.calcite.adapter.enumerable.EnumerableCalc;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTrait;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlCaseOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
-import org.eigenbase.rel.ProjectRelBase;
-import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory.FieldInfoBuilder;
-import org.eigenbase.reltype.RelDataTypeField;
-import org.eigenbase.reltype.RelDataTypeFieldImpl;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexProgram;
-import org.eigenbase.sql.SqlOperator;
-import org.eigenbase.sql.fun.SqlCaseOperator;
-import org.eigenbase.sql.fun.SqlStdOperatorTable;
-import org.eigenbase.sql.validate.SqlUserDefinedFunction;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
/**
- *
- * @author xjiang
- *
*/
-public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, EnumerableRel {
+public class OLAPProjectRel extends Project implements OLAPRel {
private OLAPContext context;
private List<RexNode> rewriteProjects;
@@ -98,13 +93,13 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
}
@Override
- public ProjectRelBase copy(RelTraitSet traitSet, RelNode child, List<RexNode> exps, RelDataType rowType) {
+ public Project copy(RelTraitSet traitSet, RelNode child, List<RexNode> exps, RelDataType rowType) {
return new OLAPProjectRel(getCluster(), traitSet, child, exps, rowType, this.flags);
}
@Override
public void implementOLAP(OLAPImplementor implementor) {
- implementor.visitChild(getChild(), this);
+ implementor.visitChild(getInput(), this);
this.context = implementor.getContext();
this.hasJoin = context.hasJoin;
@@ -117,7 +112,7 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
private ColumnRowType buildColumnRowType() {
List<TblColRef> columns = new ArrayList<TblColRef>();
List<Set<TblColRef>> sourceColumns = new ArrayList<Set<TblColRef>>();
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
for (int i = 0; i < this.rewriteProjects.size(); i++) {
RexNode rex = this.rewriteProjects.get(i);
@@ -125,12 +120,28 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
String fieldName = columnField.getName();
Set<TblColRef> sourceCollector = new HashSet<TblColRef>();
TblColRef column = translateRexNode(rex, inputColumnRowType, fieldName, sourceCollector);
+ if (column == null)
+ throw new IllegalStateException("No TblColRef found in " + rex);
columns.add(column);
sourceColumns.add(sourceCollector);
}
return new ColumnRowType(columns, sourceColumns);
}
+ private TblColRef translateFirstRexInputRef(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
+ for (RexNode operand : call.getOperands()) {
+ if (operand instanceof RexInputRef) {
+ return translateRexInputRef((RexInputRef) operand, inputColumnRowType, fieldName, sourceCollector);
+ }
+ if (operand instanceof RexCall) {
+ TblColRef r = translateFirstRexInputRef((RexCall) operand, inputColumnRowType, fieldName, sourceCollector);
+ if (r != null)
+ return r;
+ }
+ }
+ return null;
+ }
+
private TblColRef translateRexNode(RexNode rexNode, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
TblColRef column = null;
if (rexNode instanceof RexInputRef) {
@@ -175,16 +186,10 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
private TblColRef translateRexCall(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
SqlOperator operator = call.getOperator();
if (operator == SqlStdOperatorTable.EXTRACT_DATE) {
- List<RexNode> extractDateOps = call.getOperands();
- RexCall reinterpret = (RexCall) extractDateOps.get(1);
- List<RexNode> reinterpretOps = reinterpret.getOperands();
- RexInputRef inputRef = (RexInputRef) reinterpretOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
} else if (operator instanceof SqlUserDefinedFunction) {
if (operator.getName().equals("QUARTER")) {
- List<RexNode> quaterOps = call.getOperands();
- RexInputRef inputRef = (RexInputRef) quaterOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
}
} else if (operator instanceof SqlCaseOperator) {
for (RexNode operand : call.getOperands()) {
@@ -202,24 +207,21 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
}
@Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- EnumerableCalcRel enumCalcRel;
-
- RelNode child = getChild();
- if (child instanceof OLAPFilterRel) {
+ public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
+ if (getInput() instanceof OLAPFilterRel) {
// merge project & filter
- OLAPFilterRel filter = (OLAPFilterRel) getChild();
- RexProgram program = RexProgram.create(filter.getChild().getRowType(), this.rewriteProjects, filter.getCondition(), this.rowType, getCluster().getRexBuilder());
-
- enumCalcRel = new EnumerableCalcRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), filter.getChild(), this.rowType, program, ImmutableList.<RelCollation> of());
+ OLAPFilterRel filter = (OLAPFilterRel) getInput();
+ RelNode inputOfFilter = inputs.get(0).getInput(0);
+ RexProgram program = RexProgram.create(inputOfFilter.getRowType(), this.rewriteProjects, filter.getCondition(), this.rowType, getCluster().getRexBuilder());
+ return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ inputOfFilter, program, ImmutableList.<RelCollation> of());
} else {
- // keep project for tablescan
- RexProgram program = RexProgram.create(child.getRowType(), this.rewriteProjects, null, this.rowType, getCluster().getRexBuilder());
-
- enumCalcRel = new EnumerableCalcRel(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), child, this.rowType, program, ImmutableList.<RelCollation> of());
+ // keep project for table scan
+ EnumerableRel input = sole(inputs);
+ RexProgram program = RexProgram.create(input.getRowType(), this.rewriteProjects, null, this.rowType, getCluster().getRexBuilder());
+ return new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
+ input, program, ImmutableList.<RelCollation> of());
}
-
- return enumCalcRel.implement(implementor, pref);
}
@Override
@@ -229,7 +231,7 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
@Override
public void implementRewrite(RewriteImplementor implementor) {
- implementor.visitChild(this, getChild());
+ implementor.visitChild(this, getInput());
this.rewriting = true;
@@ -243,7 +245,7 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
int paramIndex = this.rowType.getFieldList().size();
List<RelDataTypeField> newFieldList = new LinkedList<RelDataTypeField>();
List<RexNode> newExpList = new LinkedList<RexNode>();
- ColumnRowType inputColumnRowType = ((OLAPRel) getChild()).getColumnRowType();
+ ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
for (Map.Entry<String, RelDataType> rewriteField : this.context.rewriteFields.entrySet()) {
String rewriteFieldName = rewriteField.getKey();
@@ -256,7 +258,7 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
RelDataTypeField newField = new RelDataTypeFieldImpl(rewriteFieldName, paramIndex++, fieldType);
newFieldList.add(newField);
// new project
- RelDataTypeField inputField = getChild().getRowType().getFieldList().get(inputIndex);
+ RelDataTypeField inputField = getInput().getRowType().getFieldList().get(inputIndex);
RexInputRef newFieldRef = new RexInputRef(inputField.getIndex(), inputField.getType());
newExpList.add(newFieldRef);
}
@@ -289,7 +291,7 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
@Override
public boolean hasSubQuery() {
- OLAPRel olapChild = (OLAPRel) getChild();
+ OLAPRel olapChild = (OLAPRel) getInput();
return olapChild.hasSubQuery();
}
[15/50] [abbrv] incubator-kylin git commit: KYLIN-839 saving memory
for snapshot tables
Posted by li...@apache.org.
KYLIN-839 saving memory for snapshot tables
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9a467fd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9a467fd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9a467fd3
Branch: refs/heads/0.7
Commit: 9a467fd3f080a8d22d5b57bb397ceec58c0bd808
Parents: 08c8e91
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jun 29 15:13:34 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jun 29 15:13:34 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/lookup/SnapshotTable.java | 117 +++++++++++--------
1 file changed, 71 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9a467fd3/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 4764dd0..130a4c2 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -18,18 +18,11 @@
package org.apache.kylin.dict.lookup;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.fs.Path;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.dict.Dictionary;
@@ -38,6 +31,13 @@ import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.metadata.model.TableDesc;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
/**
* @author yangli9
*/
@@ -51,7 +51,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
@JsonProperty("useDictionary")
private boolean useDictionary;
- private ArrayList<String[]> rows;
+ private ArrayList<int[]> rowIndices;
+ private Dictionary<String> dict;
// default constructor for JSON serialization
public SnapshotTable() {
@@ -69,16 +70,33 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
int maxIndex = tableDesc.getMaxColumnIndex();
+ TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+
TableReader reader = table.getReader();
- ArrayList<String[]> allRows = new ArrayList<String[]>();
while (reader.next()) {
String[] row = reader.getRow();
if (row.length <= maxIndex) {
throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row));
}
- allRows.add(row);
+
+ for (String cell : row) {
+ b.addValue(cell);
+ }
}
- this.rows = allRows;
+
+ this.dict = b.build(0);
+
+ reader = table.getReader();
+ ArrayList<int[]> allRowIndices = new ArrayList<int[]>();
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ int[] rowIndex = new int[row.length];
+ for (int i = 0; i < row.length; i++) {
+ rowIndex[i] = dict.getIdFromValue(row[i]);
+ }
+ allRowIndices.add(rowIndex);
+ }
+ this.rowIndices = allRowIndices;
}
public String getResourcePath() {
@@ -98,12 +116,17 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
@Override
public boolean next() throws IOException {
i++;
- return i < rows.size();
+ return i < rowIndices.size();
}
@Override
public String[] getRow() {
- return rows.get(i);
+ int[] rowIndex = rowIndices.get(i);
+ String[] row = new String[rowIndex.length];
+ for (int x = 0; x < row.length; x++) {
+ row[x] = dict.getValueFromId(rowIndex[x]);
+ }
+ return row;
}
@Override
@@ -134,9 +157,9 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
*/
@Override
public int hashCode() {
- int[] parts = new int[this.rows.size()];
+ int[] parts = new int[this.rowIndices.size()];
for (int i = 0; i < parts.length; ++i)
- parts[i] = Arrays.hashCode(this.rows.get(i));
+ parts[i] = Arrays.hashCode(this.rowIndices.get(i));
return Arrays.hashCode(parts);
}
@@ -147,76 +170,78 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
SnapshotTable that = (SnapshotTable) o;
//compare row by row
- if (this.rows.size() != that.rows.size())
+ if (this.rowIndices.size() != that.rowIndices.size())
return false;
- for (int i = 0; i < this.rows.size(); ++i) {
- if (!ArrayUtils.isEquals(this.rows.get(i), that.rows.get(i)))
+ for (int i = 0; i < this.rowIndices.size(); ++i) {
+ if (!ArrayUtils.isEquals(this.rowIndices.get(i), that.rowIndices.get(i)))
return false;
}
return true;
}
void writeData(DataOutput out) throws IOException {
- out.writeInt(rows.size());
- if (rows.size() > 0) {
- int n = rows.get(0).length;
+ out.writeInt(rowIndices.size());
+ if (rowIndices.size() > 0) {
+ int n = rowIndices.get(0).length;
out.writeInt(n);
if (this.useDictionary == true) {
- Dictionary<String> dict = buildDictionary();
dict.write(out);
- for (int i = 0; i < rows.size(); i++) {
- String[] row = rows.get(i);
+ for (int i = 0; i < rowIndices.size(); i++) {
+ int[] row = rowIndices.get(i);
for (int j = 0; j < n; j++) {
- out.writeInt(dict.getIdFromValue(row[j]));
+ out.writeInt(row[j]);
}
}
} else {
- for (int i = 0; i < rows.size(); i++) {
- String[] row = rows.get(i);
+ for (int i = 0; i < rowIndices.size(); i++) {
+ int[] row = rowIndices.get(i);
for (int j = 0; j < n; j++) {
- out.writeUTF(row[j]);
+ out.writeUTF(dict.getValueFromId(row[j]));
}
}
}
}
}
- Dictionary<String> buildDictionary() {
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- for (String[] row : rows) {
- for (String cell : row) {
- b.addValue(cell);
- }
- }
- TrieDictionary<String> dict = b.build(0);
- return dict;
- }
-
void readData(DataInput in) throws IOException {
int rowNum = in.readInt();
- rows = new ArrayList<String[]>(rowNum);
if (rowNum > 0) {
int n = in.readInt();
if (this.useDictionary == true) {
- Dictionary<String> dict = new TrieDictionary<String>();
+ this.dict = new TrieDictionary<String>();
dict.readFields(in);
+
for (int i = 0; i < rowNum; i++) {
- String[] row = new String[n];
- rows.add(row);
+ int[] row = new int[n];
+ this.rowIndices.add(row);
for (int j = 0; j < n; j++) {
- row[j] = dict.getValueFromId(in.readInt());
+ row[j] = in.readInt();
}
}
} else {
+ List<String[]> rows = new ArrayList<String[]>(rowNum);
+ ArrayList<int[]> allRowIndices = new ArrayList<int[]>();
+ TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+
for (int i = 0; i < rowNum; i++) {
String[] row = new String[n];
rows.add(row);
for (int j = 0; j < n; j++) {
row[j] = in.readUTF();
+ b.addValue(row[j]);
+ }
+ }
+ this.dict = b.build(0);
+ for (String[] row : rows) {
+ int[] rowIndex = new int[n];
+ for (int i = 0; i < n; i++) {
+ rowIndex[i] = dict.getIdFromValue(row[i]);
}
+ allRowIndices.add(rowIndex);
}
+ this.rowIndices = allRowIndices;
}
}
}