You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/10/24 15:58:21 UTC
[01/12] kylin git commit: Throw exception from pushdown instead of
exception from calcite
Repository: kylin
Updated Branches:
refs/heads/2.2.x fb2580808 -> dd0630775
Throw exception from pushdown instead of exception from calcite
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da0d1535
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da0d1535
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da0d1535
Branch: refs/heads/2.2.x
Commit: da0d1535c5c6cd682f3da98cd6961a271d5aaca5
Parents: 53cad89
Author: nichunen <ch...@kyligence.io>
Authored: Wed Oct 18 20:31:54 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Wed Oct 18 20:31:54 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/QueryService.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/da0d1535/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index ddb805c..c6a9b76 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -864,8 +864,9 @@ public class QueryService extends BasicService {
r = PushDownUtil.tryPushDownSelectQuery(sqlRequest.getProject(), correctedSql, conn.getSchema(),
sqlException);
} catch (Exception e2) {
- //exception in pushdown engine will be printed, but we'll not re-throw it, we'll
- logger.info("pushdown engine failed current query too", e2);
+ logger.error("pushdown engine failed current query too", e2);
+ //exception in pushdown, throw it instead of exception in calcite
+ throw e2;
}
if (r == null)
[04/12] kylin git commit: minor, support real admin ldap-group name
Posted by li...@apache.org.
minor, support real admin ldap-group name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f54daca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f54daca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f54daca
Branch: refs/heads/2.2.x
Commit: 6f54dacaee1dad1df53d28530abdce9ed0e7d49f
Parents: 227668a
Author: lidongsjtu <li...@apache.org>
Authored: Thu Oct 19 22:32:41 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Oct 19 23:12:42 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/security/AuthoritiesPopulator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f54daca/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 592791c..dbe2d39 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -49,7 +49,7 @@ public class AuthoritiesPopulator extends DefaultLdapAuthoritiesPopulator {
*/
public AuthoritiesPopulator(ContextSource contextSource, String groupSearchBase, String adminRole, String defaultRole) {
super(contextSource, groupSearchBase);
- this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole);
+ this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole.toUpperCase()); // spring will convert group names to uppercase by default
String[] defaultRoles = StringUtils.split(defaultRole, ",");
if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER)) {
[10/12] kylin git commit: Support function with name percentile_approx
Posted by li...@apache.org.
Support function with name percentile_approx
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d79660c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d79660c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d79660c0
Branch: refs/heads/2.2.x
Commit: d79660c0b29a8029a8dd0369f3d05d20e8b875ca
Parents: 0994a57
Author: nichunen <ch...@kyligence.io>
Authored: Tue Oct 24 19:12:00 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Tue Oct 24 19:12:00 2017 +0800
----------------------------------------------------------------------
.../percentile/PercentileMeasureType.java | 8 ++++++--
.../resources/query/sql_percentile/query01.sql | 18 ++++++++++++++++++
.../resources/query/sql_percentile/query02.sql | 20 ++++++++++++++++++++
3 files changed, 44 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79660c0/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileMeasureType.java
index 45ebe89..d95a054 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileMeasureType.java
@@ -36,6 +36,7 @@ public class PercentileMeasureType extends MeasureType<PercentileCounter> {
// compression ratio saved in DataType.precision
private final DataType dataType;
public static final String FUNC_PERCENTILE = "PERCENTILE";
+ public static final String FUNC_PERCENTILE_APPROX = "PERCENTILE_APPROX";
public static final String DATATYPE_PERCENTILE = "percentile";
public PercentileMeasureType(String funcName, DataType dataType) {
@@ -71,7 +72,8 @@ public class PercentileMeasureType extends MeasureType<PercentileCounter> {
PercentileCounter current = new PercentileCounter(dataType.getPrecision());
@Override
- public PercentileCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public PercentileCounter valueOf(String[] values, MeasureDesc measureDesc,
+ Map<TblColRef, Dictionary<String>> dictionaryMap) {
PercentileCounter counter = current;
counter.clear();
for (String v : values) {
@@ -93,7 +95,9 @@ public class PercentileMeasureType extends MeasureType<PercentileCounter> {
return true;
}
- static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.<String, Class<?>> of(PercentileMeasureType.FUNC_PERCENTILE, PercentileAggFunc.class);
+ static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.<String, Class<?>> of(
+ PercentileMeasureType.FUNC_PERCENTILE, PercentileAggFunc.class,
+ PercentileMeasureType.FUNC_PERCENTILE_APPROX, PercentileAggFunc.class);
@Override
public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79660c0/kylin-it/src/test/resources/query/sql_percentile/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_percentile/query01.sql b/kylin-it/src/test/resources/query/sql_percentile/query01.sql
index 4f6d573..35fcb93 100644
--- a/kylin-it/src/test/resources/query/sql_percentile/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_percentile/query01.sql
@@ -1,2 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select seller_id, percentile(price, 0.5) from test_kylin_fact
group by seller_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79660c0/kylin-it/src/test/resources/query/sql_percentile/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_percentile/query02.sql b/kylin-it/src/test/resources/query/sql_percentile/query02.sql
new file mode 100644
index 0000000..e78b985
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_percentile/query02.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select seller_id, percentile_approx(price, 0.5) from test_kylin_fact
+group by seller_id
\ No newline at end of file
[09/12] kylin git commit: minor,kylin acl debug (#2876)
Posted by li...@apache.org.
minor,kylin acl debug (#2876)
* minor,kylin acl debug
resove conflict
* minor,kylin remove console
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0994a57b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0994a57b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0994a57b
Branch: refs/heads/2.2.x
Commit: 0994a57b922854ad63bb79cbae77d1ea44c3824d
Parents: a65c384
Author: luguosheng1314 <55...@qq.com>
Authored: Tue Oct 24 02:59:52 2017 -0500
Committer: GitHub <no...@github.com>
Committed: Tue Oct 24 02:59:52 2017 -0500
----------------------------------------------------------------------
webapp/app/js/controllers/models.js | 2 +-
webapp/app/js/listeners.js | 10 +++++++---
webapp/app/partials/common/access.html | 12 ++++++------
webapp/app/partials/cubes/cubes.html | 6 +++---
4 files changed, 17 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/js/controllers/models.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/models.js b/webapp/app/js/controllers/models.js
index 1cc0c84..ba7e3f6 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -132,7 +132,7 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
var modelstate=false;
var i=0;
- CubeService.list({modelName:model.name}, function (_cubes) {
+ CubeService.list({modelName:model.name,projectName:$scope.projectModel.selectedProject}, function (_cubes) {
model.cubes = _cubes;
if (model.cubes.length != 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/js/listeners.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/listeners.js b/webapp/app/js/listeners.js
index 959cfca..9d9d473 100644
--- a/webapp/app/js/listeners.js
+++ b/webapp/app/js/listeners.js
@@ -17,14 +17,18 @@
*/
KylinApp.run(function ($rootScope, $http, $location, UserService, AuthenticationService, MessageService, $cookieStore, ProjectService, ProjectModel, AccessService, SweetAlert, loadingRequest) {
-
$rootScope.permissions = {
READ: {name: 'QUERY', value: 'READ', mask: 1},
- MANAGEMENT: {name: 'EDIT', value: 'MANAGEMENT', mask: 32},
+ MANAGEMENT: {name: 'MANAGEMENT', value: 'MANAGEMENT', mask: 32},
OPERATION: {name: 'OPERATION', value: 'OPERATION', mask: 64},
ADMINISTRATION: {name: 'ADMIN', value: 'ADMINISTRATION', mask: 16}
};
-
+ $rootScope.permissionsSorted = [
+ {name: 'QUERY', value: 'READ', mask: 1},
+ {name: 'OPERATION', value: 'OPERATION', mask: 64},
+ {name: 'MANAGEMENT', value: 'MANAGEMENT', mask: 32},
+ {name: 'ADMIN', value: 'ADMINISTRATION', mask: 16}
+ ];
$rootScope.$on("$routeChangeStart", function () {
AuthenticationService.ping(function (data) {
UserService.setCurUser(data);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/partials/common/access.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/common/access.html b/webapp/app/partials/common/access.html
index ee41e64..9f8d6a5 100644
--- a/webapp/app/partials/common/access.html
+++ b/webapp/app/partials/common/access.html
@@ -50,7 +50,7 @@
<td >
<label><b>Permission </b> </label>
<select ng-model="newAccess.permission" ng-init="newAccess.permission=permissions.READ.value"
- ng-options="permission.value as permission.name for (name, permission) in permissions">
+ ng-options="permission.value as permission.name for (name, permission) in permissionsSorted">
<option value=""></option>
</select>
</td>
@@ -87,14 +87,14 @@
<span ng-if="accessEntity.sid.grantedAuthority">Role</span>
</td>
<td>
- <span ng-if="accessEntity.permission.mask == 1">QUERY</span>
- <span ng-if="accessEntity.permission.mask == 32">EDIT</span>
- <span ng-if="accessEntity.permission.mask == 64">OPERATION</span>
- <span ng-if="accessEntity.permission.mask == 16">ADMIN</span>
+ <span ng-if="accessEntity.permission.mask == 1">QUERY</span>
+ <span ng-if="accessEntity.permission.mask == 64">OPERATION</span>
+ <span ng-if="accessEntity.permission.mask == 32">MANAGEMENT</span>
+ <span ng-if="accessEntity.permission.mask == 16">ADMIN</span>
</td>
<td ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission('project',entity, 16)">
<select ng-model="accessEntity.newPermission" ng-init="newAccess.permission=permissions.READ.value"
- ng-options="permission.value as permission.name for (name, permission) in permissions">
+ ng-options="permission.value as permission.name for (name, permission) in permissionsSorted">
<option value="">-- select access --</option>
</select>
<button class="btn btn-default btn-xs" ng-disabled="!accessEntity.newPermission"
http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 8528968..38c9342 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -53,7 +53,7 @@
class="fa fa-sort-desc"></i>
</th>
<th>Actions</th>
- <th ng-if="userService.hasRole('ROLE_ADMIN')">Admins</th>
+ <th>Admins</th>
</tr>
</thead>
<!--Body-->
@@ -106,8 +106,8 @@
</ul>
</div>
</td>
- <td ng-if="userService.hasRole('ROLE_ADMIN')">
- <div ng-click="$event.stopPropagation();" class="btn-group">
+ <td >
+ <div ng-click="$event.stopPropagation();" class="btn-group" ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask)">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" ng-click="listAccess(cube, 'CubeInstance')">
Action <span class="ace-icon fa fa-caret-down icon-on-right"></span>
</button>
[05/12] kylin git commit: KYLIN-2945 global dict specific info change
to use relative path
Posted by li...@apache.org.
KYLIN-2945 global dict specific info change to use relative path
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c88db54
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c88db54
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c88db54
Branch: refs/heads/2.2.x
Commit: 1c88db542a6f2bb4282282510ffdbe3cca728d98
Parents: 6f54dac
Author: Yifei Wu <yi...@kyligencedeMacBook-Pro.local>
Authored: Wed Oct 18 16:15:37 2017 +0800
Committer: Yifei Wu <yi...@Yifei.local>
Committed: Fri Oct 20 10:11:38 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 5 +-
.../org/apache/kylin/common/StorageURL.java | 2 +-
.../common/persistence/FileResourceStore.java | 6 +-
.../apache/kylin/dict/AppendTrieDictionary.java | 53 +-
.../global/AppendTrieDictionaryBuilder.java | 2 +-
.../kylin/dict/global/GlobalDictHDFSStore.java | 6 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 577 +++++++++++++++++++
.../dict/global/AppendTrieDictionaryTest.java | 525 -----------------
.../engine/mr/common/AbstractHadoopJob.java | 9 +-
.../spark/SparkBatchCubingJobBuilder2.java | 9 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 2 +-
.../hdfs/IdentifierFileResourceStore.java | 51 ++
12 files changed, 704 insertions(+), 543 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ee05d69..56c7b1f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -29,12 +29,12 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.lock.DistributedLockFactory;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.ZooKeeperUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -205,7 +205,7 @@ abstract public class KylinConfigBase implements Serializable {
// make sure path is qualified
try {
- FileSystem fs = path.getFileSystem(new Configuration());
+ FileSystem fs = path.getFileSystem(HadoopUtil.getCurrentConfiguration());
path = fs.makeQualified(path);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -272,6 +272,7 @@ abstract public class KylinConfigBase implements Serializable {
r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
+ r.put("ifile", "org.apache.kylin.storage.hdfs.IdentifierFileResourceStore");
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html
return r;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/StorageURL.java b/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
index cebbc27..2d3e04c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
+++ b/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
@@ -99,7 +99,7 @@ public class StorageURL {
this.params = ImmutableMap.copyOf(m);
}
- private StorageURL(String identifier, String scheme, Map<String, String> params) {
+ public StorageURL(String identifier, String scheme, Map<String, String> params) {
this.identifier = identifier;
this.scheme = scheme;
this.params = ImmutableMap.copyOf(params);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index c32556a..12c8aba 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -45,12 +45,16 @@ public class FileResourceStore extends ResourceStore {
public FileResourceStore(KylinConfig kylinConfig) {
super(kylinConfig);
- root = new File(kylinConfig.getMetadataUrl().getIdentifier()).getAbsoluteFile();
+ root = new File(getPath(kylinConfig)).getAbsoluteFile();
if (root.exists() == false)
throw new IllegalArgumentException(
"File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}
+ protected String getPath(KylinConfig kylinConfig) {
+ return kylinConfig.getMetadataUrl().getIdentifier();
+ }
+
@Override
protected NavigableSet<String> listResourcesImpl(String folderPath) throws IOException {
synchronized (FileResourceStore.class) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 1855274..9e68eb4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -60,19 +60,20 @@ import com.google.common.cache.RemovalNotification;
*
* @author sunyerui
*/
-@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
+@SuppressWarnings({"rawtypes", "unchecked", "serial"})
public class AppendTrieDictionary<T> extends CacheDictionary<T> {
- public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict"
+ public static final byte[] HEAD_MAGIC = new byte[]{0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74}; // "AppendTrieDict"
public static final int HEAD_SIZE_I = HEAD_MAGIC.length;
private static final Logger logger = LoggerFactory.getLogger(AppendTrieDictionary.class);
+ transient private Boolean isSaveAbsolutePath = false;
transient private String baseDir;
transient private GlobalDictMetadata metadata;
transient private LoadingCache<AppendDictSliceKey, AppendDictSlice> dictCache;
public void init(String baseDir) throws IOException {
- this.baseDir = baseDir;
- final GlobalDictStore globalDictStore = new GlobalDictHDFSStore(baseDir);
+ this.baseDir = convertToAbsolutePath(baseDir);
+ final GlobalDictStore globalDictStore = new GlobalDictHDFSStore(this.baseDir);
Long[] versions = globalDictStore.listAllVersions();
if (versions.length == 0) {
@@ -151,7 +152,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
@Override
public void write(DataOutput out) throws IOException {
- out.writeUTF(baseDir);
+ out.writeUTF(convertToRelativePath(baseDir));
}
@Override
@@ -190,4 +191,46 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
public boolean contains(Dictionary other) {
return false;
}
+
+ /**
+ * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
+ * if pass a absolute path, it may produce some problems like cannot find global dict after migration.
+ * so convert to relative path can avoid it and be better to maintain flexibility.
+ *
+ */
+ private String convertToRelativePath(String path) {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
+ if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
+ return path.substring(hdfsWorkingDir.length());
+ }
+ return path;
+ }
+
+ private String convertToAbsolutePath(String path) {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Path basicPath = new Path(path);
+ if (basicPath.toUri().getScheme() == null)
+ return kylinConfig.getHdfsWorkingDirectory() + path;
+
+ String[] paths = path.split("/resources/GlobalDict/");
+ if (paths.length == 2)
+ return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];
+
+ paths = path.split("/resources/SegmentDict/");
+ if (paths.length == 2) {
+ return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
+ } else {
+ throw new RuntimeException("the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
+ }
+ }
+
+ /**
+ * only for test
+ *
+ * @param flag
+ */
+ void setSaveAbsolutePath(Boolean flag) {
+ this.isSaveAbsolutePath = flag;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
index 54978c2..a961527 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
@@ -278,7 +278,7 @@ public class AppendTrieDictionaryBuilder {
}
// Only used for test
- void setMaxId(int id) {
+ public void setMaxId(int id) {
this.maxId = id;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
index bad427c..ec79f2c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
@@ -46,8 +46,8 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
static final Logger logger = LoggerFactory.getLogger(GlobalDictHDFSStore.class);
static final String V1_INDEX_NAME = ".index";
- static final String V2_INDEX_NAME = ".index_v2";
- static final String VERSION_PREFIX = "version_";
+ public static final String V2_INDEX_NAME = ".index_v2";
+ public static final String VERSION_PREFIX = "version_";
static final int BUFFER_SIZE = 8 * 1024 * 1024;
private final Path basePath;
@@ -295,7 +295,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
protected final FileSystem fs;
protected final Configuration conf;
- protected IndexFormatV1(FileSystem fs, Configuration conf) {
+ public IndexFormatV1(FileSystem fs, Configuration conf) {
this.fs = fs;
this.conf = conf;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
new file mode 100644
index 0000000..36ca66e
--- /dev/null
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -0,0 +1,577 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import static org.apache.kylin.dict.global.GlobalDictHDFSStore.V2_INDEX_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.dict.global.AppendDictSliceKey;
+import org.apache.kylin.dict.global.AppendTrieDictionaryBuilder;
+import org.apache.kylin.dict.global.GlobalDictHDFSStore;
+import org.apache.kylin.dict.global.GlobalDictMetadata;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
+ private static final String RESOURCE_DIR = "/dict/append_dict_test/" + UUID.randomUUID();
+ private static String BASE_DIR;
+ private static String LOCAL_BASE_DIR;
+
+ @Before
+ public void beforeTest() {
+ staticCreateTestMetadata();
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000");
+ BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
+ LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
+ }
+
+ @After
+ public void afterTest() {
+ cleanup();
+ staticCleanupTestMetadata();
+ }
+
+ private void cleanup() {
+ Path basePath = new Path(BASE_DIR);
+ try {
+ HadoopUtil.getFileSystem(basePath).delete(basePath, true);
+ } catch (IOException e) {
+ }
+ }
+
+ private static final String[] words = new String[]{"paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "字典", "字典树", "字母", // non-ascii characters
+ "", // empty
+ "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
+ "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk",
+ "paint", "tar", "try", // some dup
+ };
+
+ private AppendTrieDictionaryBuilder createBuilder() throws IOException {
+ int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
+ return new AppendTrieDictionaryBuilder(BASE_DIR, maxEntriesPerSlice, true);
+ }
+
+ @Test
+ public void testStringRepeatly() throws IOException {
+ ArrayList<String> list = new ArrayList<>();
+ Collections.addAll(list, words);
+ ArrayList<String> notfound = new ArrayList<>();
+ notfound.add("pa");
+ notfound.add("pars");
+ notfound.add("tri");
+ notfound.add("字");
+ for (int i = 0; i < 50; i++) {
+ testStringDictAppend(list, notfound, true);
+ //to speed up the test
+ cleanup();
+ }
+ }
+
+ @Test
+ public void testEnglishWords() throws Exception {
+ InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).txt");
+ ArrayList<String> str = loadStrings(is);
+ testStringDictAppend(str, null, false);
+ }
+
+ @Test
+ public void testCategoryNames() throws Exception {
+ InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
+ ArrayList<String> str = loadStrings(is);
+ testStringDictAppend(str, null, true);
+ }
+
+ private static ArrayList<String> loadStrings(InputStream is) throws Exception {
+ ArrayList<String> r = new ArrayList<String>();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+ try {
+ String word;
+ while ((word = reader.readLine()) != null) {
+ word = word.trim();
+ if (word.isEmpty() == false)
+ r.add(word);
+ }
+ } finally {
+ reader.close();
+ is.close();
+ }
+ return r;
+ }
+
+ @Ignore("need huge key set")
+ @Test
+ public void testHugeKeySet() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+
+ AppendTrieDictionary<String> dict = null;
+
+ InputStream is = new FileInputStream("src/test/resources/dict/huge_key");
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+ try {
+ String word;
+ while ((word = reader.readLine()) != null) {
+ word = word.trim();
+ if (!word.isEmpty())
+ builder.addValue(word);
+ }
+ } finally {
+ reader.close();
+ is.close();
+ }
+ dict = builder.build(0);
+ dict.dump(System.out);
+ }
+
+ private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList) throws IOException {
+ Random rnd = new Random(System.currentTimeMillis());
+ ArrayList<String> strList = new ArrayList<String>();
+ strList.addAll(list);
+ if (shuffleList) {
+ Collections.shuffle(strList, rnd);
+ }
+ BytesConverter converter = new StringBytesConverter();
+
+ AppendTrieDictionaryBuilder b = createBuilder();
+
+ TreeMap<Integer, String> checkMap = new TreeMap<>();
+ int firstAppend = rnd.nextInt(strList.size() / 2);
+ int secondAppend = firstAppend + rnd.nextInt((strList.size() - firstAppend) / 2);
+ int appendIndex = 0;
+ int checkIndex = 0;
+
+ for (; appendIndex < firstAppend; appendIndex++) {
+ b.addValue(strList.get(appendIndex));
+ }
+ AppendTrieDictionary<String> dict = b.build(0);
+ dict.dump(System.out);
+ for (; checkIndex < firstAppend; checkIndex++) {
+ String str = strList.get(checkIndex);
+ byte[] bytes = converter.convertToBytes(str);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ checkMap.put(id, str);
+ }
+
+ // reopen dict and append
+ b = createBuilder();
+
+ for (; appendIndex < secondAppend; appendIndex++) {
+ b.addValue(strList.get(appendIndex));
+ }
+ AppendTrieDictionary<String> newDict = b.build(0);
+ assert newDict.equals(dict);
+ dict = newDict;
+ dict.dump(System.out);
+ checkIndex = 0;
+ for (; checkIndex < secondAppend; checkIndex++) {
+ String str = strList.get(checkIndex);
+ byte[] bytes = converter.convertToBytes(str);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
+ if (checkIndex < firstAppend) {
+ assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
+ } else {
+ // check second append str, should be new id
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ checkMap.put(id, str);
+ }
+ }
+
+ // reopen dict and append rest str
+ b = createBuilder();
+
+ for (; appendIndex < strList.size(); appendIndex++) {
+ b.addValue(strList.get(appendIndex));
+ }
+ newDict = b.build(0);
+ assert newDict.equals(dict);
+ dict = newDict;
+ dict.dump(System.out);
+ checkIndex = 0;
+ for (; checkIndex < strList.size(); checkIndex++) {
+ String str = strList.get(checkIndex);
+ byte[] bytes = converter.convertToBytes(str);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
+ if (checkIndex < secondAppend) {
+ assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
+ } else {
+ // check third append str, should be new id
+ assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
+ checkMap.put(id, str);
+ }
+ }
+ if (notfound != null) {
+ for (String s : notfound) {
+ byte[] bytes = converter.convertToBytes(s);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertEquals(-1, id);
+ }
+ }
+
+ dict = testSerialize(dict, converter);
+ for (String str : strList) {
+ byte[] bytes = converter.convertToBytes(str);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
+ assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
+ }
+ }
+
+ private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict, BytesConverter converter) {
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dataout = new DataOutputStream(bout);
+ dict.write(dataout);
+ dataout.close();
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ DataInputStream datain = new DataInputStream(bin);
+ AppendTrieDictionary<String> r = new AppendTrieDictionary<String>();
+ r.readFields(datain);
+ datain.close();
+ return r;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testMaxInteger() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.setMaxId(Integer.MAX_VALUE - 2);
+ builder.addValue("a");
+ builder.addValue("ab");
+ builder.addValue("acd");
+ builder.addValue("ac");
+ AppendTrieDictionary dict = builder.build(0);
+ assertEquals(2147483646, dict.getIdFromValue("a", 0));
+ assertEquals(2147483647, dict.getIdFromValue("ab", 0));
+ assertEquals(-2147483647, dict.getIdFromValue("ac", 0));
+ assertEquals(-2147483648, dict.getIdFromValue("acd", 0));
+ }
+
+ @Ignore("Only occurred when value is very long (>8000 bytes)")
+ @Test
+ public void testSuperLongValue() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ String value = "a";
+ for (int i = 0; i < 10000; i++) {
+ value += "a";
+ try {
+ builder.addValue(value);
+ } catch (StackOverflowError e) {
+ System.out.println("\nstack overflow " + i);
+ throw e;
+ }
+ }
+ AppendTrieDictionary dictionary = builder.build(0);
+ dictionary.getMaxId();
+ }
+
+ @Test
+ public void testSplitContainSuperLongValue() throws IOException {
+ String superLongValue = "%5Cx1A%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%7E%29%5CxEF%5CxBF%5CxBD%5Cx1B+%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5Cx13%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5B";
+
+ createAppendTrieDict(Arrays.asList("a", superLongValue));
+ }
+
+ @Test
+ public void testSuperLongValueAsFileName() throws IOException {
+ String superLongValue = "%5Cx1A%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%7E%29%5CxEF%5CxBF%5CxBD%5Cx1B+%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5Cx13%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5B";
+
+ createAppendTrieDict(Arrays.asList("a", superLongValue));
+ }
+
+ @Test
+ public void testIllegalFileNameValue() throws IOException {
+ createAppendTrieDict(Arrays.asList("::", ":"));
+ }
+
+ @Test
+ public void testSkipAddValue() throws IOException {
+ createAppendTrieDict(new ArrayList<String>());
+ }
+
+ @Test
+ public void testSerialize() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ AppendTrieDictionary dict = builder.build(0);
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dataout = new DataOutputStream(bout);
+ dict.write(dataout);
+ dataout.close();
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ DataInputStream datain = new DataInputStream(bin);
+
+ assertNull(new Path(datain.readUTF()).toUri().getScheme());
+ datain.close();
+ }
+
+ @Test
+ public void testDeserialize() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.setMaxId(Integer.MAX_VALUE - 2);
+ builder.addValue("a");
+ builder.addValue("ab");
+ List<String> strList = Lists.newArrayList("a", "ab");
+ AppendTrieDictionary dict = builder.build(0);
+ TreeMap checkMap = new TreeMap();
+ BytesConverter converter = new StringBytesConverter();
+ for (String str: strList) {
+ byte[] bytes = converter.convertToBytes(str);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ checkMap.put(id, str);
+ }
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dataout = new DataOutputStream(bout);
+ dict.setSaveAbsolutePath(true);
+ dict.write(dataout);
+ dataout.close();
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ DataInputStream datain = new DataInputStream(bin);
+ AppendTrieDictionary<String> r = new AppendTrieDictionary<String>();
+ r.readFields(datain);
+ datain.close();
+
+ for (String str : strList) {
+ byte[] bytes = converter.convertToBytes(str);
+ int id = r.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
+ assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
+ }
+ }
+
+ private void createAppendTrieDict(List<String> valueList) throws IOException {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "1");
+
+ AppendTrieDictionaryBuilder builder = createBuilder();
+
+ for (String value : valueList) {
+ builder.addValue(value);
+ }
+
+ builder.build(0);
+ }
+
+ private static class CachedFileFilter implements FileFilter {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().startsWith("cached_");
+ }
+ }
+
+ private static class VersionFilter implements FileFilter {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().startsWith(GlobalDictHDFSStore.VERSION_PREFIX);
+ }
+ }
+
+ @Test
+ public void testMultiVersions() throws IOException, InterruptedException {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
+
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+ builder.addValue("b");
+ builder.addValue("c");
+ builder.addValue("d");
+ builder.addValue("e");
+ builder.addValue("f");
+ AppendTrieDictionary dict = builder.build(0);
+
+ assertEquals(2, dict.getIdFromValue("b"));
+
+ // re-open dict, append new data
+ builder = createBuilder();
+ builder.addValue("g");
+
+ // new data is not visible
+ try {
+ dict.getIdFromValue("g");
+ fail("Value 'g' (g) not exists!");
+ } catch (IllegalArgumentException e) {
+
+ }
+
+ // append data, and be visible for new immutable map
+ builder.addValue("h");
+
+ AppendTrieDictionary newDict = builder.build(0);
+ assert newDict.equals(dict);
+
+ assertEquals(7, newDict.getIdFromValue("g"));
+ assertEquals(8, newDict.getIdFromValue("h"));
+
+ // Check versions retention
+ File dir = new File(LOCAL_BASE_DIR);
+ assertEquals(2, dir.listFiles(new VersionFilter()).length);
+ }
+
+ @Test
+ public void testVersionRetention() throws IOException, InterruptedException {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-max-versions", "1");
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-version-ttl", "1000");
+
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+
+ //version 1
+ builder.build(0);
+
+ // Check versions retention
+ File dir = new File(LOCAL_BASE_DIR);
+ assertEquals(1, dir.listFiles(new VersionFilter()).length);
+
+ // sleep to make version 1 expired
+ Thread.sleep(1200);
+
+ //version 2
+ builder = createBuilder();
+ builder.addValue("");
+ builder.build(0);
+
+ // Check versions retention
+ assertEquals(1, dir.listFiles(new VersionFilter()).length);
+ }
+
+ @Test
+ public void testOldDirFormat() throws IOException {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
+
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+ builder.addValue("b");
+ builder.addValue("c");
+ builder.addValue("d");
+ builder.addValue("e");
+ builder.addValue("f");
+ builder.build(0);
+
+ convertDirToOldFormat(BASE_DIR);
+
+ File dir = new File(LOCAL_BASE_DIR);
+ assertEquals(0, dir.listFiles(new VersionFilter()).length);
+ assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
+
+ //convert older format to new format when builder init
+ builder = createBuilder();
+ builder.build(0);
+
+ assertEquals(1, dir.listFiles(new VersionFilter()).length);
+ }
+
+ private void convertDirToOldFormat(String baseDir) throws IOException {
+ Path basePath = new Path(baseDir);
+ FileSystem fs = HadoopUtil.getFileSystem(basePath);
+
+ // move version dir to base dir, to simulate the older format
+ GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir);
+ Long[] versions = store.listAllVersions();
+ Path versionPath = store.getVersionDir(versions[versions.length - 1]);
+ Path tmpVersionPath = new Path(versionPath.getParent().getParent(), versionPath.getName());
+ fs.rename(versionPath, tmpVersionPath);
+ fs.delete(new Path(baseDir), true);
+ fs.rename(tmpVersionPath, new Path(baseDir));
+ }
+
+ @Test
+ public void testOldIndexFormat() throws IOException {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
+
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+ builder.addValue("b");
+ builder.addValue("c");
+ builder.addValue("d");
+ builder.addValue("e");
+ builder.addValue("f");
+ builder.build(0);
+
+ convertIndexToOldFormat(BASE_DIR);
+
+ builder = createBuilder();
+ builder.addValue("g");
+ builder.addValue("h");
+ builder.addValue("i");
+ AppendTrieDictionary dict = builder.build(0);
+
+ assertEquals(1, dict.getIdFromValue("a"));
+ assertEquals(7, dict.getIdFromValue("g"));
+ }
+
+ private void convertIndexToOldFormat(String baseDir) throws IOException {
+ Path basePath = new Path(baseDir);
+ FileSystem fs = HadoopUtil.getFileSystem(basePath);
+
+ GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir);
+ Long[] versions = store.listAllVersions();
+ GlobalDictMetadata metadata = store.getMetadata(versions[versions.length - 1]);
+
+ //convert v2 index to v1 index
+ Path versionPath = store.getVersionDir(versions[versions.length - 1]);
+ Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME);
+
+ fs.delete(v2IndexFile, true);
+ GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, HadoopUtil.getCurrentConfiguration());
+ indexFormatV1.writeIndexFile(versionPath, metadata);
+
+ //convert v2 fileName format to v1 fileName format
+ for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) {
+ fs.rename(new Path(versionPath, entry.getValue()), new Path(versionPath, "cached_" + entry.getKey()));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
deleted file mode 100644
index 6b39c36..0000000
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
+++ /dev/null
@@ -1,525 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.global;
-
-import static org.apache.kylin.dict.global.GlobalDictHDFSStore.V2_INDEX_NAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.AppendTrieDictionary;
-import org.apache.kylin.dict.BytesConverter;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
- private static final String RESOURCE_DIR = "/dict/append_dict_test/" + UUID.randomUUID();
- private static String BASE_DIR;
- private static String LOCAL_BASE_DIR;
-
- @Before
- public void beforeTest() {
- staticCreateTestMetadata();
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000");
- BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
- LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/";
- }
-
- @After
- public void afterTest() {
- cleanup();
- staticCleanupTestMetadata();
- }
-
- private void cleanup() {
- Path basePath = new Path(BASE_DIR);
- try {
- HadoopUtil.getFileSystem(basePath).delete(basePath, true);
- } catch (IOException e) {
- }
- }
-
- private static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "字典", "字典树", "字母", // non-ascii characters
- "", // empty
- "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
- "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk",
- "paint", "tar", "try", // some dup
- };
-
- private AppendTrieDictionaryBuilder createBuilder(String resourceDir) throws IOException {
- int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + resourceDir + "/";
- return new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true);
- }
-
- @Test
- public void testStringRepeatly() throws IOException {
- ArrayList<String> list = new ArrayList<>();
- Collections.addAll(list, words);
- ArrayList<String> notfound = new ArrayList<>();
- notfound.add("pa");
- notfound.add("pars");
- notfound.add("tri");
- notfound.add("字");
- for (int i = 0; i < 50; i++) {
- testStringDictAppend(list, notfound, true);
- //to speed up the test
- cleanup();
- }
- }
-
- @Test
- public void testEnglishWords() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).txt");
- ArrayList<String> str = loadStrings(is);
- testStringDictAppend(str, null, false);
- }
-
- @Test
- public void testCategoryNames() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- ArrayList<String> str = loadStrings(is);
- testStringDictAppend(str, null, true);
- }
-
- private static ArrayList<String> loadStrings(InputStream is) throws Exception {
- ArrayList<String> r = new ArrayList<String>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
- try {
- String word;
- while ((word = reader.readLine()) != null) {
- word = word.trim();
- if (word.isEmpty() == false)
- r.add(word);
- }
- } finally {
- reader.close();
- is.close();
- }
- return r;
- }
-
- @Ignore("need huge key set")
- @Test
- public void testHugeKeySet() throws IOException {
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
-
- AppendTrieDictionary<String> dict = null;
-
- InputStream is = new FileInputStream("src/test/resources/dict/huge_key");
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
- try {
- String word;
- while ((word = reader.readLine()) != null) {
- word = word.trim();
- if (!word.isEmpty())
- builder.addValue(word);
- }
- } finally {
- reader.close();
- is.close();
- }
- dict = builder.build(0);
- dict.dump(System.out);
- }
-
- private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList) throws IOException {
- Random rnd = new Random(System.currentTimeMillis());
- ArrayList<String> strList = new ArrayList<String>();
- strList.addAll(list);
- if (shuffleList) {
- Collections.shuffle(strList, rnd);
- }
- BytesConverter converter = new StringBytesConverter();
-
- AppendTrieDictionaryBuilder b = createBuilder(RESOURCE_DIR);
-
- TreeMap<Integer, String> checkMap = new TreeMap<>();
- int firstAppend = rnd.nextInt(strList.size() / 2);
- int secondAppend = firstAppend + rnd.nextInt((strList.size() - firstAppend) / 2);
- int appendIndex = 0;
- int checkIndex = 0;
-
- for (; appendIndex < firstAppend; appendIndex++) {
- b.addValue(strList.get(appendIndex));
- }
- AppendTrieDictionary<String> dict = b.build(0);
- dict.dump(System.out);
- for (; checkIndex < firstAppend; checkIndex++) {
- String str = strList.get(checkIndex);
- byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
- assertNotEquals(String.format("Value %s not exist", str), -1, id);
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
- checkMap.put(id, str);
- }
-
- // reopen dict and append
- b = createBuilder(RESOURCE_DIR);
-
- for (; appendIndex < secondAppend; appendIndex++) {
- b.addValue(strList.get(appendIndex));
- }
- AppendTrieDictionary<String> newDict = b.build(0);
- assert newDict.equals(dict);
- dict = newDict;
- dict.dump(System.out);
- checkIndex = 0;
- for (; checkIndex < secondAppend; checkIndex++) {
- String str = strList.get(checkIndex);
- byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
- assertNotEquals(String.format("Value %s not exist", str), -1, id);
- if (checkIndex < firstAppend) {
- assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
- } else {
- // check second append str, should be new id
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
- checkMap.put(id, str);
- }
- }
-
- // reopen dict and append rest str
- b = createBuilder(RESOURCE_DIR);
-
- for (; appendIndex < strList.size(); appendIndex++) {
- b.addValue(strList.get(appendIndex));
- }
- newDict = b.build(0);
- assert newDict.equals(dict);
- dict = newDict;
- dict.dump(System.out);
- checkIndex = 0;
- for (; checkIndex < strList.size(); checkIndex++) {
- String str = strList.get(checkIndex);
- byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
- assertNotEquals(String.format("Value %s not exist", str), -1, id);
- if (checkIndex < secondAppend) {
- assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
- } else {
- // check third append str, should be new id
- assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
- checkMap.put(id, str);
- }
- }
- if (notfound != null) {
- for (String s : notfound) {
- byte[] bytes = converter.convertToBytes(s);
- int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
- assertEquals(-1, id);
- }
- }
-
- dict = testSerialize(dict, converter);
- for (String str : strList) {
- byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
- assertNotEquals(String.format("Value %s not exist", str), -1, id);
- assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
- }
- }
-
- private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict, BytesConverter converter) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream dataout = new DataOutputStream(bout);
- dict.write(dataout);
- dataout.close();
- ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
- DataInputStream datain = new DataInputStream(bin);
- AppendTrieDictionary<String> r = new AppendTrieDictionary<String>();
- r.readFields(datain);
- datain.close();
- return r;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testMaxInteger() throws IOException {
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- builder.setMaxId(Integer.MAX_VALUE - 2);
- builder.addValue("a");
- builder.addValue("ab");
- builder.addValue("acd");
- builder.addValue("ac");
- AppendTrieDictionary dict = builder.build(0);
- assertEquals(2147483646, dict.getIdFromValue("a", 0));
- assertEquals(2147483647, dict.getIdFromValue("ab", 0));
- assertEquals(-2147483647, dict.getIdFromValue("ac", 0));
- assertEquals(-2147483648, dict.getIdFromValue("acd", 0));
- }
-
- @Ignore("Only occurred when value is very long (>8000 bytes)")
- @Test
- public void testSuperLongValue() throws IOException {
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- String value = "a";
- for (int i = 0; i < 10000; i++) {
- value += "a";
- try {
- builder.addValue(value);
- } catch (StackOverflowError e) {
- System.out.println("\nstack overflow " + i);
- throw e;
- }
- }
- AppendTrieDictionary dictionary = builder.build(0);
- dictionary.getMaxId();
- }
-
- @Test
- public void testSplitContainSuperLongValue() throws IOException {
- String superLongValue = "%5Cx1A%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%7E%29%5CxEF%5CxBF%5CxBD%5Cx1B+%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5Cx13%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5B";
-
- createAppendTrieDict(Arrays.asList("a", superLongValue));
- }
-
- @Test
- public void testSuperLongValueAsFileName() throws IOException {
- String superLongValue = "%5Cx1A%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%7E%29%5CxEF%5CxBF%5CxBD%5Cx1B+%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5Cx13%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5CxEF%5CxBF%5CxBD%5B";
-
- createAppendTrieDict(Arrays.asList("a", superLongValue));
- }
-
- @Test
- public void testIllegalFileNameValue() throws IOException {
- createAppendTrieDict(Arrays.asList("::", ":"));
- }
-
- @Test
- public void testSkipAddValue() throws IOException {
- createAppendTrieDict(new ArrayList<String>());
- }
-
- private void createAppendTrieDict(List<String> valueList) throws IOException {
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "1");
-
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
-
- for (String value : valueList) {
- builder.addValue(value);
- }
-
- builder.build(0);
- }
-
- private static class CachedFileFilter implements FileFilter {
- @Override
- public boolean accept(File pathname) {
- return pathname.getName().startsWith("cached_");
- }
- }
-
- private static class VersionFilter implements FileFilter {
- @Override
- public boolean accept(File pathname) {
- return pathname.getName().startsWith(GlobalDictHDFSStore.VERSION_PREFIX);
- }
- }
-
- @Test
- public void testMultiVersions() throws IOException, InterruptedException {
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
-
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- builder.addValue("a");
- builder.addValue("b");
- builder.addValue("c");
- builder.addValue("d");
- builder.addValue("e");
- builder.addValue("f");
- AppendTrieDictionary dict = builder.build(0);
-
- assertEquals(2, dict.getIdFromValue("b"));
-
- // re-open dict, append new data
- builder = createBuilder(RESOURCE_DIR);
- builder.addValue("g");
-
- // new data is not visible
- try {
- dict.getIdFromValue("g");
- fail("Value 'g' (g) not exists!");
- } catch (IllegalArgumentException e) {
-
- }
-
- // append data, and be visible for new immutable map
- builder.addValue("h");
-
- AppendTrieDictionary newDict = builder.build(0);
- assert newDict.equals(dict);
-
- assertEquals(7, newDict.getIdFromValue("g"));
- assertEquals(8, newDict.getIdFromValue("h"));
-
- // Check versions retention
- File dir = new File(LOCAL_BASE_DIR);
- assertEquals(2, dir.listFiles(new VersionFilter()).length);
- }
-
- @Test
- public void testVersionRetention() throws IOException, InterruptedException {
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-max-versions", "1");
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-version-ttl", "1000");
-
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- builder.addValue("a");
-
- //version 1
- builder.build(0);
-
- // Check versions retention
- File dir = new File(LOCAL_BASE_DIR);
- assertEquals(1, dir.listFiles(new VersionFilter()).length);
-
- // sleep to make version 1 expired
- Thread.sleep(1200);
-
- //version 2
- builder = createBuilder(RESOURCE_DIR);
- builder.addValue("");
- builder.build(0);
-
- // Check versions retention
- assertEquals(1, dir.listFiles(new VersionFilter()).length);
- }
-
- @Test
- public void testOldDirFormat() throws IOException {
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
-
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- builder.addValue("a");
- builder.addValue("b");
- builder.addValue("c");
- builder.addValue("d");
- builder.addValue("e");
- builder.addValue("f");
- builder.build(0);
-
- convertDirToOldFormat(BASE_DIR);
-
- File dir = new File(LOCAL_BASE_DIR);
- assertEquals(0, dir.listFiles(new VersionFilter()).length);
- assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
-
- //convert older format to new format when builder init
- builder = createBuilder(RESOURCE_DIR);
- builder.build(0);
-
- assertEquals(1, dir.listFiles(new VersionFilter()).length);
- }
-
- private void convertDirToOldFormat(String baseDir) throws IOException {
- Path basePath = new Path(baseDir);
- FileSystem fs = HadoopUtil.getFileSystem(basePath);
-
- // move version dir to base dir, to simulate the older format
- GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir);
- Long[] versions = store.listAllVersions();
- Path versionPath = store.getVersionDir(versions[versions.length - 1]);
- Path tmpVersionPath = new Path(versionPath.getParent().getParent(), versionPath.getName());
- fs.rename(versionPath, tmpVersionPath);
- fs.delete(new Path(baseDir), true);
- fs.rename(tmpVersionPath, new Path(baseDir));
- }
-
- @Test
- public void testOldIndexFormat() throws IOException {
- KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "4");
-
- AppendTrieDictionaryBuilder builder = createBuilder(RESOURCE_DIR);
- builder.addValue("a");
- builder.addValue("b");
- builder.addValue("c");
- builder.addValue("d");
- builder.addValue("e");
- builder.addValue("f");
- builder.build(0);
-
- convertIndexToOldFormat(BASE_DIR);
-
- builder = createBuilder(RESOURCE_DIR);
- builder.addValue("g");
- builder.addValue("h");
- builder.addValue("i");
- AppendTrieDictionary dict = builder.build(0);
-
- assertEquals(1, dict.getIdFromValue("a"));
- assertEquals(7, dict.getIdFromValue("g"));
- }
-
- private void convertIndexToOldFormat(String baseDir) throws IOException {
- Path basePath = new Path(baseDir);
- FileSystem fs = HadoopUtil.getFileSystem(basePath);
-
- GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir);
- Long[] versions = store.listAllVersions();
- GlobalDictMetadata metadata = store.getMetadata(versions[versions.length - 1]);
-
- //convert v2 index to v1 index
- Path versionPath = store.getVersionDir(versions[versions.length - 1]);
- Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME);
-
- fs.delete(v2IndexFile, true);
- GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, HadoopUtil.getCurrentConfiguration());
- indexFormatV1.writeIndexFile(versionPath, metadata);
-
- //convert v2 fileName format to v1 fileName format
- for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) {
- fs.rename(new Path(versionPath, entry.getValue()), new Path(versionPath, "cached_" + entry.getKey()));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index babf69b..fcedcc1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OptionsHelper;
@@ -452,7 +453,10 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath());
logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath());
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- kylinConfig.setMetadataUrl(metaDir.getAbsolutePath());
+ Map<String, String> paramsMap = new HashMap<>();
+ paramsMap.put("path", metaDir.getAbsolutePath());
+ StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap);
+ kylinConfig.setMetadataUrl(storageURL.toString());
return kylinConfig;
} else {
return KylinConfig.getInstanceFromEnv();
@@ -469,8 +473,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
logger.info("Ready to load KylinConfig from uri: {}", uri);
KylinConfig config;
FileSystem fs;
- int cut = uri.indexOf('@');
- String realHdfsPath = uri.substring(0, cut) + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
+ String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
try {
fs = HadoopUtil.getFileSystem(realHdfsPath);
InputStream is = fs.open(new Path(realHdfsPath));
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 47ea3d0..7d76ce4 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -19,6 +19,7 @@
package org.apache.kylin.engine.spark;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
@@ -30,6 +31,9 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
+import java.util.Map;
+
/**
*/
public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
@@ -89,6 +93,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
}
private static String getSegmentMetadataUrl(KylinConfig kylinConfig, String segmentID) {
- return kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID + "@hdfs";
+ Map<String, String> param = new HashMap<>();
+ param.put("path", kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID);
+ return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString();
+// return kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID + "@hdfs";
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index d185f4e..720e7e2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -58,7 +58,7 @@ public class HDFSResourceStore extends ResourceStore {
StorageURL metadataUrl = kylinConfig.getMetadataUrl();
Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));
- String path = metadataUrl.getIdentifier();
+ String path = metadataUrl.getParameter("path");
fs = HadoopUtil.getFileSystem(path);
Path metadataPath = new Path(path);
if (fs.exists(metadataPath) == false) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c88db54/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/IdentifierFileResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/IdentifierFileResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/IdentifierFileResourceStore.java
new file mode 100644
index 0000000..b23a916
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/IdentifierFileResourceStore.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.storage.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
+import org.apache.kylin.common.persistence.FileResourceStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+/**
+ * it need identifier to transfer relative path to absolute path when building cube like reading global dict and
+ * using path to locate the data and make use of it, so this ResourceStore separate identifier from the data path
+ * saved in params.
+ *
+ */
+public class IdentifierFileResourceStore extends FileResourceStore {
+ private static final Logger logger = LoggerFactory.getLogger(IdentifierFileResourceStore.class);
+
+ private static final String IFILE_SCHEME = "ifile";
+
+ private File root;
+
+ public IdentifierFileResourceStore(KylinConfig kylinConfig) throws Exception {
+ super(kylinConfig);
+ }
+
+ protected String getPath(KylinConfig kylinConfig) {
+ StorageURL metadataUrl = kylinConfig.getMetadataUrl();
+ Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme()));
+ return metadataUrl.getParameter("path");
+ }
+}
[03/12] kylin git commit: KYLIN-2930 make tablescan phy type always
ARRAY
Posted by li...@apache.org.
KYLIN-2930 make tablescan phy type always ARRAY
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/227668a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/227668a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/227668a3
Branch: refs/heads/2.2.x
Commit: 227668a34343f28f2efb30c6bf8ec106bc6239ac
Parents: c21fe43
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Oct 10 15:04:46 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Thu Oct 19 10:03:42 2017 +0800
----------------------------------------------------------------------
kylin-it/src/test/resources/query/sql_union/query03.sql | 1 +
.../main/java/org/apache/kylin/query/relnode/OLAPTableScan.java | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/227668a3/kylin-it/src/test/resources/query/sql_union/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_union/query03.sql b/kylin-it/src/test/resources/query/sql_union/query03.sql
new file mode 100644
index 0000000..e92d8e0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_union/query03.sql
@@ -0,0 +1 @@
+select count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID < 1000 union select count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID > 9000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/227668a3/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 2ee4513..e9e3566 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -25,6 +25,7 @@ import java.util.Stack;
import org.apache.calcite.adapter.enumerable.EnumerableRel;
import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.adapter.enumerable.JavaRowFormat;
import org.apache.calcite.adapter.enumerable.PhysType;
import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
import org.apache.calcite.linq4j.tree.Blocks;
@@ -334,7 +335,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
context.setReturnTupleInfo(rowType, columnRowType);
String execFunction = genExecFunc();
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), this.rowType, pref.preferArray());
+ PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), JavaRowFormat.ARRAY);
MethodCallExpression exprCall = Expressions.call(table.getExpression(OLAPTable.class), execFunction,
implementor.getRootExpression(), Expressions.constant(context.id));
return implementor.result(physType, Blocks.toBlock(exprCall));
[11/12] kylin git commit: Shade commons-pool and commons-jocl library
in kylin-job jar
Posted by li...@apache.org.
Shade commons-pool and commons-jocl library in kylin-job jar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e49fcaf5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e49fcaf5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e49fcaf5
Branch: refs/heads/2.2.x
Commit: e49fcaf52a97656c5198e50c0591e0ede437c26c
Parents: d79660c
Author: nichunen <ch...@kyligence.io>
Authored: Tue Oct 24 21:34:40 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Tue Oct 24 21:34:40 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e49fcaf5/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 63c4d1c..4145c4e 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -212,6 +212,14 @@
<pattern>org.apache.commons.dbcp</pattern>
<shadedPattern>${shadeBase}.org.apache.commons.dbcp</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons.pool</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.pool</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.jocl</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.jocl</shadedPattern>
+ </relocation>
</relocations>
<filters>
<filter>
[08/12] kylin git commit: minor, fix union query for ci
Posted by li...@apache.org.
minor, fix union query for ci
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a65c3847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a65c3847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a65c3847
Branch: refs/heads/2.2.x
Commit: a65c3847e94350251292789d6c449cbed511230b
Parents: 509952f
Author: Roger Shi <ro...@hotmail.com>
Authored: Sun Oct 22 14:45:27 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Sun Oct 22 20:57:30 2017 +0800
----------------------------------------------------------------------
kylin-it/src/test/resources/query/sql_union/query03.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a65c3847/kylin-it/src/test/resources/query/sql_union/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_union/query03.sql b/kylin-it/src/test/resources/query/sql_union/query03.sql
index e92d8e0..44377cb 100644
--- a/kylin-it/src/test/resources/query/sql_union/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_union/query03.sql
@@ -1 +1 @@
-select count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID < 1000 union select count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID > 9000
\ No newline at end of file
+select count(*) as cnt from TEST_KYLIN_FACT where ORDER_ID < 100 union select count(*) as cnt from TEST_KYLIN_FACT where ORDER_ID > 200
\ No newline at end of file
[12/12] kylin git commit: Merge commit
'fb25808088eb1aa857c674b42f969fdcc0a35d7c'
Posted by li...@apache.org.
Merge commit 'fb25808088eb1aa857c674b42f969fdcc0a35d7c'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd063077
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd063077
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd063077
Branch: refs/heads/2.2.x
Commit: dd063077521728b1f0301ecf55a075f2f3e06949
Parents: e49fcaf fb25808
Author: lidongsjtu <li...@apache.org>
Authored: Tue Oct 24 22:12:05 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Oct 24 22:12:05 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/security/AuthoritiesPopulator.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dd063077/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
----------------------------------------------------------------------
[07/12] kylin git commit: KYLIN-2945 fix CI
Posted by li...@apache.org.
KYLIN-2945 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/509952f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/509952f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/509952f1
Branch: refs/heads/2.2.x
Commit: 509952f13feea806483b4a60cfbe5d515ae9bdf5
Parents: 1a3a37a
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 22 20:51:01 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 22 20:51:01 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/509952f1/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 720e7e2..14c080e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -59,6 +59,12 @@ public class HDFSResourceStore extends ResourceStore {
Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));
String path = metadataUrl.getParameter("path");
+ if (path == null) {
+ // missing path is not expected, but don't fail it
+ path = kylinConfig.getHdfsWorkingDirectory() + "tmp_metadata";
+ logger.warn("Missing path, fall back to " + path);
+ }
+
fs = HadoopUtil.getFileSystem(path);
Path metadataPath = new Path(path);
if (fs.exists(metadataPath) == false) {
[02/12] kylin git commit: minor, append context info in Rel writer
Posted by li...@apache.org.
minor, append context info in Rel writer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c21fe432
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c21fe432
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c21fe432
Branch: refs/heads/2.2.x
Commit: c21fe4325ee6a846a7f2432df8e3330fa5fa556f
Parents: da0d153
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Oct 13 10:57:57 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Thu Oct 19 10:03:36 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/query/relnode/OLAPAggregateRel.java | 7 +++++++
.../java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 7 +++++++
.../main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java | 7 +++++++
.../java/org/apache/kylin/query/relnode/OLAPLimitRel.java | 4 +++-
.../java/org/apache/kylin/query/relnode/OLAPProjectRel.java | 7 +++++++
.../main/java/org/apache/kylin/query/relnode/OLAPSortRel.java | 6 ++++++
.../java/org/apache/kylin/query/relnode/OLAPTableScan.java | 5 ++++-
.../apache/kylin/query/relnode/OLAPToEnumerableConverter.java | 6 ++++++
.../java/org/apache/kylin/query/relnode/OLAPUnionRel.java | 7 +++++--
.../java/org/apache/kylin/query/relnode/OLAPWindowRel.java | 4 +++-
10 files changed, 55 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 1540483..ca7c8ea 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
@@ -487,4 +488,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.traitSet = this.traitSet.replace(trait);
return oldTraitSet;
}
+
+ @Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("ctx",
+ context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 8f86ae0..1887ecb 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
@@ -437,4 +438,10 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
this.traitSet = this.traitSet.replace(trait);
return oldTraitSet;
}
+
+ @Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("ctx",
+ context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6d0a657..2a97a4c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -43,6 +43,7 @@ import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
@@ -373,4 +374,10 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
this.traitSet = this.traitSet.replace(trait);
return oldTraitSet;
}
+
+ @Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("ctx",
+ context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 2a248cc..700f615 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -66,7 +66,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
@Override
public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw).itemIf("offset", localOffset, localOffset != null).itemIf("fetch", localFetch, localFetch != null);
+ return super.explainTerms(pw)
+ .item("ctx", context == null ? "" : String.valueOf(context.id) + "@" + context.realization)
+ .itemIf("offset", localOffset, localOffset != null).itemIf("fetch", localFetch, localFetch != null);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 8e93659..311e420 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
@@ -372,4 +373,10 @@ public class OLAPProjectRel extends Project implements OLAPRel {
public boolean isMerelyPermutation() {
return isMerelyPermutation;
}
+
+ @Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("ctx",
+ context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
index 03ba9c5..9a90d33 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
@@ -31,6 +31,7 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexNode;
@@ -147,4 +148,9 @@ public class OLAPSortRel extends Sort implements OLAPRel {
return oldTraitSet;
}
+ @Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("ctx",
+ context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index b739326..2ee4513 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -203,7 +203,10 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
@Override
public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
+
+ return super.explainTerms(pw)
+ .item("ctx", context == null ? "" : String.valueOf(context.id) + "@" + context.realization)
+ .item("fields", Primitive.asList(fields));
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 99dee11..cdef91d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -80,6 +80,12 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
intercept.intercept(contexts);
}
+ if (System.getProperty("calcite.debug") != null) {
+ String dumpPlan = RelOptUtil.dumpPlan("", this, false, SqlExplainLevel.DIGEST_ATTRIBUTES);
+ System.out.println("EXECUTION PLAN AFTER OLAPCONTEXT IS SET");
+ System.out.println(dumpPlan);
+ }
+
RealizationChooser.selectRealization(contexts);
doAccessControl(contexts);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java
index dadec58..f409045 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java
@@ -69,9 +69,12 @@ public class OLAPUnionRel extends Union implements OLAPRel {
@Override
public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw).itemIf("all", all, true);
- }
+ boolean contextNotNull = context != null;
+ return super.explainTerms(pw)
+ .item("ctx", context == null ? "" : String.valueOf(context.id) + "@" + context.realization)
+ .itemIf("all", all, true);
+ }
@Override
public void implementOLAP(OLAPImplementor implementor) {
// Always create new OlapContext to combine columns from all children contexts.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPWindowRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPWindowRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPWindowRel.java
index 6470504..5cd94fd 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPWindowRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPWindowRel.java
@@ -65,7 +65,9 @@ public class OLAPWindowRel extends Window implements OLAPRel {
@Override
public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw) //
+
+ return super.explainTerms(pw)
+ .item("ctx", context == null ? "" : String.valueOf(context.id) + "@" + context.realization)//
.itemIf("constants", constants, !constants.isEmpty()) //
.itemIf("groups", groups, !groups.isEmpty());
}
[06/12] kylin git commit: Shade commons-dbcp library in kylin-job jar
Posted by li...@apache.org.
Shade commons-dbcp library in kylin-job jar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a3a37a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a3a37a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a3a37a6
Branch: refs/heads/2.2.x
Commit: 1a3a37a619f8526065d0c5b140663c138f655165
Parents: 1c88db5
Author: nichunen <ch...@kyligence.io>
Authored: Fri Oct 20 11:25:35 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Fri Oct 20 11:25:35 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a3a37a6/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 3435ed4..63c4d1c 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -193,10 +193,6 @@
<shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
</relocation>
<relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
<pattern>org.apache.commons.codec</pattern>
<shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern>
</relocation>
@@ -212,6 +208,10 @@
<pattern>org.roaringbitmap</pattern>
<shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons.dbcp</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.dbcp</shadedPattern>
+ </relocation>
</relocations>
<filters>
<filter>