You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/28 03:35:09 UTC
[6/7] kylin git commit: KYLIN-2535 Modify getMetaStoreId method and
code review
KYLIN-2535 Modify getMetaStoreId method and code review
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a5a3643
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a5a3643
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a5a3643
Branch: refs/heads/master
Commit: 7a5a3643b238e28e59015d488638307fbd46390d
Parents: 61fbebd
Author: xiefan46 <95...@qq.com>
Authored: Fri Apr 21 17:29:51 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Apr 28 11:34:10 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 +-
.../kylin/common/persistence/ResourceStore.java | 35 +-
.../kylin/common/persistence/StringEntity.java | 69 +++
.../common/persistence/ResourceStoreTest.java | 64 +--
.../hbase/ITAclTableMigrationToolTest.java | 50 +-
.../storage/hbase/ITHBaseResourceStoreTest.java | 8 +-
.../apache/kylin/rest/constant/Constant.java | 2 -
.../kylin/rest/job/StorageCleanupJob.java | 1 -
.../kylin/rest/security/AclEntityFactory.java | 3 +-
.../kylin/rest/security/AclPermissionType.java | 3 -
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../rest/security/RealAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/service/AclService.java | 6 +-
.../kylin/rest/service/AclServiceOld.java | 460 -------------------
.../rest/service/AclTableMigrationJob.java | 411 -----------------
.../rest/service/AclTableMigrationTool.java | 340 ++++++++++++++
.../kylin/rest/service/LegacyAclService.java | 460 +++++++++++++++++++
.../kylin/rest/service/LegacyUserService.java | 288 ++++++++++++
.../kylin/rest/service/UserServiceOld.java | 286 ------------
.../rest/controller/AccessControllerTest.java | 2 -
.../kylin/rest/service/AccessServiceTest.java | 1 -
.../kylin/storage/hbase/HBaseConnection.java | 30 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 34 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 17 +-
.../kylin/tool/AbstractInfoExtractor.java | 4 +-
.../apache/kylin/tool/AclTableMigrationCLI.java | 31 +-
.../org/apache/kylin/tool/util/ToolUtil.java | 28 +-
27 files changed, 1317 insertions(+), 1338 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 440ff42..7644343 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -227,7 +227,7 @@ abstract public class KylinConfigBase implements Serializable {
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers", //
- new String[]{"org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager"});
+ new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
}
public String[] getCubeDimensionCustomEncodingFactories() {
@@ -237,7 +237,7 @@ abstract public class KylinConfigBase implements Serializable {
public Map<String, String> getCubeCustomMeasureTypes() {
return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
-
+
public DistributedLock getDistributedLock() {
String clsName = getOptional("kylin.metadata.distributed-lock-impl", "org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock");
return (DistributedLock) ClassUtil.newInstance(clsName);
@@ -951,7 +951,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int[] getQueryMetricsPercentilesIntervals() {
- String[] dft = {"60", "300", "3600"};
+ String[] dft = { "60", "300", "3600" };
return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index b0e06f5..d5fbc2e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -29,6 +29,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -66,6 +67,8 @@ abstract public class ResourceStore {
protected static final String DEFAULT_STORE_NAME = "kylin_metadata";
+ public static final String METASTORE_UUID_TAG = "/UUID";
+
private static final ConcurrentMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
private static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
@@ -88,8 +91,13 @@ abstract public class ResourceStore {
logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
for (Class<? extends ResourceStore> cls : getKnownImpl(kylinConfig)) {
try {
- return cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ if (!store.exists(METASTORE_UUID_TAG)) {
+ store.putResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0, StringEntity.serializer);
+ }
+ return store;
} catch (Throwable e) {
+
es.add(e);
}
}
@@ -131,6 +139,18 @@ abstract public class ResourceStore {
abstract protected NavigableSet<String> listResourcesImpl(String folderPath) throws IOException;
+ public String createMetaStoreUUID() throws IOException {
+ return UUID.randomUUID().toString();
+ }
+
+ public String getMetaStoreUUID() throws IOException {
+ if (!exists(ResourceStore.METASTORE_UUID_TAG)) {
+ putResource(ResourceStore.METASTORE_UUID_TAG, new StringEntity(createMetaStoreUUID()), 0, StringEntity.serializer);
+ }
+ StringEntity entity = getResource(ResourceStore.METASTORE_UUID_TAG, StringEntity.class, StringEntity.serializer);
+ return entity.toString();
+ }
+
/**
* Return true if a resource exists, return false in case of folder or non-exist
*/
@@ -169,14 +189,14 @@ abstract public class ResourceStore {
}
/**
- * Read all resources under a folder. Return empty list if folder not exist.
+ * Read all resources under a folder. Return empty list if folder not exist.
*/
final public <T extends RootPersistentEntity> List<T> getAllResources(String folderPath, Class<T> clazz, Serializer<T> serializer) throws IOException {
return getAllResources(folderPath, Long.MIN_VALUE, Long.MAX_VALUE, clazz, serializer);
}
/**
- * Read all resources under a folder having last modified time between given range. Return empty list if folder not exist.
+ * Read all resources under a folder having last modified time between given range. Return empty list if folder not exist.
*/
final public <T extends RootPersistentEntity> List<T> getAllResources(String folderPath, long timeStart, long timeEndExclusive, Class<T> clazz, Serializer<T> serializer) throws IOException {
final List<RawResource> allResources = getAllResourcesImpl(folderPath, timeStart, timeEndExclusive);
@@ -201,10 +221,14 @@ abstract public class ResourceStore {
abstract protected List<RawResource> getAllResourcesImpl(String folderPath, long timeStart, long timeEndExclusive) throws IOException;
- /** returns null if not exists */
+ /**
+ * returns null if not exists
+ */
abstract protected RawResource getResourceImpl(String resPath) throws IOException;
- /** returns 0 if not exists */
+ /**
+ * returns 0 if not exists
+ */
abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
/**
@@ -350,4 +374,5 @@ abstract public class ResourceStore {
return metaDirURI;
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java b/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
new file mode 100644
index 0000000..84d7303
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+public class StringEntity extends RootPersistentEntity {
+
+ public static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 76ab029..1f563b0 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -18,22 +18,19 @@
package org.apache.kylin.common.persistence;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.NavigableSet;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Be called by LocalFileResourceStoreTest, ITHBaseResourceStoreTest and ITHDFSResourceStoreTest.
@@ -202,51 +199,6 @@ public class ResourceStoreTest {
return System.currentTimeMillis() - startTime;
}
- @SuppressWarnings("serial")
- public static class StringEntity extends RootPersistentEntity {
-
- public static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) {
String oldUrl = kylinConfig.getMetadataUrl();
kylinConfig.setProperty("kylin.metadata.url", newUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
index b6ab848..35535e5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
@@ -16,10 +16,17 @@
* limitations under the License.
*/
-
package org.apache.kylin.storage.hbase;
-import com.fasterxml.jackson.core.JsonProcessingException;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -36,9 +43,9 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
import org.apache.kylin.rest.service.AclService;
-import org.apache.kylin.rest.service.AclTableMigrationJob;
+import org.apache.kylin.rest.service.AclTableMigrationTool;
+import org.apache.kylin.rest.service.LegacyUserService;
import org.apache.kylin.rest.service.UserService;
-import org.apache.kylin.rest.service.UserServiceOld;
import org.apache.kylin.rest.util.Serializer;
import org.junit.After;
import org.junit.Before;
@@ -47,47 +54,38 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.GrantedAuthority;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
+import com.fasterxml.jackson.core.JsonProcessingException;
public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
private KylinConfig kylinConfig;
- private static final String PWD_PREFIX = "PWD:";
-
private String STORE_WITH_OLD_TABLE = "STORE_WITH_OLD_TABLE";
private String STORE_WITHOUT_OLD_TABLE = "STORE_WITHOUT_OLD_TABLE";
private Logger logger = LoggerFactory.getLogger(ITAclTableMigrationToolTest.class);
- private TableName aclTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclTableMigrationJob.ACL_TABLE_NAME);
+ private TableName aclTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclHBaseStorage.ACL_TABLE_NAME);
- private TableName userTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclTableMigrationJob.USER_TABLE_NAME);
+ private TableName userTable = TableName.valueOf(STORE_WITH_OLD_TABLE + AclHBaseStorage.USER_TABLE_NAME);
- private Serializer<UserServiceOld.UserGrantedAuthority[]> ugaSerializer = new Serializer<UserServiceOld.UserGrantedAuthority[]>(UserServiceOld.UserGrantedAuthority[].class);
+ private Serializer<LegacyUserService.UserGrantedAuthority[]> ugaSerializer = new Serializer<LegacyUserService.UserGrantedAuthority[]>(LegacyUserService.UserGrantedAuthority[].class);
- private AclTableMigrationJob aclTableMigrationJob;
+ private AclTableMigrationTool aclTableMigrationJob;
@Before
public void setup() throws Exception {
this.createTestMetadata();
kylinConfig = KylinConfig.getInstanceFromEnv();
if (!(ResourceStore.getStore(kylinConfig) instanceof HBaseResourceStore)) {
- logger.info("Do not use hbase store. Return");
+ logger.info("HBase Enviroment not found. Ignore this test");
return;
}
cleanUpAll();
createTestHTables();
addRecordsToTable();
- aclTableMigrationJob = new AclTableMigrationJob();
+ aclTableMigrationJob = new AclTableMigrationTool();
}
@Test
@@ -113,7 +111,6 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
ResourceStoreTest.replaceMetadataUrl(kylinConfig, oldUrl);
}
-
@After
public void after() throws Exception {
cleanUpAll();
@@ -129,8 +126,8 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
private void createTestHTables() throws IOException {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
Admin hbaseAdmin = new HBaseAdmin(conf);
- creatTable(hbaseAdmin, conf, aclTable, new String[]{AclHBaseStorage.ACL_INFO_FAMILY, AclHBaseStorage.ACL_ACES_FAMILY});
- creatTable(hbaseAdmin, conf, userTable, new String[]{AclHBaseStorage.USER_AUTHORITY_FAMILY});
+ creatTable(hbaseAdmin, conf, aclTable, new String[] { AclHBaseStorage.ACL_INFO_FAMILY, AclHBaseStorage.ACL_ACES_FAMILY });
+ creatTable(hbaseAdmin, conf, userTable, new String[] { AclHBaseStorage.USER_AUTHORITY_FAMILY });
}
private void addRecordsToTable() throws Exception {
@@ -195,18 +192,17 @@ public class ITAclTableMigrationToolTest extends HBaseMetadataTestCase {
if (authorities == null)
authorities = Collections.emptyList();
- UserServiceOld.UserGrantedAuthority[] serializing = new UserServiceOld.UserGrantedAuthority[authorities.size() + 1];
+ LegacyUserService.UserGrantedAuthority[] serializing = new LegacyUserService.UserGrantedAuthority[authorities.size() + 1];
// password is stored as the [0] authority
- serializing[0] = new UserServiceOld.UserGrantedAuthority(PWD_PREFIX + "password");
+ serializing[0] = new LegacyUserService.UserGrantedAuthority(LegacyUserService.PWD_PREFIX + "password");
int i = 1;
for (GrantedAuthority a : authorities) {
- serializing[i++] = new UserServiceOld.UserGrantedAuthority(a.getAuthority());
+ serializing[i++] = new LegacyUserService.UserGrantedAuthority(a.getAuthority());
}
byte[] value = ugaSerializer.serialize(serializing);
return Pair.newPair(key, value);
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index 74322bb..fd91397 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -18,21 +18,21 @@
package org.apache.kylin.storage.hbase;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStoreTest;
-import org.apache.kylin.common.persistence.ResourceStoreTest.StringEntity;
+import org.apache.kylin.common.persistence.StringEntity;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
private KylinConfig kylinConfig;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java b/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
index eff7b4f..f068e5f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
@@ -43,6 +43,4 @@ public class Constant {
public final static String SERVER_MODE_JOB = "job";
public final static String SERVER_MODE_ALL = "all";
-
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 4384ab2..ae35de2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -389,5 +389,4 @@ public class StorageCleanupJob extends AbstractApplication {
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/security/AclEntityFactory.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclEntityFactory.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclEntityFactory.java
index 5744bfd..c799b0a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclEntityFactory.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclEntityFactory.java
@@ -27,8 +27,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
/**
* @author xduo
*/
-public class AclEntityFactory implements AclEntityType{
-
+public class AclEntityFactory implements AclEntityType {
public static RootPersistentEntity createAclEntity(String entityType, String uuid) {
if (CUBE_INSTANCE.equals(entityType)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/security/AclPermissionType.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclPermissionType.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclPermissionType.java
index 45ba4b1..f1f06d1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclPermissionType.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclPermissionType.java
@@ -17,9 +17,6 @@
*/
package org.apache.kylin.rest.security;
-/**
- * Created by xiefan on 17-4-14.
- */
public interface AclPermissionType {
static final String MANAGEMENT = "MANAGEMENT";
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index 5ebb371..72d44bb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,9 +21,9 @@ package org.apache.kylin.rest.security;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.service.AclServiceOld;
+import org.apache.kylin.rest.service.LegacyAclService;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.service.UserServiceOld;
+import org.apache.kylin.rest.service.LegacyUserService;
import java.io.IOException;
@@ -54,10 +54,10 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
return realAcl.prepareHBaseTable(clazz);
}
- if (clazz == AclServiceOld.class) {
+ if (clazz == LegacyAclService.class) {
mockedAclTable = new MockHTable(aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
return aclTableName;
- } else if (clazz == UserServiceOld.class) {
+ } else if (clazz == LegacyUserService.class) {
mockedUserTable = new MockHTable(userTableName, USER_AUTHORITY_FAMILY, QueryService.USER_QUERY_FAMILY);
return userTableName;
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 657fbbc..f42b3f2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -22,9 +22,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.service.AclServiceOld;
+import org.apache.kylin.rest.service.LegacyAclService;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.service.UserServiceOld;
+import org.apache.kylin.rest.service.LegacyUserService;
import org.apache.kylin.storage.hbase.HBaseConnection;
import java.io.IOException;
@@ -46,11 +46,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
String tableNameBase = kylinConfig.getMetadataUrlPrefix();
- if (clazz == AclServiceOld.class) {
+ if (clazz == LegacyAclService.class) {
aclTableName = tableNameBase + ACL_TABLE_NAME;
HBaseConnection.createHTableIfNeeded(hbaseUrl, aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
return aclTableName;
- } else if (clazz == UserServiceOld.class) {
+ } else if (clazz == LegacyUserService.class) {
userTableName = tableNameBase + USER_TABLE_NAME;
HBaseConnection.createHTableIfNeeded(hbaseUrl, userTableName, USER_AUTHORITY_FAMILY, QueryService.USER_QUERY_FAMILY);
return userTableName;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 4e6c8b8..6fc07dd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -64,6 +64,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+<<<<<<< HEAD
<<<<<<< HEAD
/**
@@ -71,6 +72,8 @@ import java.util.Map;
*/
=======
>>>>>>> 62cdc0f... KYLIN-2535 AclService and UserService store records via ResourceStore interface
+=======
+>>>>>>> c37eac3... KYLIN-2535 Modify getMetaStoreId method and code review
@Component("aclService")
public class AclService implements MutableAclService {
@@ -82,7 +85,6 @@ public class AclService implements MutableAclService {
public static final String DIR_PREFIX = "/acl/";
-
@Autowired
protected PermissionGrantingStrategy permissionGrantingStrategy;
@@ -355,7 +357,6 @@ public class AclService implements MutableAclService {
}
}
-
}
class AclRecord extends RootPersistentEntity {
@@ -458,7 +459,6 @@ class DomainObjectInfo {
}
}
-
class SidInfo {
private String sid;
private boolean isPrincipal;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/service/AclServiceOld.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclServiceOld.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclServiceOld.java
deleted file mode 100644
index 8109562..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclServiceOld.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.service;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.rest.security.AclHBaseStorage;
-import org.apache.kylin.rest.util.Serializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.security.acls.domain.AccessControlEntryImpl;
-import org.springframework.security.acls.domain.AclAuthorizationStrategy;
-import org.springframework.security.acls.domain.AclImpl;
-import org.springframework.security.acls.domain.AuditLogger;
-import org.springframework.security.acls.domain.GrantedAuthoritySid;
-import org.springframework.security.acls.domain.ObjectIdentityImpl;
-import org.springframework.security.acls.domain.PermissionFactory;
-import org.springframework.security.acls.domain.PrincipalSid;
-import org.springframework.security.acls.model.AccessControlEntry;
-import org.springframework.security.acls.model.Acl;
-import org.springframework.security.acls.model.AlreadyExistsException;
-import org.springframework.security.acls.model.ChildrenExistException;
-import org.springframework.security.acls.model.MutableAcl;
-import org.springframework.security.acls.model.MutableAclService;
-import org.springframework.security.acls.model.NotFoundException;
-import org.springframework.security.acls.model.ObjectIdentity;
-import org.springframework.security.acls.model.PermissionGrantingStrategy;
-import org.springframework.security.acls.model.Sid;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.util.FieldUtils;
-import org.springframework.util.Assert;
-
-import javax.annotation.PostConstruct;
-import java.io.IOException;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-
-/**
- * @author xduo
- */
-//@Component("aclService")
-@Deprecated
-public class AclServiceOld implements MutableAclService {
-
- private static final Logger logger = LoggerFactory.getLogger(AclServiceOld.class);
-
- private static String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
- private static String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
- private static String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
- private static String ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN = "i";
-
- private Serializer<SidInfo> sidSerializer = new Serializer<SidInfo>(SidInfo.class);
- private Serializer<DomainObjectInfo> domainObjSerializer = new Serializer<DomainObjectInfo>(DomainObjectInfo.class);
- private Serializer<AceInfo> aceSerializer = new Serializer<AceInfo>(AceInfo.class);
-
- private String aclTableName = null;
-
- private final Field fieldAces = FieldUtils.getField(AclImpl.class, "aces");
-
- private final Field fieldAcl = FieldUtils.getField(AccessControlEntryImpl.class, "acl");
-
- @Autowired
- protected PermissionGrantingStrategy permissionGrantingStrategy;
-
- @Autowired
- protected PermissionFactory aclPermissionFactory;
-
- @Autowired
- protected AclAuthorizationStrategy aclAuthorizationStrategy;
-
- @Autowired
- protected AuditLogger auditLogger;
-
- @Autowired
- protected AclHBaseStorage aclHBaseStorage;
-
- public AclServiceOld() throws IOException {
- fieldAces.setAccessible(true);
- fieldAcl.setAccessible(true);
- }
-
- @PostConstruct
- public void init() throws IOException {
- aclTableName = aclHBaseStorage.prepareHBaseTable(AclServiceOld.class);
- }
-
- @Override
- public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
- List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- Table htable = null;
- try {
- htable = aclHBaseStorage.getTable(aclTableName);
-
- Scan scan = new Scan();
- SingleColumnValueFilter parentFilter = new SingleColumnValueFilter(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), CompareOp.EQUAL, domainObjSerializer.serialize(new DomainObjectInfo(parentIdentity)));
- parentFilter.setFilterIfMissing(true);
- scan.setFilter(parentFilter);
-
- ResultScanner scanner = htable.getScanner(scan);
- for (Result result = scanner.next(); result != null; result = scanner.next()) {
- String id = Bytes.toString(result.getRow());
- String type = Bytes.toString(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN)));
-
- oids.add(new ObjectIdentityImpl(type, id));
- }
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } finally {
- IOUtils.closeQuietly(htable);
- }
-
- return oids;
- }
-
- @Override
- public Acl readAclById(ObjectIdentity object) throws NotFoundException {
- Map<ObjectIdentity, Acl> aclsMap = readAclsById(Arrays.asList(object), null);
- // Assert.isTrue(aclsMap.containsKey(object), "There should have been an Acl entry for ObjectIdentity " + object);
-
- return aclsMap.get(object);
- }
-
- @Override
- public Acl readAclById(ObjectIdentity object, List<Sid> sids) throws NotFoundException {
- Map<ObjectIdentity, Acl> aclsMap = readAclsById(Arrays.asList(object), sids);
- Assert.isTrue(aclsMap.containsKey(object), "There should have been an Acl entry for ObjectIdentity " + object);
-
- return aclsMap.get(object);
- }
-
- @Override
- public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> objects) throws NotFoundException {
- return readAclsById(objects, null);
- }
-
- @Override
- public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
- Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- Table htable = null;
- Result result = null;
- try {
- htable = aclHBaseStorage.getTable(aclTableName);
-
- for (ObjectIdentity oid : oids) {
- result = htable.get(new Get(Bytes.toBytes(String.valueOf(oid.getIdentifier()))));
-
- if (null != result && !result.isEmpty()) {
- SidInfo owner = sidSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN)));
- Sid ownerSid = (null == owner) ? null : (owner.isPrincipal() ? new PrincipalSid(owner.getSid()) : new GrantedAuthoritySid(owner.getSid()));
- boolean entriesInheriting = Bytes.toBoolean(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN)));
-
- Acl parentAcl = null;
- DomainObjectInfo parentInfo = domainObjSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN)));
- if (null != parentInfo) {
- ObjectIdentity parentObj = new ObjectIdentityImpl(parentInfo.getType(), parentInfo.getId());
- parentAcl = readAclById(parentObj, null);
- }
-
- AclImpl acl = new AclImpl(oid, oid.getIdentifier(), aclAuthorizationStrategy, permissionGrantingStrategy, parentAcl, null, entriesInheriting, ownerSid);
- genAces(sids, result, acl);
-
- aclMaps.put(oid, acl);
- } else {
- throw new NotFoundException("Unable to find ACL information for object identity '" + oid + "'");
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } finally {
- IOUtils.closeQuietly(htable);
- }
-
- return aclMaps;
- }
-
- @Override
- public MutableAcl createAcl(ObjectIdentity objectIdentity) throws AlreadyExistsException {
- Acl acl = null;
-
- try {
- acl = readAclById(objectIdentity);
- } catch (NotFoundException e) {
- //do nothing?
- }
- if (null != acl) {
- throw new AlreadyExistsException("ACL of " + objectIdentity + " exists!");
- }
-
- Authentication auth = SecurityContextHolder.getContext().getAuthentication();
- PrincipalSid sid = new PrincipalSid(auth);
-
- Table htable = null;
- try {
- htable = aclHBaseStorage.getTable(aclTableName);
-
- Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
-
- htable.put(put);
-
- logger.debug("ACL of " + objectIdentity + " created successfully.");
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } finally {
- IOUtils.closeQuietly(htable);
- }
-
- return (MutableAcl) readAclById(objectIdentity);
- }
-
- @Override
- public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- Table htable = null;
- try {
- htable = aclHBaseStorage.getTable(aclTableName);
-
- Delete delete = new Delete(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
-
- List<ObjectIdentity> children = findChildren(objectIdentity);
- if (!deleteChildren && children.size() > 0) {
- throw new ChildrenExistException("Children exists for " + objectIdentity);
- }
-
- for (ObjectIdentity oid : children) {
- deleteAcl(oid, deleteChildren);
- }
-
- htable.delete(delete);
-
- logger.debug("ACL of " + objectIdentity + " deleted successfully.");
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } finally {
- IOUtils.closeQuietly(htable);
- }
- }
-
- @Override
- public MutableAcl updateAcl(MutableAcl acl) throws NotFoundException {
- try {
- readAclById(acl.getObjectIdentity());
- } catch (NotFoundException e) {
- throw e;
- }
-
- Table htable = null;
- try {
- htable = aclHBaseStorage.getTable(aclTableName);
-
- Delete delete = new Delete(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
- delete.deleteFamily(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
- htable.delete(delete);
-
- Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
-
- if (null != acl.getParentAcl()) {
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
- }
-
- for (AccessControlEntry ace : acl.getEntries()) {
- AceInfo aceInfo = new AceInfo(ace);
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
- }
-
- if (!put.isEmpty()) {
- htable.put(put);
-
- logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
- }
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage(), e);
- } finally {
- IOUtils.closeQuietly(htable);
- }
-
- return (MutableAcl) readAclById(acl.getObjectIdentity());
- }
-
- private void genAces(List<Sid> sids, Result result, AclImpl acl) throws JsonParseException, JsonMappingException, IOException {
- List<AceInfo> aceInfos = new ArrayList<AceInfo>();
- if (null != sids) {
- // Just return aces in sids
- for (Sid sid : sids) {
- String sidName = null;
- if (sid instanceof PrincipalSid) {
- sidName = ((PrincipalSid) sid).getPrincipal();
- } else if (sid instanceof GrantedAuthoritySid) {
- sidName = ((GrantedAuthoritySid) sid).getGrantedAuthority();
- }
-
- AceInfo aceInfo = aceSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(sidName)));
- if (null != aceInfo) {
- aceInfos.add(aceInfo);
- }
- }
- } else {
- NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
- for (byte[] qualifier : familyMap.keySet()) {
- AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(qualifier));
-
- if (null != aceInfo) {
- aceInfos.add(aceInfo);
- }
- }
- }
-
- List<AccessControlEntry> newAces = new ArrayList<AccessControlEntry>();
- for (int i = 0; i < aceInfos.size(); i++) {
- AceInfo aceInfo = aceInfos.get(i);
-
- if (null != aceInfo) {
- Sid sid = aceInfo.getSidInfo().isPrincipal() ? new PrincipalSid(aceInfo.getSidInfo().getSid()) : new GrantedAuthoritySid(aceInfo.getSidInfo().getSid());
- AccessControlEntry ace = new AccessControlEntryImpl(Long.valueOf(i), acl, sid, aclPermissionFactory.buildFromMask(aceInfo.getPermissionMask()), true, false, false);
- newAces.add(ace);
- }
- }
-
- this.setAces(acl, newAces);
- }
-
- private void setAces(AclImpl acl, List<AccessControlEntry> aces) {
- try {
- fieldAces.set(acl, aces);
- } catch (IllegalAccessException e) {
- throw new IllegalStateException("Could not set AclImpl entries", e);
- }
- }
-
- protected static class DomainObjectInfo {
- private String id;
- private String type;
-
- public DomainObjectInfo() {
- }
-
- public DomainObjectInfo(ObjectIdentity oid) {
- super();
- this.id = (String) oid.getIdentifier();
- this.type = oid.getType();
- }
-
- public Serializable getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
- }
-
- protected static class SidInfo {
- private String sid;
- private boolean isPrincipal;
-
- public SidInfo() {
- }
-
- public SidInfo(Sid sid) {
- if (sid instanceof PrincipalSid) {
- this.sid = ((PrincipalSid) sid).getPrincipal();
- this.isPrincipal = true;
- } else if (sid instanceof GrantedAuthoritySid) {
- this.sid = ((GrantedAuthoritySid) sid).getGrantedAuthority();
- this.isPrincipal = false;
- }
- }
-
- public String getSid() {
- return sid;
- }
-
- public void setSid(String sid) {
- this.sid = sid;
- }
-
- public boolean isPrincipal() {
- return isPrincipal;
- }
-
- public void setPrincipal(boolean isPrincipal) {
- this.isPrincipal = isPrincipal;
- }
- }
-
- protected static class AceInfo {
- private SidInfo sidInfo;
- private int permissionMask;
-
- public AceInfo() {
- }
-
- public AceInfo(AccessControlEntry ace) {
- super();
- this.sidInfo = new SidInfo(ace.getSid());
- this.permissionMask = ace.getPermission().getMask();
- }
-
- public SidInfo getSidInfo() {
- return sidInfo;
- }
-
- public void setSidInfo(SidInfo sidInfo) {
- this.sidInfo = sidInfo;
- }
-
- public int getPermissionMask() {
- return permissionMask;
- }
-
- public void setPermissionMask(int permissionMask) {
- this.permissionMask = permissionMask;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationJob.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationJob.java
deleted file mode 100644
index d1213f1..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationJob.java
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.service;
-
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.rest.security.AclHBaseStorage;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.HBaseResourceStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-
-public class AclTableMigrationJob {
-
- private static String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
-
- private static String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
-
- private static String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
-
- private static String ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN = "i";
-
- private static final org.apache.kylin.rest.util.Serializer<AclServiceOld.SidInfo> sidSerializer = new org.apache.kylin.rest.util.Serializer<AclServiceOld.SidInfo>(AclServiceOld.SidInfo.class);
-
- private static final org.apache.kylin.rest.util.Serializer<AclServiceOld.DomainObjectInfo> domainObjSerializer = new org.apache.kylin.rest.util.Serializer<AclServiceOld.DomainObjectInfo>(AclServiceOld.DomainObjectInfo.class);
-
- private static final org.apache.kylin.rest.util.Serializer<AclServiceOld.AceInfo> aceSerializer = new org.apache.kylin.rest.util.Serializer<AclServiceOld.AceInfo>(AclServiceOld.AceInfo.class);
-
- private static final org.apache.kylin.rest.util.Serializer<UserServiceOld.UserGrantedAuthority[]> ugaSerializer = new org.apache.kylin.rest.util.Serializer<UserServiceOld.UserGrantedAuthority[]>(UserServiceOld.UserGrantedAuthority[].class);
-
- public static final String ACL_TABLE_NAME = "_acl";
-
- public static final String USER_TABLE_NAME = "_user";
-
- public static final String MIGRATE_OK_PREFIX = AclService.DIR_PREFIX + "MIGRATE_OK_";
-
- private static final String PWD_PREFIX = "PWD:";
-
- private static final Logger logger = LoggerFactory.getLogger(AclTableMigrationJob.class);
-
- public void migrate(KylinConfig kylinConfig) throws IOException {
- if (!checkIfNeedMigrate(kylinConfig)) {
- logger.info("Do not need to migrate acl table data");
- return;
- } else {
- logger.info("Start to migrate acl table data");
- ResourceStore store = ResourceStore.getStore(kylinConfig);
- String userTableName = kylinConfig.getMetadataUrlPrefix() + USER_TABLE_NAME;
- //System.out.println("user table name : " + userTableName);
- String aclTableName = kylinConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME;
- if (needMigrateTable(aclTableName, store)) {
- logger.info("Migrate table : {}", aclTableName);
- migrate(store, ACL_TABLE_NAME, kylinConfig);
- }
- if (needMigrateTable(userTableName, store)) {
- logger.info("Migrate table : {}", userTableName);
- migrate(store, USER_TABLE_NAME, kylinConfig);
- }
- }
- }
-
- public boolean checkIfNeedMigrate(KylinConfig kylinConfig) throws IOException {
- if (!kylinConfig.getServerMode().equals("all")) {
- logger.info("Only with server mode all are allow to migrate acl tables. Will do nothing and return");
- return false;
- }
- ResourceStore store = ResourceStore.getStore(kylinConfig);
- if (!(store instanceof HBaseResourceStore)) {
- logger.info("Do not found Hbase environment. Not necessary to migrate data");
- return false;
- }
-
- String userTableName = kylinConfig.getMetadataUrlPrefix() + USER_TABLE_NAME;
- String aclTableName = kylinConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME;
- if (needMigrateTable(aclTableName, store) || needMigrateTable(userTableName, store))
- return true;
- return false;
-
- }
-
- private boolean needMigrateTable(String tableName, ResourceStore store) throws IOException {
- if (checkTableExist(tableName) && !isTableAlreadyMigrate(store, tableName))
- return true;
- return false;
- }
-
- private void migrate(ResourceStore store, String tableType, KylinConfig kylinConfig) throws IOException {
- logger.info("to migrate");
- if (!kylinConfig.getServerMode().equals("all")) {
- logger.warn("Only with server mode all are allow to migrate acl tables. Will do nothing and return");
- return;
- }
-
- switch (tableType) {
- case ACL_TABLE_NAME:
- String aclTableName = kylinConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME;
- convertToResourceStore(kylinConfig, aclTableName, store, new ResultConverter() {
- @Override
- public void converter(ResultScanner rs, ResourceStore store) throws IOException {
- if (rs == null)
- return;
- logger.info("111");
- Result result = rs.next();
- while (result != null) {
- AclRecord record = new AclRecord();
- DomainObjectInfo object = getDomainObjectInfoFromRs(result);
- record.setDomainObjectInfo(object);
- record.setParentDomainObjectInfo(getParentDomainObjectInfoFromRs(result));
- record.setOwnerInfo(getOwnerSidInfo(result));
- record.setEntriesInheriting(getInheriting(result));
- record.setAllAceInfo(getAllAceInfo(result));
- store.deleteResource(AclService.getQueryKeyById(object.getId()));
- store.putResource(AclService.getQueryKeyById(object.getId()), record, 0, AclService.AclRecordSerializer.getInstance());
- result = rs.next();
- }
- }
- });
- break;
- case USER_TABLE_NAME:
- String userTableName = kylinConfig.getMetadataUrlPrefix() + USER_TABLE_NAME;
-
- convertToResourceStore(kylinConfig, userTableName, store, new ResultConverter() {
- @Override
- public void converter(ResultScanner rs, ResourceStore store) throws IOException {
- if (rs == null)
- return;
- Result result = rs.next();
- while (result != null) {
- User user = hbaseRowToUser(result);
- UserInfo userInfo = convert(user);
- store.deleteResource(UserService.getId(userInfo.getUsername()));
- store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0, UserService.UserInfoSerializer.getInstance());
- result = rs.next();
- }
- }
- });
- break;
- default:
- throw new IOException("Unrecognized table type");
-
- }
-
-
- }
-
- private boolean checkTableExist(String tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- Admin hbaseAdmin = new HBaseAdmin(conf);
- return hbaseAdmin.tableExists(TableName.valueOf(tableName));
- }
-
- private boolean isTableAlreadyMigrate(ResourceStore store, String tableName) throws IOException {
- return store.exists(MIGRATE_OK_PREFIX + tableName);
- }
-
- private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store, ResultConverter converter) throws IOException {
-
- Table table = null;
- ResultScanner rs = null;
- Scan scan = new Scan();
- try {
- table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName));
- rs = table.getScanner(scan);
- converter.converter(rs, store);
- store.putResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"), StringEntity.serializer);
- } finally {
- IOUtils.closeQuietly(rs);
- IOUtils.closeQuietly(table);
- }
-
- }
-
-
- private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
- String type = String.valueOf(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN)));
- String id = String.valueOf(result.getRow());
- DomainObjectInfo newInfo = new DomainObjectInfo();
- newInfo.setId(id);
- newInfo.setType(type);
- return newInfo;
- }
-
- private DomainObjectInfo getParentDomainObjectInfoFromRs(Result result) throws IOException {
- AclServiceOld.DomainObjectInfo parentInfo = domainObjSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN)));
- return convert(parentInfo);
- }
-
- private boolean getInheriting(Result result) {
- boolean entriesInheriting = Bytes.toBoolean(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN)));
- return entriesInheriting;
- }
-
- private SidInfo getOwnerSidInfo(Result result) throws IOException {
- AclServiceOld.SidInfo owner = sidSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN)));
- return convert(owner);
- }
-
- private Map<String, AceInfo> getAllAceInfo(Result result) throws IOException {
- Map<String, AceInfo> allAceInfoMap = new HashMap<>();
- NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
- for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
- String sid = String.valueOf(entry.getKey());
- AclServiceOld.AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(entry.getValue()));
- if (null != aceInfo) {
- allAceInfoMap.put(sid, convert(aceInfo));
- }
- }
- return allAceInfoMap;
- }
-
- private DomainObjectInfo convert(AclServiceOld.DomainObjectInfo oldInfo) {
- if (oldInfo == null)
- return null;
- DomainObjectInfo newInfo = new DomainObjectInfo();
- newInfo.setId(String.valueOf(oldInfo.getId()));
- newInfo.setType(oldInfo.getType());
- return newInfo;
- }
-
- private SidInfo convert(AclServiceOld.SidInfo oldInfo) {
- if (oldInfo == null)
- return null;
- SidInfo newInfo = new SidInfo();
- newInfo.setPrincipal(oldInfo.isPrincipal());
- newInfo.setSid(oldInfo.getSid());
- return newInfo;
- }
-
- private AceInfo convert(AclServiceOld.AceInfo oldInfo) {
- if (oldInfo == null)
- return null;
- AceInfo newInfo = new AceInfo();
- newInfo.setPermissionMask(oldInfo.getPermissionMask());
- newInfo.setSidInfo(convert(oldInfo.getSidInfo()));
- return newInfo;
- }
-
- private UserInfo convert(User user) {
- if (user == null)
- return null;
- UserInfo newInfo = new UserInfo();
- newInfo.setUsername(user.getUserName());
- newInfo.setPassword(user.getPassword());
- List<String> authorities = new ArrayList<>();
- for (String auth : user.getAuthorities()) {
- authorities.add(auth);
- }
- newInfo.setAuthorities(authorities);
- return newInfo;
- }
-
- private User hbaseRowToUser(Result result) throws JsonParseException, JsonMappingException, IOException {
- if (null == result || result.isEmpty())
- return null;
-
- String username = Bytes.toString(result.getRow());
-
- byte[] valueBytes = result.getValue(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
- UserServiceOld.UserGrantedAuthority[] deserialized = ugaSerializer.deserialize(valueBytes);
-
- String password = "";
- List<UserServiceOld.UserGrantedAuthority> authorities = Collections.emptyList();
-
- // password is stored at [0] of authorities for backward compatibility
- if (deserialized != null) {
- if (deserialized.length > 0 && deserialized[0].getAuthority().startsWith(PWD_PREFIX)) {
- password = deserialized[0].getAuthority().substring(PWD_PREFIX.length());
- authorities = Arrays.asList(deserialized).subList(1, deserialized.length);
- } else {
- authorities = Arrays.asList(deserialized);
- }
- }
- List<String> authoritiesStr = new ArrayList<>();
- for (UserServiceOld.UserGrantedAuthority auth : authorities) {
- if (auth != null) {
- authoritiesStr.add(auth.getAuthority());
- }
- }
- return new User(username, password, authoritiesStr);
- }
-
- interface ResultConverter {
- void converter(ResultScanner rs, ResourceStore store) throws IOException;
- }
-
- @SuppressWarnings("serial")
- public static class StringEntity extends RootPersistentEntity {
-
- public static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
- public static class User {
- String userName;
-
- String password;
-
- List<String> authorities;
-
- public User(String userName, String password, List<String> authorities) {
- this.userName = userName;
- this.password = password;
- this.authorities = authorities;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- public String getPassword() {
- return password;
- }
-
- public void setPassword(String password) {
- this.password = password;
- }
-
- public List<String> getAuthorities() {
- return authorities;
- }
-
- public void setAuthorities(List<String> authorities) {
- this.authorities = authorities;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a5a3643/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
new file mode 100644
index 0000000..da017b3
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.StringEntity;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.rest.security.AclHBaseStorage;
+import org.apache.kylin.rest.util.Serializer;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.HBaseResourceStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+public class AclTableMigrationTool {
+
+ private static final Serializer<LegacyAclService.SidInfo> sidSerializer = new Serializer<LegacyAclService.SidInfo>(LegacyAclService.SidInfo.class);
+
+ private static final Serializer<LegacyAclService.DomainObjectInfo> domainObjSerializer = new Serializer<LegacyAclService.DomainObjectInfo>(LegacyAclService.DomainObjectInfo.class);
+
+ private static final Serializer<LegacyAclService.AceInfo> aceSerializer = new Serializer<LegacyAclService.AceInfo>(LegacyAclService.AceInfo.class);
+
+ private static final Serializer<LegacyUserService.UserGrantedAuthority[]> ugaSerializer = new Serializer<LegacyUserService.UserGrantedAuthority[]>(LegacyUserService.UserGrantedAuthority[].class);
+
+ public static final String MIGRATE_OK_PREFIX = AclService.DIR_PREFIX + "MIGRATE_OK_";
+
+ private static final Logger logger = LoggerFactory.getLogger(AclTableMigrationTool.class);
+
+ public void migrate(KylinConfig kylinConfig) throws IOException {
+ if (!checkIfNeedMigrate(kylinConfig)) {
+ logger.info("Do not need to migrate acl table data");
+ return;
+ } else {
+ if (!kylinConfig.getServerMode().equals("all")) {
+ throw new IllegalStateException("Please make sure that you have config kylin.server.mode=all before migrating data");
+ }
+ logger.info("Start to migrate acl table data");
+ ResourceStore store = ResourceStore.getStore(kylinConfig);
+ String userTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.USER_TABLE_NAME;
+ //System.out.println("user table name : " + userTableName);
+ String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.ACL_TABLE_NAME;
+ if (needMigrateTable(aclTableName, store)) {
+ logger.info("Migrate table : {}", aclTableName);
+ migrate(store, AclHBaseStorage.ACL_TABLE_NAME, kylinConfig);
+ }
+ if (needMigrateTable(userTableName, store)) {
+ logger.info("Migrate table : {}", userTableName);
+ migrate(store, AclHBaseStorage.USER_TABLE_NAME, kylinConfig);
+ }
+ }
+ }
+
+ public boolean checkIfNeedMigrate(KylinConfig kylinConfig) throws IOException {
+ ResourceStore store = ResourceStore.getStore(kylinConfig);
+ if (!(store instanceof HBaseResourceStore)) {
+ logger.info("HBase enviroment not found. Not necessary to migrate data");
+ return false;
+ }
+
+ String userTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.USER_TABLE_NAME;
+ String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.ACL_TABLE_NAME;
+ if (needMigrateTable(aclTableName, store) || needMigrateTable(userTableName, store))
+ return true;
+ return false;
+
+ }
+
+ private boolean needMigrateTable(String tableName, ResourceStore store) throws IOException {
+ if (checkTableExist(tableName) && !isTableAlreadyMigrate(store, tableName))
+ return true;
+ return false;
+ }
+
+ private void migrate(ResourceStore store, String tableType, KylinConfig kylinConfig) throws IOException {
+
+ switch (tableType) {
+ case AclHBaseStorage.ACL_TABLE_NAME:
+ String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.ACL_TABLE_NAME;
+ convertToResourceStore(kylinConfig, aclTableName, store, new ResultConverter() {
+ @Override
+ public void convertResult(ResultScanner rs, ResourceStore store) throws IOException {
+ if (rs == null)
+ return;
+ Result result = rs.next();
+ while (result != null) {
+ AclRecord record = new AclRecord();
+ DomainObjectInfo object = getDomainObjectInfoFromRs(result);
+ record.setDomainObjectInfo(object);
+ record.setParentDomainObjectInfo(getParentDomainObjectInfoFromRs(result));
+ record.setOwnerInfo(getOwnerSidInfo(result));
+ record.setEntriesInheriting(getInheriting(result));
+ record.setAllAceInfo(getAllAceInfo(result));
+ store.deleteResource(AclService.getQueryKeyById(object.getId()));
+ store.putResource(AclService.getQueryKeyById(object.getId()), record, 0, AclService.AclRecordSerializer.getInstance());
+ result = rs.next();
+ }
+ }
+ });
+ break;
+ case AclHBaseStorage.USER_TABLE_NAME:
+ String userTableName = kylinConfig.getMetadataUrlPrefix() + AclHBaseStorage.USER_TABLE_NAME;
+
+ convertToResourceStore(kylinConfig, userTableName, store, new ResultConverter() {
+ @Override
+ public void convertResult(ResultScanner rs, ResourceStore store) throws IOException {
+ if (rs == null)
+ return;
+ Result result = rs.next();
+ while (result != null) {
+ User user = hbaseRowToUser(result);
+ UserInfo userInfo = convert(user);
+ store.deleteResource(UserService.getId(userInfo.getUsername()));
+ store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0, UserService.UserInfoSerializer.getInstance());
+ result = rs.next();
+ }
+ }
+ });
+ break;
+ default:
+ throw new IOException("Unrecognized table type");
+
+ }
+
+ }
+
+ private boolean checkTableExist(String tableName) throws IOException {
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ Admin hbaseAdmin = new HBaseAdmin(conf);
+ return hbaseAdmin.tableExists(TableName.valueOf(tableName));
+ }
+
+ private boolean isTableAlreadyMigrate(ResourceStore store, String tableName) throws IOException {
+ return store.exists(MIGRATE_OK_PREFIX + tableName);
+ }
+
+ private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store, ResultConverter converter) throws IOException {
+
+ Table table = null;
+ ResultScanner rs = null;
+ Scan scan = new Scan();
+ try {
+ table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName));
+ rs = table.getScanner(scan);
+ converter.convertResult(rs, store);
+ store.putResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"), StringEntity.serializer);
+ } finally {
+ IOUtils.closeQuietly(rs);
+ IOUtils.closeQuietly(table);
+ }
+
+ }
+
+ private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
+ String type = String.valueOf(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(LegacyAclService.ACL_INFO_FAMILY_TYPE_COLUMN)));
+ String id = String.valueOf(result.getRow());
+ DomainObjectInfo newInfo = new DomainObjectInfo();
+ newInfo.setId(id);
+ newInfo.setType(type);
+ return newInfo;
+ }
+
+ private DomainObjectInfo getParentDomainObjectInfoFromRs(Result result) throws IOException {
+ LegacyAclService.DomainObjectInfo parentInfo = domainObjSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(LegacyAclService.ACL_INFO_FAMILY_PARENT_COLUMN)));
+ return convert(parentInfo);
+ }
+
+ private boolean getInheriting(Result result) {
+ boolean entriesInheriting = Bytes.toBoolean(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(LegacyAclService.ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN)));
+ return entriesInheriting;
+ }
+
+ private SidInfo getOwnerSidInfo(Result result) throws IOException {
+ LegacyAclService.SidInfo owner = sidSerializer.deserialize(result.getValue(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(LegacyAclService.ACL_INFO_FAMILY_OWNER_COLUMN)));
+ return convert(owner);
+ }
+
+ private Map<String, AceInfo> getAllAceInfo(Result result) throws IOException {
+ Map<String, AceInfo> allAceInfoMap = new HashMap<>();
+ NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
+ for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
+ String sid = String.valueOf(entry.getKey());
+ LegacyAclService.AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(entry.getValue()));
+ if (null != aceInfo) {
+ allAceInfoMap.put(sid, convert(aceInfo));
+ }
+ }
+ return allAceInfoMap;
+ }
+
+ private DomainObjectInfo convert(LegacyAclService.DomainObjectInfo oldInfo) {
+ if (oldInfo == null)
+ return null;
+ DomainObjectInfo newInfo = new DomainObjectInfo();
+ newInfo.setId(String.valueOf(oldInfo.getId()));
+ newInfo.setType(oldInfo.getType());
+ return newInfo;
+ }
+
+ private SidInfo convert(LegacyAclService.SidInfo oldInfo) {
+ if (oldInfo == null)
+ return null;
+ SidInfo newInfo = new SidInfo();
+ newInfo.setPrincipal(oldInfo.isPrincipal());
+ newInfo.setSid(oldInfo.getSid());
+ return newInfo;
+ }
+
+ private AceInfo convert(LegacyAclService.AceInfo oldInfo) {
+ if (oldInfo == null)
+ return null;
+ AceInfo newInfo = new AceInfo();
+ newInfo.setPermissionMask(oldInfo.getPermissionMask());
+ newInfo.setSidInfo(convert(oldInfo.getSidInfo()));
+ return newInfo;
+ }
+
+ private UserInfo convert(User user) {
+ if (user == null)
+ return null;
+ UserInfo newInfo = new UserInfo();
+ newInfo.setUsername(user.getUserName());
+ newInfo.setPassword(user.getPassword());
+ List<String> authorities = new ArrayList<>();
+ for (String auth : user.getAuthorities()) {
+ authorities.add(auth);
+ }
+ newInfo.setAuthorities(authorities);
+ return newInfo;
+ }
+
+ private User hbaseRowToUser(Result result) throws JsonParseException, JsonMappingException, IOException {
+ if (null == result || result.isEmpty())
+ return null;
+
+ String username = Bytes.toString(result.getRow());
+
+ byte[] valueBytes = result.getValue(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
+ LegacyUserService.UserGrantedAuthority[] deserialized = ugaSerializer.deserialize(valueBytes);
+
+ String password = "";
+ List<LegacyUserService.UserGrantedAuthority> authorities = Collections.emptyList();
+
+ // password is stored at [0] of authorities for backward compatibility
+ if (deserialized != null) {
+ if (deserialized.length > 0 && deserialized[0].getAuthority().startsWith(LegacyUserService.PWD_PREFIX)) {
+ password = deserialized[0].getAuthority().substring(LegacyUserService.PWD_PREFIX.length());
+ authorities = Arrays.asList(deserialized).subList(1, deserialized.length);
+ } else {
+ authorities = Arrays.asList(deserialized);
+ }
+ }
+ List<String> authoritiesStr = new ArrayList<>();
+ for (LegacyUserService.UserGrantedAuthority auth : authorities) {
+ if (auth != null) {
+ authoritiesStr.add(auth.getAuthority());
+ }
+ }
+ return new User(username, password, authoritiesStr);
+ }
+
+ interface ResultConverter {
+ void convertResult(ResultScanner rs, ResourceStore store) throws IOException;
+ }
+
+ public static class User {
+ String userName;
+
+ String password;
+
+ List<String> authorities;
+
+ public User(String userName, String password, List<String> authorities) {
+ this.userName = userName;
+ this.password = password;
+ this.authorities = authorities;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public List<String> getAuthorities() {
+ return authorities;
+ }
+
+ public void setAuthorities(List<String> authorities) {
+ this.authorities = authorities;
+ }
+ }
+}