You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/04/24 09:22:11 UTC
incubator-ranger git commit: Normalized configuration property names
Repository: incubator-ranger
Updated Branches:
refs/heads/master a3a472d99 -> cd7f5b57a
Normalized configuration property names
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/cd7f5b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/cd7f5b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/cd7f5b57
Branch: refs/heads/master
Commit: cd7f5b57a465b2162f4e732f69638149eee79d7b
Parents: a3a472d
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Fri Apr 22 13:52:33 2016 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Apr 24 00:09:59 2016 -0700
----------------------------------------------------------------------
.../conf/templates/installprop2xml.properties | 48 +++---
.../conf/templates/ranger-tagsync-template.xml | 36 +++--
tagsync/scripts/install.properties | 74 +++++----
tagsync/scripts/setup.py | 52 +++----
tagsync/scripts/updatetagadminpassword.py | 7 +-
.../ranger/tagsync/process/TagSyncConfig.java | 92 ++++-------
.../ranger/tagsync/process/TagSynchronizer.java | 153 +++++++++++++------
.../source/atlas/AtlasHiveResourceMapper.java | 11 +-
.../source/atlas/AtlasResourceMapper.java | 12 +-
.../tagsync/source/file/FileTagSource.java | 7 +-
.../main/resources/ranger-tagsync-default.xml | 33 +---
.../src/main/resources/ranger-tagsync-site.xml | 72 +++++++--
.../tagsync/process/TestTagSynchronizer.java | 37 +----
13 files changed, 341 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/conf/templates/installprop2xml.properties
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties
index 8f0ea75..27d3203 100644
--- a/tagsync/conf/templates/installprop2xml.properties
+++ b/tagsync/conf/templates/installprop2xml.properties
@@ -14,33 +14,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-unix_user = unix_user
-unix_group = unix_group
-TAG_SOURCE = TAG_SOURCE
-
-logdir = ranger.tagsync.logdir
TAGSYNC_ENABLED = ranger.tagsync.enabled
-TAGADMIN_ENDPOINT = ranger.tagsync.tagadmin.rest.url
-TAGADMIN_SSL_CONFIG_FILENAME = ranger.tagsync.tagadmin.rest.ssl.config.file
-#TAGADMIN_USERNAME = ranger.tagsync.tagadmin.basicauth.username
-#TAGADMIN_PASSWORD = ranger.tagsync.tagadmin.basicauth.password
+TAG_DEST_RANGER_ENDPOINT = ranger.tagsync.dest.ranger.endpoint
+TAG_DEST_RANGER_SSL_CONFIG_FILENAME = ranger.tagsync.dest.ranger.ssl.config.filename
+
+TAG_SOURCE_ATLAS_ENABLED = ranger.tagsync.source.atlas
+
+TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = atlas.kafka.bootstrap.servers
+TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = atlas.kafka.zookeeper.connect
+TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = atlas.kafka.entities.group.id
+
+TAG_SOURCE_ATLASREST_ENABLED = ranger.tagsync.source.atlasrest
+
+TAG_SOURCE_ATLASREST_ENDPOINT = ranger.tagsync.source.atlasrest.endpoint
+TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = ranger.tagsync.source.atlasrest.download.interval.millis
-TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.tagadmin.keystore
+TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file
-TAGSYNC_FILESOURCE_FILENAME = ranger.tagsync.filesource.filename
-TAG_SOURCE_FILE_MOD_TIME_CHECK_INTERVAL = ranger.tagsync.filesource.modtime.check.interval
+TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename
+TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = ranger.tagsync.source.file.check.interval.millis
-TAG_SOURCE_ATLAS_REST_URL = ranger.tagsync.atlasrestsource.endpoint
-TAG_SOURCE_ATLAS_REST_DOWNLOAD_INTERVAL = ranger.tagsync.atlasrestsource.download.interval
+TAG_SOURCE_NONE_ENABLED = ranger.tagsync.source.none
-TAGSYNC_ATLAS_KAFKA_ENDPOINTS = atlas.kafka.bootstrap.servers
-TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = atlas.kafka.zookeeper.connect
-TAGSYNC_ATLAS_CONSUMER_GROUP = atlas.kafka.entities.group.id
+TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service.mapping
+TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers
-TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.service.mapping
-TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.source.atlas.custom.resource.mappers
+TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.keystore.filename
+
+unix_user = unix_user
+unix_group = unix_group
+
+logdir = ranger.tagsync.logdir
tagsync_principal = ranger.tagsync.kerberos.principal
tagsync_keytab = ranger.tagsync.kerberos.keytab
+
+# TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to??
+hadoop_conf = hadoop_conf
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/conf/templates/ranger-tagsync-template.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml
index d82b6d1..03bb3fa 100644
--- a/tagsync/conf/templates/ranger-tagsync-template.xml
+++ b/tagsync/conf/templates/ranger-tagsync-template.xml
@@ -24,53 +24,57 @@
<value></value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.url</name>
+ <name>ranger.tagsync.dest.ranger.endpoint</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.filesource.modtime.check.interval</name>
+ <name>ranger.tagsync.dest.ranger.ssl.config.filename</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.atlasrestsource.download.interval</name>
+ <name>ranger.tagsync.source.atlas</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name>
+ <name>ranger.tagsync.source.atlasrest</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.filesource.filename</name>
+ <name>ranger.tagsync.source.atlasrest.endpoint</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.source.impl.class</name>
+ <name>ranger.tagsync.source.atlasrest.download.interval.millis</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.keystore</name>
+ <name>ranger.tagsync.source.file</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.alias</name>
+ <name>ranger.tagsync.source.file.filename</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.password</name>
+ <name>ranger.tagsync.source.file.check.interval.millis</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.atlas.to.service.mapping</name>
+ <name>ranger.tagsync.source.none</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.atlasrestsource.endpoint</name>
+ <name>ranger.tagsync.atlas.to.ranger.service.mapping</name>
<value></value>
</property>
<property>
- <name>ranger.tagsync.source.atlas.custom.resource.mappers</name>
+ <name>ranger.tagsync.atlas.custom.resource.mappers</name>
<value></value>
- </property>
+ </property>
+ <property>
+ <name>ranger.tagsync.keystore.filename</name>
+ <value></value>
+ </property>
<property>
<name>ranger.tagsync.kerberos.keytab</name>
<value></value>
@@ -78,5 +82,9 @@
<property>
<name>ranger.tagsync.kerberos.principal</name>
<value></value>
- </property>
+ </property>
+ <property>
+ <name>hadoop_conf</name>
+ <value></value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/install.properties
----------------------------------------------------------------------
diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties
index 695371d..90878d8 100644
--- a/tagsync/scripts/install.properties
+++ b/tagsync/scripts/install.properties
@@ -12,57 +12,37 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
#
-# This file provides list of deployment variables for the Ranger KMS Web Application
#
+# This file provides list of deployment variables for the Ranger KMS Web Application
#
#
-
-#User and group for the usersync process
-unix_user=ranger
-unix_group=ranger
-
#
-# The file where all credential is kept in cryptic format
#
-TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks
-
-# Logs are stored in logdir
-logdir = log
-
-# URL for TagAdmin
-TAGADMIN_ENDPOINT = http://localhost:6080
-
-#Set to run in kerberos environment
-tagsync_principal=
-tagsync_keytab=
-hadoop_conf=/etc/hadoop/conf
-
-# SSL config file name for TagAdmin
-TAGADMIN_SSL_CONFIG_FILENAME =
-
-# Source for tags (either 'atlas' or 'file' or 'atlasrest')
+# URL for tag destination - Ranger
+TAG_DEST_RANGER_ENDPOINT = http://localhost:6080
-TAG_SOURCE = atlas
-TAG_SOURCE_ATLAS_REST_URL = http://localhost:21000
+# SSL config file name for HTTPS messages to tag destination - Ranger
+TAG_DEST_RANGER_SSL_CONFIG_FILENAME =
-# Interval for checking the source for any changes in case of TAG_SOURCE = atlasrest
-TAG_SOURCE_ATLAS_REST_DOWNLOAD_INTERVAL = 900000
+TAG_SOURCE_ATLAS_ENABLED = true
-# File name to be used for reading tags information if TAG_SOURCE = file
+# Endpoint specifications needed by Atlas
+TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = localhost:6667
+TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = localhost:2181
+TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer
-TAGSYNC_FILESOURCE_FILENAME = /etc/ranger/data/tags.json
+TAG_SOURCE_ATLASREST_ENABLED = false
+TAG_SOURCE_ATLASREST_ENDPOINT = http://localhost:21000
+TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = 900000
-# Interval for checking the source for any changes in case of TAG_SOURCE = file
-TAG_SOURCE_FILE_MOD_TIME_CHECK_INTERVAL = 60000
+TAG_SOURCE_FILE_ENABLED = false
-# Endpoint specifications needed by Atlas
+TAG_SOURCE_FILE_FILENAME = /etc/ranger/data/tags.json
+TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000
-TAGSYNC_ATLAS_KAFKA_ENDPOINTS = localhost:6667
-TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = localhost:2181
-TAGSYNC_ATLAS_CONSUMER_GROUP = ranger_entities_consumer
+TAG_SOURCE_NONE_ENABLED = false
# Mapping from Atlas hive cluster-name to Ranger service-name
# this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2
@@ -87,4 +67,22 @@ TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING=
# RangerServiceResource structures are specified here. If there are no custom mappers,
# then it can be left blank
-TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS=
+TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS=
+
+#
+# The file where all credential is kept in cryptic format
+#
+TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks
+
+#User and group for the tagsync process
+unix_user=ranger
+unix_group=ranger
+
+# Logs are stored in logdir
+logdir = log
+
+#Set to run in kerberos environment
+tagsync_principal=
+tagsync_keytab=
+
+hadoop_conf=/etc/hadoop/conf
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/setup.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py
index 1e74a47..4404cdc 100755
--- a/tagsync/scripts/setup.py
+++ b/tagsync/scripts/setup.py
@@ -68,17 +68,22 @@ logFolderPermMode = 0777
rootOwnerId = 0
initPrefixList = ['S99', 'K00']
-TAG_SOURCE_KEY = 'TAG_SOURCE'
-TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAGSYNC_ATLAS_KAFKA_ENDPOINTS'
-TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT'
-TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAGSYNC_ATLAS_CONSUMER_GROUP'
+TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS'
+TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT'
+TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID'
TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping'
TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.'
TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.'
TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service'
-TAG_SOURCE_ATLAS = 'atlas'
-TAG_SOURCE_ATLASREST = 'atlasrest'
-TAG_SOURCE_FILE = 'file'
+
+TAG_SOURCE_ATLAS_ENABLED_KEY = 'TAG_SOURCE_ATLAS_ENABLED'
+TAG_SOURCE_ATLAS_ENABLED = 'ranger.tagsync.source.atlas'
+
+TAG_SOURCE_ATLASREST_ENABLED_KEY = 'TAG_SOURCE_ATLASREST_ENABLED'
+TAG_SOURCE_ATLASREST_ENABLED = 'ranger.tagsync.source.atlasrest'
+
+TAG_SOURCE_FILE_ENABLED_KEY = 'TAG_SOURCE_FILE_ENABLED'
+TAG_SOURCE_FILE_ENABLED = 'ranger.tagsync.source.file'
hadoopConfFileName = 'core-site.xml'
ENV_HADOOP_CONF_FILE = "ranger-tagsync-env-hadoopconfdir.sh"
@@ -218,13 +223,19 @@ def convertInstallPropsToXML(props):
else:
print "Direct Key not found:%s" % (k)
- ret['ranger.tagsync.sink.impl.class'] = 'org.apache.ranger.tagsync.sink.tagadmin.TagAdminRESTSink'
+ atlasOutFile.close()
- if (TAG_SOURCE_KEY in ret):
- ret['ranger.tagsync.source.impl.class'] = ret[TAG_SOURCE_KEY]
- del ret[TAG_SOURCE_KEY]
+ if (TAG_SOURCE_ATLAS_ENABLED_KEY in ret):
+ ret[TAG_SOURCE_ATLAS_ENABLED] = ret[TAG_SOURCE_ATLAS_ENABLED_KEY]
+ del ret[TAG_SOURCE_ATLAS_ENABLED_KEY]
- atlasOutFile.close()
+ if (TAG_SOURCE_ATLASREST_ENABLED_KEY in ret):
+ ret[TAG_SOURCE_ATLASREST_ENABLED] = ret[TAG_SOURCE_ATLASREST_ENABLED_KEY]
+ del ret[TAG_SOURCE_ATLASREST_ENABLED_KEY]
+
+ if (TAG_SOURCE_FILE_ENABLED_KEY in ret):
+ ret[TAG_SOURCE_FILE_ENABLED] = ret[TAG_SOURCE_FILE_ENABLED_KEY]
+ del ret[TAG_SOURCE_FILE_ENABLED_KEY]
return ret
@@ -333,18 +344,6 @@ def main():
fn = join(installTemplateDirName,templateFileName)
outfn = join(confFolderName, outputFileName)
- atlasOutFn = join(confFolderName, atlasApplicationPropFileName)
-
- atlasOutFile = file(atlasOutFn, "a")
-
- atlasOutFile.write("atlas.notification.embedded=false" + "\n")
- atlasOutFile.write("atlas.kafka.acks=1" + "\n")
- atlasOutFile.write("atlas.kafka.data=${sys:atlas.home}/data/kafka" + "\n")
- atlasOutFile.write("atlas.kafka.hook.group.id=atlas" + "\n")
-
- atlasOutFile.close()
-
-
if ( os.path.isdir(logFolderName) ):
logStat = os.stat(logFolderName)
logStat.st_uid
@@ -389,15 +388,14 @@ def main():
initializeInitD()
- tagsyncKSPath = mergeProps['ranger.tagsync.tagadmin.keystore']
+ tagsyncKSPath = mergeProps['ranger.tagsync.keystore.filename']
if (tagsyncKSPath == ''):
- mergeProps['ranger.tagsync.tagadmin.password'] = 'rangertagsync'
+ mergeProps['ranger.tagsync.dest.ranger.password'] = 'rangertagsync'
else:
tagadminPasswd = 'rangertagsync'
tagadminAlias = 'tagadmin.user.password'
- mergeProps['ranger.tagsync.tagadmin.alias'] = tagadminAlias
updatePropertyInJCKSFile(tagsyncKSPath,tagadminAlias,tagadminPasswd)
os.chown(tagsyncKSPath,ownerId,groupId)
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/scripts/updatetagadminpassword.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py
index ba0c911..445d89b 100644
--- a/tagsync/scripts/updatetagadminpassword.py
+++ b/tagsync/scripts/updatetagadminpassword.py
@@ -109,7 +109,7 @@ def main():
log("[I] Using Java:" + str(JAVA_BIN),"info")
globalDict=import_properties_from_xml(CFG_FILE,globalDict)
- TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.tagadmin.keystore']
+ TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.keystore.filename']
log("[I] TAGSYNC_KEYSTORE_FILENAME:" + str(TAGSYNC_KEYSTORE_FILENAME),"info")
TAGSYNC_TAGADMIN_ALIAS="tagadmin.user.password"
TAGSYNC_TAGADMIN_PASSWORD = ''
@@ -120,7 +120,7 @@ def main():
while TAGSYNC_TAGADMIN_PASSWORD == "":
TAGSYNC_TAGADMIN_PASSWORD=getpass.getpass("Enter tagadmin user password:")
- if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_ALIAS != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "":
+ if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "":
log("[I] Storing tagadmin tagsync password in credential store:","info")
cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,TAGSYNC_TAGADMIN_ALIAS,TAGSYNC_TAGADMIN_PASSWORD,TAGSYNC_KEYSTORE_FILENAME)
ret=subprocess.call(shlex.split(cmd))
@@ -129,8 +129,7 @@ def main():
ret=subprocess.call(shlex.split(cmd))
if ret == 0:
if os.path.isfile(CFG_FILE):
- write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.keystore",TAGSYNC_KEYSTORE_FILENAME)
- write_properties_to_xml(CFG_FILE,"ranger.tagsync.tagadmin.alias", TAGSYNC_TAGADMIN_ALIAS)
+ write_properties_to_xml(CFG_FILE,"ranger.tagsync.keystore.filename",TAGSYNC_KEYSTORE_FILENAME)
else:
log("[E] Required file not found: ["+CFG_FILE+"]","error")
else:
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
index b547b8f..d14a9ce 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
@@ -30,63 +30,58 @@ import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.Properties;
-//import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.credentialapi.CredentialReader;
public class TagSyncConfig extends Configuration {
private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ;
- public static final String CONFIG_FILE = "ranger-tagsync-site.xml";
+ private static final String CONFIG_FILE = "ranger-tagsync-site.xml";
- public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml";
+ private static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml";
+
+ private static final String CORE_SITE_FILE = "core-site.xml";
public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ;
public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ;
- private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url";
+ private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.dest.ranger.endpoint";
- private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file";
+ private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.dest.ranger.ssl.config.filename";
- public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.filesource.filename";
+ private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class";
- private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.filesource.modtime.check.interval";
+ private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password";
- private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.atlasrestsource.download.interval";
+ private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password";
- private static final String TAGSYNC_SOURCE_CLASS_PROP = "ranger.tagsync.source.impl.class";
+ private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval";
- private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.sink.impl.class";
+ private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.atlas.custom.resource.mappers";
- private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.atlasrestsource.endpoint";
+ private static final String TAGSYNC_ATLASSOURCE_ENDPOINT_PROP = "ranger.tagsync.source.atlasrest.endpoint";
- public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas.";
+ private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis";
- public static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service";
+ public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename";
- public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_";
+ private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis";
- private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.tagadmin.keystore";
- private static final String TAGSYNC_TAGADMIN_ALIAS_PROP = "ranger.tagsync.tagadmin.alias";
- private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.tagadmin.password";
- private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync";
- private static final String TAGSYNC_SOURCE_ATLAS_CUSTOM_RESOURCE_MAPPERS_PROP = "ranger.tagsync.source.atlas.custom.resource.mappers";
-
- private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000;
+ private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename";
- private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000;
-
- private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.tagadmin.connection.check.interval";
+ private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync";
private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 2000;
+ private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000;
+ private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000;
private static final String AUTH_TYPE = "hadoop.security.authentication";
private static final String NAME_RULES = "hadoop.security.auth_to_local";
private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal";
private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab";
- private static final String CORE_SITE_FILE = "core-site.xml";
private static String LOCAL_HOSTNAME = "unknown" ;
+
private Properties props;
static {
@@ -98,8 +93,7 @@ public class TagSyncConfig extends Configuration {
}
public static TagSyncConfig getInstance() {
- TagSyncConfig newConfig = new TagSyncConfig();
- return newConfig;
+ return new TagSyncConfig();
}
public Properties getProperties() {
@@ -195,8 +189,7 @@ public class TagSyncConfig extends Configuration {
}
static public String getTagSyncLogdir(Properties prop) {
- String val = prop.getProperty(TAGSYNC_LOGDIR_PROP);
- return val;
+ return prop.getProperty(TAGSYNC_LOGDIR_PROP);
}
static public long getTagSourceFileModTimeCheckIntervalInMillis(Properties prop) {
@@ -225,47 +218,28 @@ public class TagSyncConfig extends Configuration {
return ret;
}
- static public String getTagSourceClassName(String sourceName) {
- if (StringUtils.equalsIgnoreCase(sourceName, "atlas")) {
- return "org.apache.ranger.tagsync.source.atlas.AtlasTagSource";
- } else if (StringUtils.equalsIgnoreCase(sourceName, "file")) {
- return "org.apache.ranger.tagsync.source.file.FileTagSource";
- } else if (StringUtils.equalsIgnoreCase(sourceName, "atlasrest")) {
- return "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource";
- } else
- return sourceName;
- }
-
- static public String getTagSource(Properties prop) {
- return prop.getProperty(TAGSYNC_SOURCE_CLASS_PROP);
- }
-
static public String getTagSinkClassName(Properties prop) {
String val = prop.getProperty(TAGSYNC_SINK_CLASS_PROP);
- if (StringUtils.equalsIgnoreCase(val, "tagadmin")) {
+ if (StringUtils.equalsIgnoreCase(val, "ranger")) {
return "org.apache.ranger.tagsync.sink.tagadmin.TagAdminRESTSink";
} else
return val;
}
static public String getTagAdminRESTUrl(Properties prop) {
- String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP);
- return val;
+ return prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP);
}
static public String getTagAdminRESTSslConfigFile(Properties prop) {
- String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
- return val;
+ return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
}
static public String getTagSourceFileName(Properties prop) {
- String val = prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP);
- return val;
+ return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP);
}
static public String getAtlasEndpoint(Properties prop) {
- String val = prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP);
- return val;
+ return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP);
}
static public String getTagAdminPassword(Properties prop) {
@@ -277,18 +251,16 @@ public class TagSyncConfig extends Configuration {
return password;
}
}
- if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_KEYSTORE_PROP) && prop.containsKey(TAGSYNC_TAGADMIN_ALIAS_PROP)) {
+ if (prop != null && prop.containsKey(TAGSYNC_TAGADMIN_KEYSTORE_PROP)) {
String path = prop.getProperty(TAGSYNC_TAGADMIN_KEYSTORE_PROP);
- String alias = prop.getProperty(TAGSYNC_TAGADMIN_ALIAS_PROP, "tagadmin.user.password");
- if (path != null && alias != null) {
- if (!path.trim().isEmpty() && !alias.trim().isEmpty()) {
+ if (path != null) {
+ if (!path.trim().isEmpty()) {
try {
- password = CredentialReader.getDecryptedString(path.trim(), alias.trim());
+ password = CredentialReader.getDecryptedString(path.trim(), TAGSYNC_DEST_RANGER_PASSWORD_ALIAS);
} catch (Exception ex) {
password = null;
}
if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) {
- prop.setProperty(TAGSYNC_TAGADMIN_PASSWORD_PROP, password);
return password;
}
}
@@ -321,7 +293,7 @@ public class TagSyncConfig extends Configuration {
// return prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL);
String principal = null;
try {
- return SecureClientLogin.getPrincipal(prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL, ""), LOCAL_HOSTNAME);
+ principal = SecureClientLogin.getPrincipal(prop.getProperty(TAGSYNC_KERBEROS_PRICIPAL, ""), LOCAL_HOSTNAME);
} catch (IOException ignored) {
// do nothing
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
index 02fbc2c..4436a77 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
@@ -35,6 +35,9 @@ public class TagSynchronizer {
private List<TagSource> tagSources;
private Properties properties = null;
+ private static final String TAGSYNC_SOURCE_BASE = "ranger.tagsync.source.";
+ private static final String PROP_CLASS_NAME = "class";
+
private final Object shutdownNotifier = new Object();
private volatile boolean isShutdownInProgress = false;
@@ -90,29 +93,22 @@ public class TagSynchronizer {
boolean ret = false;
- String tagSourceNames = TagSyncConfig.getTagSource(properties);
-
- if (StringUtils.isNotBlank(tagSourceNames)) {
-
- LOG.info("Initializing TAG source and sink");
+ LOG.info("Initializing TAG source and sink");
- tagSink = initializeTagSink(properties);
+ tagSink = initializeTagSink(properties);
- if (tagSink != null) {
+ if (tagSink != null) {
- tagSources = initializeTagSources(tagSourceNames, properties);
+ tagSources = initializeTagSources(properties);
- for (TagSource tagSource : tagSources) {
- tagSource.setTagSink(tagSink);
- }
- ret = true;
+ for (TagSource tagSource : tagSources) {
+ tagSource.setTagSink(tagSink);
}
- } else {
- LOG.error("'ranger.tagsync.source.impl.class' value is not specified or is empty!");
+ ret = true;
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== TagSynchronizer.initialize(" + tagSourceNames + ") : " + ret);
+ LOG.debug("<== TagSynchronizer.initialize() : " + ret);
}
return ret;
@@ -184,7 +180,6 @@ public class TagSynchronizer {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagSynchronizer.initializeTagSink()");
}
-
TagSink ret = null;
try {
@@ -200,7 +195,6 @@ public class TagSynchronizer {
if (!ret.initialize(properties)) {
LOG.error("Failed to initialize TAG sink " + tagSinkClassName);
-
ret = null;
}
} catch (Throwable t) {
@@ -209,58 +203,127 @@ public class TagSynchronizer {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== TagSynchronizer.initializeTagSink()");
+ LOG.debug("<== TagSynchronizer.initializeTagSink(), result:" + (ret == null ? "false" : "true"));
}
return ret;
}
- static public List<TagSource> initializeTagSources(String tagSourceNames, Properties properties) {
+ static public List<TagSource> initializeTagSources(Properties properties) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> TagSynchronizer.initializeTagSources(" + tagSourceNames + ")");
+ LOG.debug("==> TagSynchronizer.initializeTagSources()");
}
List<TagSource> ret = new ArrayList<TagSource>();
- String[] tagSourcesArray = tagSourceNames.split(",");
-
- List<String> tagSourceList = Arrays.asList(tagSourcesArray);
-
- try {
- for (String tagSourceName : tagSourceList) {
+ List<String> tagSourceNameList = new ArrayList<String>();
- String tagSourceClassName = TagSyncConfig.getTagSourceClassName(tagSourceName.trim());
- if (LOG.isDebugEnabled()) {
- LOG.debug("tagSourceClassName=" + tagSourceClassName);
- }
+ for (Object propNameObj : properties.keySet()) {
+ String propName = propNameObj.toString();
+ if (!propName.startsWith(TAGSYNC_SOURCE_BASE)) {
+ continue;
+ }
+ String tagSourceName = propName.substring(TAGSYNC_SOURCE_BASE.length());
+ List<String> splits = toArray(tagSourceName, ".");
+ if (splits.size() > 1) {
+ continue;
+ }
+ String value = properties.getProperty(propName);
+ if (value.equalsIgnoreCase("enable")
+ || value.equalsIgnoreCase("enabled")
+ || value.equalsIgnoreCase("true")) {
+ tagSourceNameList.add(tagSourceName);
+ LOG.info("Tag source " + propName + " is set to "
+ + value);
+ }
+ }
- if (!StringUtils.equalsIgnoreCase(tagSourceClassName, TagSource.TAG_SOURCE_NONE)) {
- @SuppressWarnings("unchecked")
- Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(tagSourceClassName);
- TagSource tagSource = tagSourceClass.newInstance();
+ List<String> initializedTagSourceNameList = new ArrayList<String>();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created instance of " + tagSourceClassName);
- }
+ for (String tagSourceName : tagSourceNameList) {
+ String tagSourcePropPrefix = TAGSYNC_SOURCE_BASE + tagSourceName;
+ TagSource tagSource = getTagSourceFromConfig(properties,
+ tagSourcePropPrefix, tagSourceName);
+ if (tagSource != null) {
+ try {
if (!tagSource.initialize(properties)) {
- LOG.error("Failed to initialize TAG source " + tagSourceClassName);
-
+ LOG.error("Failed to initialize TAG source " + tagSourceName);
ret.clear();
break;
+ } else {
+ ret.add(tagSource);
+ initializedTagSourceNameList.add(tagSourceName);
}
- ret.add(tagSource);
+ } catch(Exception exception) {
+ LOG.error("tag-source:" + tagSourceName + " initialization failed with ", exception);
+ ret.clear();
+ break;
}
}
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagSynchronizer.initializeTagSources(" + initializedTagSourceNameList + ")");
+ }
+ return ret;
+ }
+
+ static private TagSource getTagSourceFromConfig(Properties props,
+ String propPrefix, String tagSourceName) {
+ TagSource tagSource = null;
+ String className = getStringProperty(props, propPrefix + "."
+ + PROP_CLASS_NAME);
+ if (StringUtils.isBlank(className)) {
+ if (tagSourceName.equals("file")) {
+ className = "org.apache.ranger.tagsync.source.file.FileTagSource";
+ } else if (tagSourceName.equalsIgnoreCase("atlas")) {
+ className = "org.apache.ranger.tagsync.source.atlas.AtlasTagSource";
+ } else if (tagSourceName.equals("atlasrest")) {
+ className = "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource";
+ } else if (!tagSourceName.equalsIgnoreCase(TagSource.TAG_SOURCE_NONE)) {
+ LOG.error("tagSource name doesn't have any class associated with it. tagSourceName="
+ + tagSourceName + ", propertyPrefix=" + propPrefix);
+ }
+ }
+ if (StringUtils.isNotBlank(className)) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<TagSource> tagSourceClass = (Class<TagSource>) Class.forName(className);
- } catch (Throwable t) {
- LOG.error("Failed to initialize TAG sources. Error details: ", t);
- ret.clear();
+ tagSource = tagSourceClass.newInstance();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created instance of " + className);
+ }
+ } catch (Exception e) {
+ LOG.fatal("Can't instantiate tagSource class for tagSourceName="
+ + tagSourceName + ", className=" + className
+ + ", propertyPrefix=" + propPrefix, e);
+ }
}
+ return tagSource;
+ }
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== TagSynchronizer.initializeTagSources(" + tagSourceNames + ")");
+ private static String getStringProperty(Properties props, String propName) {
+ String ret = null;
+
+ if (props != null && propName != null) {
+ String val = props.getProperty(propName);
+ if (val != null) {
+ ret = val;
+ }
}
return ret;
}
+
+ private static List<String> toArray(String destListStr, String delim) {
+ List<String> list = new ArrayList<String>();
+ if (destListStr != null && !destListStr.isEmpty()) {
+ StringTokenizer tokenizer = new StringTokenizer(destListStr,
+ delim.trim());
+ while (tokenizer.hasMoreTokens()) {
+ list.add(tokenizer.nextToken());
+ }
+ }
+ return list;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java
index 9a6fc13..a318f3b 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceResource;
-import org.apache.ranger.tagsync.process.TagSyncConfig;
import java.util.Map;
import java.util.HashMap;
@@ -48,11 +47,13 @@ public class AtlasHiveResourceMapper extends AtlasResourceMapper {
public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE = "name";
- static protected final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName";
+ public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName";
- private static String clusterDelimiter = "@";
+ public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_";
- private static String qualifiedNameDelimiter = "\\.";
+ public static final String clusterDelimiter = "@";
+
+ public static final String qualifiedNameDelimiter = "\\.";
public static final String[] supportedEntityTypes = { ENTITY_TYPE_HIVE_DB, ENTITY_TYPE_HIVE_TABLE, ENTITY_TYPE_HIVE_COLUMN };
@@ -139,7 +140,7 @@ public class AtlasHiveResourceMapper extends AtlasResourceMapper {
String ret = getRangerServiceName(COMPONENT_NAME, clusterName);
if (StringUtils.isBlank(ret)) {
- ret = clusterName + TagSyncConfig.TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + COMPONENT_NAME;
+ ret = clusterName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + COMPONENT_NAME;
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
index fd94928..d47d052 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java
@@ -24,7 +24,6 @@ import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.model.RangerServiceResource;
-import org.apache.ranger.tagsync.process.TagSyncConfig;
import java.util.Properties;
import java.util.List;
@@ -33,6 +32,11 @@ import java.util.Map;
public abstract class AtlasResourceMapper {
private static final Log LOG = LogFactory.getLog(AtlasResourceMapper.class);
+ protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas.";
+
+ protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service";
+
+ protected static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = ".instance.";
protected Properties properties;
public AtlasResourceMapper() {
@@ -48,9 +52,9 @@ public abstract class AtlasResourceMapper {
protected String getRangerServiceName(String componentName, String atlasInstanceName) {
- String propName = TagSyncConfig.TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName
- + ".instance." + atlasInstanceName
- + TagSyncConfig.TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
+ String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName
+ + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + atlasInstanceName
+ + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
return properties.getProperty(propName);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java
index e22681e..5f444b8 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java
@@ -46,7 +46,6 @@ public class FileTagSource extends AbstractTagSource implements Runnable {
private long lastModifiedTimeInMillis = -1L;
private Gson gsonBuilder;
- private Properties properties;
private long fileModTimeCheckIntervalInMs;
private Thread myThread = null;
@@ -99,11 +98,13 @@ public class FileTagSource extends AbstractTagSource implements Runnable {
LOG.debug("==> FileTagSource.initialize()");
}
+ Properties properties;
+
if (props == null || MapUtils.isEmpty(props)) {
LOG.error("No properties specified for FileTagSource initialization");
- this.properties = new Properties();
+ properties = new Properties();
} else {
- this.properties = props;
+ properties = props;
}
gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/resources/ranger-tagsync-default.xml
----------------------------------------------------------------------
diff --git a/tagsync/src/main/resources/ranger-tagsync-default.xml b/tagsync/src/main/resources/ranger-tagsync-default.xml
index 3784df8..131e319 100644
--- a/tagsync/src/main/resources/ranger-tagsync-default.xml
+++ b/tagsync/src/main/resources/ranger-tagsync-default.xml
@@ -14,8 +14,6 @@
limitations under the License. See accompanying LICENSE file.
-->
-<!-- Put site-specific property overrides in this file. -->
-
<configuration>
<property>
<name>ranger.tagsync.enabled</name>
@@ -26,38 +24,17 @@
<value>log</value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.url</name>
- <value></value>
+ <name>ranger.tagsync.dest.ranger.impl.class</name>
+ <value>ranger</value>
<description></description>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name>
- <value></value>
+ <name>ranger.tagsync.dest.ranger.endpoint</name>
+ <value>http://localhost:6080</value>
<description></description>
</property>
<property>
- <name>ranger.tagsync.filesource.modtime.check.interval</name>
- <value>60000</value>
- <description></description>
- </property>
- <property>
- <name>ranger.tagsync.filesource.filename</name>
+ <name>ranger.tagsync.dest.ranger.password</name>
<value></value>
- <description></description>
- </property>
- <property>
- <name>ranger.tagsync.source.impl.class</name>
- <value></value>
- <description></description>
- </property>
- <property>
- <name>ranger.tagsync.sink.impl.class</name>
- <value>tagadmin</value>
- <description></description>
- </property>
- <property>
- <name>ranger.tagsync.atlasrestsource.download.interval</name>
- <value>600000</value>
- <description></description>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/main/resources/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml
index 44a9641..fe300b4 100644
--- a/tagsync/src/main/resources/ranger-tagsync-site.xml
+++ b/tagsync/src/main/resources/ranger-tagsync-site.xml
@@ -19,35 +19,77 @@
<configuration>
<property>
<name>ranger.tagsync.enabled</name>
- <value>true</value>
+ <value />
</property>
<property>
<name>ranger.tagsync.logdir</name>
<value>log</value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.url</name>
- <value>localhost:6080</value>
- <description></description>
+ <name>ranger.tagsync.dest.ranger.endpoint</name>
+ <value>http://localhost:6080</value>
</property>
<property>
- <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name>
- <value></value>
- <description></description>
+ <name>ranger.tagsync.dest.ranger.ssl.config.filename</name>
+ <value />
</property>
<property>
- <name>ranger.tagsync.filesource.modtime.check.interval</name>
- <value>60000</value>
- <description></description>
+ <name>ranger.tagsync.source.atlas</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.endpoint</name>
+ <value>http://localhost:21000</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.atlasrest.download.interval.millis</name>
+ <value>900000</value>
</property>
<property>
- <name>ranger.tagsync.filesource.filename</name>
+ <name>ranger.tagsync.source.file</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.file.filename</name>
<value>/etc/ranger/data/tags.json</value>
- <description></description>
</property>
<property>
- <name>ranger.tagsync.source.impl.class</name>
- <value>File</value>
- <description></description>
+ <name>ranger.tagsync.source.file.check.interval.millis</name>
+ <value>60000</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.none</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.atlas.to.ranger.service.mapping</name>
+ <value />
+ </property>
+ <property>
+ <name>ranger.tagsync.atlas.custom.resource.mappers</name>
+ <value />
+ </property>
+ <property>
+ <name>ranger.tagsync.keystore.filename</name>
+ <value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value>
+ </property>
+ <property>
+ <name>ranger.tagsync.kerberos.keytab</name>
+ <value />
</property>
+ <property>
+ <name>ranger.tagsync.kerberos.principal</name>
+ <value />
+ </property>
+ <property>
+ <name>ranger.tagsync.dest.ranger.password</name>
+ <value>rangertagsync</value>
+ </property>
+
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cd7f5b57/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
index 43c22d0..c644e0d 100644
--- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
+++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
@@ -60,43 +60,18 @@ public class TestTagSynchronizer {
System.out.println("testTagSynchronizer() called");
- //tagSynchronizer.run();
-
- tagSynchronizer.shutdown("From testTagSynchronizer: time=up");
-
- System.out.println("Exiting test");
-
-
- }
-
- @Test
- public void testTagDownload() {
-
- boolean initDone = true;
-
- /* For tagSynchronizer.initialize() to succeed, edit ranger-tagsync-site.xml file to contain correct
- values of the following properties:
- ranger.tagsync.tagadmin.rest.url, ranger.tagsync.tagadmin.password
-
- For example:
- <property>
- <name>ranger.tagsync.tagadmin.rest.url</name>
- <value>http://tagsync-test:6080</value>
- </property>
- <property>
- <name>ranger.tagsync.tagadmin.password</name>
- <value>rangertagsync</value>
- </property>
- */
+ boolean initDone = tagSynchronizer.initialize();
+ System.out.println("TagSynchronizer initialization result=" + initDone);
-// initDone = tagSynchronizer.initialize(null);
+ if (initDone) {
+ tagSynchronizer.shutdown("From testTagSynchronizer: time=up");
+ }
- System.out.println("TagSynchronizer initialization result=" + initDone);
+ System.out.println("Exiting test");
assert(initDone);
- System.out.println("Exiting testTagDownload()");
}
@Test